← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_pack_repository.py

1st cut merge of bzr.dev r3907

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_pack_repository.py
22
22
from cStringIO import StringIO
23
23
from stat import S_ISDIR
24
24
 
25
 
from bzrlib.index import GraphIndex, InMemoryGraphIndex
 
25
from bzrlib.btree_index import BTreeGraphIndex
 
26
from bzrlib.index import GraphIndex
26
27
from bzrlib import (
27
28
    bzrdir,
28
29
    errors,
36
37
    upgrade,
37
38
    workingtree,
38
39
    )
 
40
from bzrlib.smart import (
 
41
    client,
 
42
    server,
 
43
    )
39
44
from bzrlib.tests import (
40
45
    TestCase,
41
46
    TestCaseWithTransport,
44
49
    )
45
50
from bzrlib.transport import (
46
51
    fakenfs,
 
52
    memory,
47
53
    get_transport,
48
54
    )
 
55
from bzrlib.tests.per_repository import TestCaseWithRepository
49
56
 
50
57
 
51
58
class TestPackRepository(TestCaseWithTransport):
65
72
        """Packs do not need ordered data retrieval."""
66
73
        format = self.get_format()
67
74
        repo = self.make_repository('.', format=format)
68
 
        self.assertEqual('unsorted', repo._fetch_order)
 
75
        self.assertEqual('unordered', repo._fetch_order)
69
76
 
70
77
    def test_attribute__fetch_uses_deltas(self):
71
78
        """Packs reuse deltas."""
111
118
        self.assertFalse(t.has('knits'))
112
119
        # revision-indexes file-container directory
113
120
        self.assertEqual([],
114
 
            list(GraphIndex(t, 'pack-names', None).iter_all_entries()))
 
121
            list(self.index_class(t, 'pack-names', None).iter_all_entries()))
115
122
        self.assertTrue(S_ISDIR(t.stat('packs').st_mode))
116
123
        self.assertTrue(S_ISDIR(t.stat('upload').st_mode))
117
124
        self.assertTrue(S_ISDIR(t.stat('indices').st_mode))
152
159
        tree = self.make_branch_and_tree('.', format=format)
153
160
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
154
161
        self.assertEqual([],
155
 
            list(GraphIndex(trans, 'pack-names', None).iter_all_entries()))
 
162
            list(self.index_class(trans, 'pack-names', None).iter_all_entries()))
156
163
        tree.commit('foobarbaz')
157
 
        index = GraphIndex(trans, 'pack-names', None)
 
164
        index = self.index_class(trans, 'pack-names', None)
158
165
        index_nodes = list(index.iter_all_entries())
159
166
        self.assertEqual(1, len(index_nodes))
160
167
        node = index_nodes[0]
173
180
        tree1.branch.repository.fetch(tree2.branch.repository)
174
181
        trans = tree1.branch.repository.bzrdir.get_repository_transport(None)
175
182
        self.assertEqual([],
176
 
            list(GraphIndex(trans, 'pack-names', None).iter_all_entries()))
 
183
            list(self.index_class(trans, 'pack-names', None).iter_all_entries()))
177
184
 
178
185
    def test_commit_across_pack_shape_boundary_autopacks(self):
179
186
        format = self.get_format()
187
194
        for x in range(9):
188
195
            tree.commit('commit %s' % x)
189
196
        # there should be 9 packs:
190
 
        index = GraphIndex(trans, 'pack-names', None)
 
197
        index = self.index_class(trans, 'pack-names', None)
191
198
        self.assertEqual(9, len(list(index.iter_all_entries())))
192
199
        # insert some files in obsolete_packs which should be removed by pack.
193
200
        trans.put_bytes('obsolete_packs/foo', '123')
194
201
        trans.put_bytes('obsolete_packs/bar', '321')
195
202
        # committing one more should coalesce to 1 of 10.
196
203
        tree.commit('commit triggering pack')
197
 
        index = GraphIndex(trans, 'pack-names', None)
 
204
        index = self.index_class(trans, 'pack-names', None)
198
205
        self.assertEqual(1, len(list(index.iter_all_entries())))
199
206
        # packing should not damage data
200
207
        tree = tree.bzrdir.open_workingtree()
210
217
        large_pack_name = list(index.iter_all_entries())[0][1][0]
211
218
        # finally, committing again should not touch the large pack.
212
219
        tree.commit('commit not triggering pack')
213
 
        index = GraphIndex(trans, 'pack-names', None)
 
220
        index = self.index_class(trans, 'pack-names', None)
214
221
        self.assertEqual(2, len(list(index.iter_all_entries())))
215
222
        pack_names = [node[1][0] for node in index.iter_all_entries()]
216
223
        self.assertTrue(large_pack_name in pack_names)
239
246
        tree.commit('more work')
240
247
        tree.branch.repository.pack()
241
248
        # there should be 1 pack:
242
 
        index = GraphIndex(trans, 'pack-names', None)
 
249
        index = self.index_class(trans, 'pack-names', None)
243
250
        self.assertEqual(1, len(list(index.iter_all_entries())))
244
251
        self.assertEqual(2, len(tree.branch.repository.all_revision_ids()))
245
252
 
394
401
        finally:
395
402
            r1.unlock()
396
403
 
 
404
    def test_concurrent_pack_triggers_reload(self):
 
405
        # create 2 packs, which we will then collapse
 
406
        tree = self.make_branch_and_tree('tree')
 
407
        tree.lock_write()
 
408
        try:
 
409
            rev1 = tree.commit('one')
 
410
            rev2 = tree.commit('two')
 
411
            r2 = repository.Repository.open('tree')
 
412
            r2.lock_read()
 
413
            try:
 
414
                # Now r2 has read the pack-names file, but will need to reload
 
415
                # it after r1 has repacked
 
416
                tree.branch.repository.pack()
 
417
                self.assertEqual({rev2:(rev1,)}, r2.get_parent_map([rev2]))
 
418
            finally:
 
419
                r2.unlock()
 
420
        finally:
 
421
            tree.unlock()
 
422
 
 
423
    def test_concurrent_pack_during_get_record_reloads(self):
 
424
        tree = self.make_branch_and_tree('tree')
 
425
        tree.lock_write()
 
426
        try:
 
427
            rev1 = tree.commit('one')
 
428
            rev2 = tree.commit('two')
 
429
            keys = [(rev1,), (rev2,)]
 
430
            r2 = repository.Repository.open('tree')
 
431
            r2.lock_read()
 
432
            try:
 
433
                # At this point, we will start grabbing a record stream, and
 
434
                # trigger a repack mid-way
 
435
                packed = False
 
436
                result = {}
 
437
                record_stream = r2.revisions.get_record_stream(keys,
 
438
                                    'unordered', False)
 
439
                for record in record_stream:
 
440
                    result[record.key] = record
 
441
                    if not packed:
 
442
                        tree.branch.repository.pack()
 
443
                        packed = True
 
444
                # The first record will be found in the original location, but
 
445
                # after the pack, we have to reload to find the next record
 
446
                self.assertEqual(sorted(keys), sorted(result.keys()))
 
447
            finally:
 
448
                r2.unlock()
 
449
        finally:
 
450
            tree.unlock()
 
451
 
397
452
    def test_lock_write_does_not_physically_lock(self):
398
453
        repo = self.make_repository('.', format=self.get_format())
399
454
        repo.lock_write()
413
468
    def test_break_lock_breaks_physical_lock(self):
414
469
        repo = self.make_repository('.', format=self.get_format())
415
470
        repo._pack_collection.lock_names()
 
471
        repo.control_files.leave_in_place()
 
472
        repo.unlock()
416
473
        repo2 = repository.Repository.open('.')
417
474
        self.assertTrue(repo.get_physical_lock_status())
418
475
        self.prepare_for_break_lock()
481
538
        self.assertEqual(self.format_supports_external_lookups,
482
539
            repo._format.supports_external_lookups)
483
540
 
 
541
    def test_abort_write_group_does_not_raise_when_suppressed(self):
 
542
        """Similar to per_repository.test_write_group's test of the same name.
 
543
 
 
544
        Also requires that the exception is logged.
 
545
        """
 
546
        self.vfs_transport_factory = memory.MemoryServer
 
547
        repo = self.make_repository('repo')
 
548
        token = repo.lock_write()
 
549
        self.addCleanup(repo.unlock)
 
550
        repo.start_write_group()
 
551
        # Damage the repository on the filesystem
 
552
        self.get_transport('').rename('repo', 'foo')
 
553
        # abort_write_group will not raise an error
 
554
        self.assertEqual(None, repo.abort_write_group(suppress_errors=True))
 
555
        # But it does log an error
 
556
        log_file = self._get_log(keep_log_file=True)
 
557
        self.assertContainsRe(log_file, 'abort_write_group failed')
 
558
        self.assertContainsRe(log_file, r'INFO  bzr: ERROR \(ignored\):')
 
559
        if token is not None:
 
560
            repo.leave_lock_in_place()
 
561
        
 
562
    def test_abort_write_group_does_raise_when_not_suppressed(self):
 
563
        self.vfs_transport_factory = memory.MemoryServer
 
564
        repo = self.make_repository('repo')
 
565
        token = repo.lock_write()
 
566
        self.addCleanup(repo.unlock)
 
567
        repo.start_write_group()
 
568
        # Damage the repository on the filesystem
 
569
        self.get_transport('').rename('repo', 'foo')
 
570
        # abort_write_group will not raise an error
 
571
        self.assertRaises(Exception, repo.abort_write_group)
 
572
        if token is not None:
 
573
            repo.leave_lock_in_place()
 
574
        
484
575
 
485
576
class TestPackRepositoryStacking(TestCaseWithTransport):
486
577
 
495
586
    def get_format(self):
496
587
        return bzrdir.format_registry.make_bzrdir(self.format_name)
497
588
 
498
 
    def test_stack_checks_compatibility(self):
 
589
    def test_stack_checks_rich_root_compatibility(self):
499
590
        # early versions of the packing code relied on pack internals to
500
591
        # stack, but the current version should be able to stack on any
501
592
        # format.
507
598
        if repo.supports_rich_root():
508
599
            # can only stack on repositories that have compatible internal
509
600
            # metadata
510
 
            matching_format_name = 'pack-0.92-subtree'
 
601
            if getattr(repo._format, 'supports_tree_reference', False):
 
602
                matching_format_name = 'pack-0.92-subtree'
 
603
            else:
 
604
                matching_format_name = 'rich-root-pack'
511
605
            mismatching_format_name = 'pack-0.92'
512
606
        else:
513
607
            matching_format_name = 'pack-0.92'
524
618
            r'KnitPackRepository.*/repo/.*\n'
525
619
            r'different rich-root support')
526
620
 
 
621
    def test_stack_checks_serializers_compatibility(self):
 
622
        repo = self.make_repository('repo', format=self.get_format())
 
623
        if getattr(repo._format, 'supports_tree_reference', False):
 
624
            # can only stack on repositories that have compatible internal
 
625
            # metadata
 
626
            matching_format_name = 'pack-0.92-subtree'
 
627
            mismatching_format_name = 'rich-root-pack'
 
628
        else:
 
629
            if repo.supports_rich_root():
 
630
                matching_format_name = 'rich-root-pack'
 
631
                mismatching_format_name = 'pack-0.92-subtree'
 
632
            else:
 
633
                raise TestNotApplicable('No formats use non-v5 serializer'
 
634
                    ' without having rich-root also set')
 
635
        base = self.make_repository('base', format=matching_format_name)
 
636
        repo.add_fallback_repository(base)
 
637
        # you can't stack on something with incompatible data
 
638
        bad_repo = self.make_repository('mismatch',
 
639
            format=mismatching_format_name)
 
640
        e = self.assertRaises(errors.IncompatibleRepositories,
 
641
            repo.add_fallback_repository, bad_repo)
 
642
        self.assertContainsRe(str(e),
 
643
            r'(?m)KnitPackRepository.*/mismatch/.*\nis not compatible with\n'
 
644
            r'KnitPackRepository.*/repo/.*\n'
 
645
            r'different serializers')
 
646
 
527
647
    def test_adding_pack_does_not_record_pack_names_from_other_repositories(self):
528
648
        base = self.make_branch_and_tree('base', format=self.get_format())
529
649
        base.commit('foo')
550
670
        for x in range(9):
551
671
            tree.commit('commit %s' % x)
552
672
        # there should be 9 packs:
553
 
        index = GraphIndex(trans, 'pack-names', None)
 
673
        index = self.index_class(trans, 'pack-names', None)
554
674
        self.assertEqual(9, len(list(index.iter_all_entries())))
555
675
        # committing one more should coalesce to 1 of 10.
556
676
        tree.commit('commit triggering pack')
557
 
        index = GraphIndex(trans, 'pack-names', None)
 
677
        index = self.index_class(trans, 'pack-names', None)
558
678
        self.assertEqual(1, len(list(index.iter_all_entries())))
559
679
        # packing should not damage data
560
680
        tree = tree.bzrdir.open_workingtree()
570
690
        large_pack_name = list(index.iter_all_entries())[0][1][0]
571
691
        # finally, committing again should not touch the large pack.
572
692
        tree.commit('commit not triggering pack')
573
 
        index = GraphIndex(trans, 'pack-names', None)
 
693
        index = self.index_class(trans, 'pack-names', None)
574
694
        self.assertEqual(2, len(list(index.iter_all_entries())))
575
695
        pack_names = [node[1][0] for node in index.iter_all_entries()]
576
696
        self.assertTrue(large_pack_name in pack_names)
577
697
 
578
698
 
 
699
class TestSmartServerAutopack(TestCaseWithTransport):
 
700
 
 
701
    def setUp(self):
 
702
        super(TestSmartServerAutopack, self).setUp()
 
703
        # Create a smart server that publishes whatever the backing VFS server
 
704
        # does.
 
705
        self.smart_server = server.SmartTCPServer_for_testing()
 
706
        self.smart_server.setUp(self.get_server())
 
707
        self.addCleanup(self.smart_server.tearDown)
 
708
        # Log all HPSS calls into self.hpss_calls.
 
709
        client._SmartClient.hooks.install_named_hook(
 
710
            'call', self.capture_hpss_call, None)
 
711
        self.hpss_calls = []
 
712
 
 
713
    def capture_hpss_call(self, params):
 
714
        self.hpss_calls.append(params.method)
 
715
 
 
716
    def get_format(self):
 
717
        return bzrdir.format_registry.make_bzrdir(self.format_name)
 
718
 
 
719
    def test_autopack_rpc_is_used_when_using_hpss(self):
 
720
        # Make local and remote repos
 
721
        tree = self.make_branch_and_tree('local', format=self.get_format())
 
722
        self.make_branch_and_tree('remote', format=self.get_format())
 
723
        remote_branch_url = self.smart_server.get_url() + 'remote'
 
724
        remote_branch = bzrdir.BzrDir.open(remote_branch_url).open_branch()
 
725
        # Make 9 local revisions, and push them one at a time to the remote
 
726
        # repo to produce 9 pack files.
 
727
        for x in range(9):
 
728
            tree.commit('commit %s' % x)
 
729
            tree.branch.push(remote_branch)
 
730
        # Make one more push to trigger an autopack
 
731
        self.hpss_calls = []
 
732
        tree.commit('commit triggering pack')
 
733
        tree.branch.push(remote_branch)
 
734
        self.assertTrue('PackRepository.autopack' in self.hpss_calls)
 
735
 
 
736
 
579
737
def load_tests(basic_tests, module, test_loader):
580
738
    # these give the bzrdir canned format name, and the repository on-disk
581
739
    # format string
582
740
    scenarios_params = [
583
741
         dict(format_name='pack-0.92',
584
742
              format_string="Bazaar pack repository format 1 (needs bzr 0.92)\n",
585
 
              format_supports_external_lookups=False),
 
743
              format_supports_external_lookups=False,
 
744
              index_class=GraphIndex),
586
745
         dict(format_name='pack-0.92-subtree',
587
746
              format_string="Bazaar pack repository format 1 "
588
747
              "with subtree support (needs bzr 0.92)\n",
589
 
              format_supports_external_lookups=False),
 
748
              format_supports_external_lookups=False,
 
749
              index_class=GraphIndex),
590
750
         dict(format_name='1.6',
591
751
              format_string="Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n",
592
 
              format_supports_external_lookups=True),
593
 
         dict(format_name='1.6-rich-root',
 
752
              format_supports_external_lookups=True,
 
753
              index_class=GraphIndex),
 
754
         dict(format_name='1.6.1-rich-root',
594
755
              format_string="Bazaar RepositoryFormatKnitPack5RichRoot "
595
 
                  "(bzr 1.6)\n",
596
 
              format_supports_external_lookups=True),
597
 
         dict(format_name='development0',
598
 
              format_string="Bazaar development format 0 "
599
 
                  "(needs bzr.dev from before 1.3)\n",
600
 
              format_supports_external_lookups=False),
601
 
         dict(format_name='development0-subtree',
602
 
              format_string="Bazaar development format 0 "
603
 
                  "with subtree support (needs bzr.dev from before 1.3)\n",
604
 
              format_supports_external_lookups=False),
605
 
         dict(format_name='development',
606
 
              format_string="Bazaar development format 1 "
607
 
                  "(needs bzr.dev from before 1.6)\n",
608
 
              format_supports_external_lookups=True),
609
 
         dict(format_name='development-subtree',
610
 
              format_string="Bazaar development format 1 "
611
 
                  "with subtree support (needs bzr.dev from before 1.6)\n",
612
 
              format_supports_external_lookups=True),
 
756
                  "(bzr 1.6.1)\n",
 
757
              format_supports_external_lookups=True,
 
758
              index_class=GraphIndex),
 
759
         dict(format_name='1.9',
 
760
              format_string="Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n",
 
761
              format_supports_external_lookups=True,
 
762
              index_class=BTreeGraphIndex),
 
763
         dict(format_name='1.9-rich-root',
 
764
              format_string="Bazaar RepositoryFormatKnitPack6RichRoot "
 
765
                  "(bzr 1.9)\n",
 
766
              format_supports_external_lookups=True,
 
767
              index_class=BTreeGraphIndex),
 
768
         dict(format_name='development2',
 
769
              format_string="Bazaar development format 2 "
 
770
                  "(needs bzr.dev from before 1.8)\n",
 
771
              format_supports_external_lookups=True,
 
772
              index_class=BTreeGraphIndex),
 
773
         dict(format_name='development2-subtree',
 
774
              format_string="Bazaar development format 2 "
 
775
                  "with subtree support (needs bzr.dev from before 1.8)\n",
 
776
              format_supports_external_lookups=True,
 
777
              index_class=BTreeGraphIndex),
613
778
         ]
614
779
    adapter = tests.TestScenarioApplier()
615
780
    # name of the scenario is the format name