~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_repository.py

  • Committer: Matthew Revell
  • Date: 2008-03-05 14:11:35 UTC
  • mto: This revision was merged to the branch mainline in revision 3263.
  • Revision ID: matthew.revell@canonical.com-20080305141135-96jf8j24b9crru6v
Added an admin-guide directory to doc/en and outlined headings for the admin guide in index.txt

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2006, 2007 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
16
 
17
17
"""Tests for the Repository facility that are not interface tests.
18
18
 
19
 
For interface tests see tests/per_repository/*.py.
 
19
For interface tests see tests/repository_implementations/*.py.
20
20
 
21
21
For concrete class tests see this file, and for storage formats tests
22
22
also see this file.
23
23
"""
24
24
 
 
25
import md5
25
26
from stat import S_ISDIR
26
27
from StringIO import StringIO
27
 
import sys
28
28
 
29
29
import bzrlib
30
30
from bzrlib.errors import (NotBranchError,
32
32
                           UnknownFormatError,
33
33
                           UnsupportedFormatError,
34
34
                           )
35
 
from bzrlib import (
36
 
    graph,
37
 
    tests,
38
 
    )
39
 
from bzrlib.branchbuilder import BranchBuilder
40
 
from bzrlib.btree_index import BTreeBuilder, BTreeGraphIndex
 
35
from bzrlib import graph
41
36
from bzrlib.index import GraphIndex, InMemoryGraphIndex
42
37
from bzrlib.repository import RepositoryFormat
43
38
from bzrlib.smart import server
44
39
from bzrlib.tests import (
45
40
    TestCase,
46
41
    TestCaseWithTransport,
47
 
    TestSkipped,
48
42
    test_knit,
49
43
    )
50
 
from bzrlib.transport import (
51
 
    fakenfs,
52
 
    get_transport,
53
 
    )
 
44
from bzrlib.transport import get_transport
 
45
from bzrlib.transport.memory import MemoryServer
 
46
from bzrlib.util import bencode
54
47
from bzrlib import (
55
 
    bencode,
56
48
    bzrdir,
57
49
    errors,
58
50
    inventory,
59
 
    osutils,
60
51
    progress,
61
52
    repository,
62
53
    revision as _mod_revision,
63
54
    symbol_versioning,
64
55
    upgrade,
65
 
    versionedfile,
66
56
    workingtree,
67
57
    )
68
 
from bzrlib.repofmt import (
69
 
    groupcompress_repo,
70
 
    knitrepo,
71
 
    pack_repo,
72
 
    weaverepo,
73
 
    )
 
58
from bzrlib.repofmt import knitrepo, weaverepo, pack_repo
74
59
 
75
60
 
76
61
class TestDefaultFormat(TestCase):
105
90
class SampleRepositoryFormat(repository.RepositoryFormat):
106
91
    """A sample format
107
92
 
108
 
    this format is initializable, unsupported to aid in testing the
 
93
    this format is initializable, unsupported to aid in testing the 
109
94
    open and open(unsupported=True) routines.
110
95
    """
111
96
 
132
117
    def test_find_format(self):
133
118
        # is the right format object found for a repository?
134
119
        # create a branch with a few known format objects.
135
 
        # this is not quite the same as
 
120
        # this is not quite the same as 
136
121
        self.build_tree(["foo/", "bar/"])
137
122
        def check_format(format, url):
138
123
            dir = format._matchingbzrdir.initialize(url)
141
126
            found_format = repository.RepositoryFormat.find_format(dir)
142
127
            self.failUnless(isinstance(found_format, format.__class__))
143
128
        check_format(weaverepo.RepositoryFormat7(), "bar")
144
 
 
 
129
        
145
130
    def test_find_format_no_repository(self):
146
131
        dir = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
147
132
        self.assertRaises(errors.NoRepositoryPresent,
173
158
 
174
159
class TestFormat6(TestCaseWithTransport):
175
160
 
176
 
    def test_attribute__fetch_order(self):
177
 
        """Weaves need topological data insertion."""
178
 
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
179
 
        repo = weaverepo.RepositoryFormat6().initialize(control)
180
 
        self.assertEqual('topological', repo._format._fetch_order)
181
 
 
182
 
    def test_attribute__fetch_uses_deltas(self):
183
 
        """Weaves do not reuse deltas."""
184
 
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
185
 
        repo = weaverepo.RepositoryFormat6().initialize(control)
186
 
        self.assertEqual(False, repo._format._fetch_uses_deltas)
187
 
 
188
 
    def test_attribute__fetch_reconcile(self):
189
 
        """Weave repositories need a reconcile after fetch."""
190
 
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
191
 
        repo = weaverepo.RepositoryFormat6().initialize(control)
192
 
        self.assertEqual(True, repo._format._fetch_reconcile)
193
 
 
194
161
    def test_no_ancestry_weave(self):
195
162
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
196
163
        repo = weaverepo.RepositoryFormat6().initialize(control)
200
167
                          control.transport.get,
201
168
                          'ancestry.weave')
202
169
 
 
170
    def test_exposed_versioned_files_are_marked_dirty(self):
 
171
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
 
172
        repo = weaverepo.RepositoryFormat6().initialize(control)
 
173
        repo.lock_write()
 
174
        inv = repo.get_inventory_weave()
 
175
        repo.unlock()
 
176
        self.assertRaises(errors.OutSideTransaction,
 
177
            inv.add_lines, 'foo', [], [])
 
178
 
203
179
    def test_supports_external_lookups(self):
204
180
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
205
181
        repo = weaverepo.RepositoryFormat6().initialize(control)
207
183
 
208
184
 
209
185
class TestFormat7(TestCaseWithTransport):
210
 
 
211
 
    def test_attribute__fetch_order(self):
212
 
        """Weaves need topological data insertion."""
213
 
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
214
 
        repo = weaverepo.RepositoryFormat7().initialize(control)
215
 
        self.assertEqual('topological', repo._format._fetch_order)
216
 
 
217
 
    def test_attribute__fetch_uses_deltas(self):
218
 
        """Weaves do not reuse deltas."""
219
 
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
220
 
        repo = weaverepo.RepositoryFormat7().initialize(control)
221
 
        self.assertEqual(False, repo._format._fetch_uses_deltas)
222
 
 
223
 
    def test_attribute__fetch_reconcile(self):
224
 
        """Weave repositories need a reconcile after fetch."""
225
 
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
226
 
        repo = weaverepo.RepositoryFormat7().initialize(control)
227
 
        self.assertEqual(True, repo._format._fetch_reconcile)
228
 
 
 
186
    
229
187
    def test_disk_layout(self):
230
188
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
231
189
        repo = weaverepo.RepositoryFormat7().initialize(control)
247
205
                             'w\n'
248
206
                             'W\n',
249
207
                             t.get('inventory.weave').read())
250
 
        # Creating a file with id Foo:Bar results in a non-escaped file name on
251
 
        # disk.
252
 
        control.create_branch()
253
 
        tree = control.create_workingtree()
254
 
        tree.add(['foo'], ['Foo:Bar'], ['file'])
255
 
        tree.put_file_bytes_non_atomic('Foo:Bar', 'content\n')
256
 
        try:
257
 
            tree.commit('first post', rev_id='first')
258
 
        except errors.IllegalPath:
259
 
            if sys.platform != 'win32':
260
 
                raise
261
 
            self.knownFailure('Foo:Bar cannot be used as a file-id on windows'
262
 
                              ' in repo format 7')
263
 
            return
264
 
        self.assertEqualDiff(
265
 
            '# bzr weave file v5\n'
266
 
            'i\n'
267
 
            '1 7fe70820e08a1aac0ef224d9c66ab66831cc4ab1\n'
268
 
            'n first\n'
269
 
            '\n'
270
 
            'w\n'
271
 
            '{ 0\n'
272
 
            '. content\n'
273
 
            '}\n'
274
 
            'W\n',
275
 
            t.get('weaves/74/Foo%3ABar.weave').read())
276
208
 
277
209
    def test_shared_disk_layout(self):
278
210
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
301
233
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
302
234
        repo = weaverepo.RepositoryFormat7().initialize(control, shared=True)
303
235
        t = control.get_repository_transport(None)
304
 
        # TODO: Should check there is a 'lock' toplevel directory,
 
236
        # TODO: Should check there is a 'lock' toplevel directory, 
305
237
        # regardless of contents
306
238
        self.assertFalse(t.has('lock/held/info'))
307
239
        repo.lock_write()
353
285
                             'W\n',
354
286
                             t.get('inventory.weave').read())
355
287
 
 
288
    def test_exposed_versioned_files_are_marked_dirty(self):
 
289
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
 
290
        repo = weaverepo.RepositoryFormat7().initialize(control)
 
291
        repo.lock_write()
 
292
        inv = repo.get_inventory_weave()
 
293
        repo.unlock()
 
294
        self.assertRaises(errors.OutSideTransaction,
 
295
            inv.add_lines, 'foo', [], [])
 
296
 
356
297
    def test_supports_external_lookups(self):
357
298
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
358
299
        repo = weaverepo.RepositoryFormat7().initialize(control)
360
301
 
361
302
 
362
303
class TestFormatKnit1(TestCaseWithTransport):
363
 
 
364
 
    def test_attribute__fetch_order(self):
365
 
        """Knits need topological data insertion."""
366
 
        repo = self.make_repository('.',
367
 
                format=bzrdir.format_registry.get('knit')())
368
 
        self.assertEqual('topological', repo._format._fetch_order)
369
 
 
370
 
    def test_attribute__fetch_uses_deltas(self):
371
 
        """Knits reuse deltas."""
372
 
        repo = self.make_repository('.',
373
 
                format=bzrdir.format_registry.get('knit')())
374
 
        self.assertEqual(True, repo._format._fetch_uses_deltas)
375
 
 
 
304
    
376
305
    def test_disk_layout(self):
377
306
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
378
307
        repo = knitrepo.RepositoryFormatKnit1().initialize(control)
392
321
        # self.assertEqualDiff('', t.get('lock').read())
393
322
        self.assertTrue(S_ISDIR(t.stat('knits').st_mode))
394
323
        self.check_knits(t)
395
 
        # Check per-file knits.
396
 
        branch = control.create_branch()
397
 
        tree = control.create_workingtree()
398
 
        tree.add(['foo'], ['Nasty-IdC:'], ['file'])
399
 
        tree.put_file_bytes_non_atomic('Nasty-IdC:', '')
400
 
        tree.commit('1st post', rev_id='foo')
401
 
        self.assertHasKnit(t, 'knits/e8/%254easty-%2549d%2543%253a',
402
 
            '\nfoo fulltext 0 81  :')
403
324
 
404
 
    def assertHasKnit(self, t, knit_name, extra_content=''):
 
325
    def assertHasKnit(self, t, knit_name):
405
326
        """Assert that knit_name exists on t."""
406
 
        self.assertEqualDiff('# bzr knit index 8\n' + extra_content,
 
327
        self.assertEqualDiff('# bzr knit index 8\n',
407
328
                             t.get(knit_name + '.kndx').read())
 
329
        # no default content
 
330
        self.assertTrue(t.has(knit_name + '.knit'))
408
331
 
409
332
    def check_knits(self, t):
410
333
        """check knit content for a repository."""
454
377
        self.assertTrue(S_ISDIR(t.stat('knits').st_mode))
455
378
        self.check_knits(t)
456
379
 
 
380
    def test_exposed_versioned_files_are_marked_dirty(self):
 
381
        format = bzrdir.BzrDirMetaFormat1()
 
382
        format.repository_format = knitrepo.RepositoryFormatKnit1()
 
383
        repo = self.make_repository('.', format=format)
 
384
        repo.lock_write()
 
385
        inv = repo.get_inventory_weave()
 
386
        repo.unlock()
 
387
        self.assertRaises(errors.OutSideTransaction,
 
388
            inv.add_lines, 'foo', [], [])
 
389
 
457
390
    def test_deserialise_sets_root_revision(self):
458
391
        """We must have a inventory.root.revision
459
392
 
464
397
        repo = self.make_repository('.',
465
398
                format=bzrdir.format_registry.get('knit')())
466
399
        inv_xml = '<inventory format="5">\n</inventory>\n'
467
 
        inv = repo._deserialise_inventory('test-rev-id', inv_xml)
 
400
        inv = repo.deserialise_inventory('test-rev-id', inv_xml)
468
401
        self.assertEqual('test-rev-id', inv.root.revision)
469
402
 
470
403
    def test_deserialise_uses_global_revision_id(self):
476
409
        # Arguably, the deserialise_inventory should detect a mismatch, and
477
410
        # raise an error, rather than silently using one revision_id over the
478
411
        # other.
479
 
        self.assertRaises(AssertionError, repo._deserialise_inventory,
 
412
        self.assertRaises(AssertionError, repo.deserialise_inventory,
480
413
            'test-rev-id', inv_xml)
481
 
        inv = repo._deserialise_inventory('other-rev-id', inv_xml)
 
414
        inv = repo.deserialise_inventory('other-rev-id', inv_xml)
482
415
        self.assertEqual('other-rev-id', inv.root.revision)
483
416
 
484
417
    def test_supports_external_lookups(self):
487
420
        self.assertFalse(repo._format.supports_external_lookups)
488
421
 
489
422
 
 
423
class KnitRepositoryStreamTests(test_knit.KnitTests):
 
424
    """Tests for knitrepo._get_stream_as_bytes."""
 
425
 
 
426
    def test_get_stream_as_bytes(self):
 
427
        # Make a simple knit
 
428
        k1 = self.make_test_knit()
 
429
        k1.add_lines('text-a', [], test_knit.split_lines(test_knit.TEXT_1))
 
430
        
 
431
        # Serialise it, check the output.
 
432
        bytes = knitrepo._get_stream_as_bytes(k1, ['text-a'])
 
433
        data = bencode.bdecode(bytes)
 
434
        format, record = data
 
435
        self.assertEqual('knit-plain', format)
 
436
        self.assertEqual(['text-a', ['fulltext'], []], record[:3])
 
437
        self.assertRecordContentEqual(k1, 'text-a', record[3])
 
438
 
 
439
    def test_get_stream_as_bytes_all(self):
 
440
        """Get a serialised data stream for all the records in a knit.
 
441
 
 
442
        Much like test_get_stream_all, except for get_stream_as_bytes.
 
443
        """
 
444
        k1 = self.make_test_knit()
 
445
        # Insert the same data as BasicKnitTests.test_knit_join, as they seem
 
446
        # to cover a range of cases (no parents, one parent, multiple parents).
 
447
        test_data = [
 
448
            ('text-a', [], test_knit.TEXT_1),
 
449
            ('text-b', ['text-a'], test_knit.TEXT_1),
 
450
            ('text-c', [], test_knit.TEXT_1),
 
451
            ('text-d', ['text-c'], test_knit.TEXT_1),
 
452
            ('text-m', ['text-b', 'text-d'], test_knit.TEXT_1),
 
453
           ]
 
454
        # This test is actually a bit strict as the order in which they're
 
455
        # returned is not defined.  This matches the current (deterministic)
 
456
        # behaviour.
 
457
        expected_data_list = [
 
458
            # version, options, parents
 
459
            ('text-a', ['fulltext'], []),
 
460
            ('text-b', ['line-delta'], ['text-a']),
 
461
            ('text-m', ['line-delta'], ['text-b', 'text-d']),
 
462
            ('text-c', ['fulltext'], []),
 
463
            ('text-d', ['line-delta'], ['text-c']),
 
464
            ]
 
465
        for version_id, parents, lines in test_data:
 
466
            k1.add_lines(version_id, parents, test_knit.split_lines(lines))
 
467
 
 
468
        bytes = knitrepo._get_stream_as_bytes(
 
469
            k1, ['text-a', 'text-b', 'text-m', 'text-c', 'text-d', ])
 
470
 
 
471
        data = bencode.bdecode(bytes)
 
472
        format = data.pop(0)
 
473
        self.assertEqual('knit-plain', format)
 
474
 
 
475
        for expected, actual in zip(expected_data_list, data):
 
476
            expected_version = expected[0]
 
477
            expected_options = expected[1]
 
478
            expected_parents = expected[2]
 
479
            version, options, parents, bytes = actual
 
480
            self.assertEqual(expected_version, version)
 
481
            self.assertEqual(expected_options, options)
 
482
            self.assertEqual(expected_parents, parents)
 
483
            self.assertRecordContentEqual(k1, version, bytes)
 
484
 
 
485
 
490
486
class DummyRepository(object):
491
487
    """A dummy repository for testing."""
492
488
 
493
 
    _format = None
494
489
    _serializer = None
495
490
 
496
491
    def supports_rich_root(self):
497
 
        if self._format is not None:
498
 
            return self._format.rich_root_data
499
492
        return False
500
493
 
501
 
    def get_graph(self):
502
 
        raise NotImplementedError
503
 
 
504
 
    def get_parent_map(self, revision_ids):
505
 
        raise NotImplementedError
506
 
 
507
494
 
508
495
class InterDummy(repository.InterRepository):
509
496
    """An inter-repository optimised code path for DummyRepository.
516
503
    @staticmethod
517
504
    def is_compatible(repo_source, repo_target):
518
505
        """InterDummy is compatible with DummyRepository."""
519
 
        return (isinstance(repo_source, DummyRepository) and
 
506
        return (isinstance(repo_source, DummyRepository) and 
520
507
            isinstance(repo_target, DummyRepository))
521
508
 
522
509
 
535
522
 
536
523
    def assertGetsDefaultInterRepository(self, repo_a, repo_b):
537
524
        """Asserts that InterRepository.get(repo_a, repo_b) -> the default.
538
 
 
 
525
        
539
526
        The effective default is now InterSameDataRepository because there is
540
527
        no actual sane default in the presence of incompatible data models.
541
528
        """
552
539
        # pair that it returns true on for the is_compatible static method
553
540
        # check
554
541
        dummy_a = DummyRepository()
555
 
        dummy_a._format = RepositoryFormat()
556
542
        dummy_b = DummyRepository()
557
 
        dummy_b._format = RepositoryFormat()
558
543
        repo = self.make_repository('.')
559
544
        # hack dummies to look like repo somewhat.
560
545
        dummy_a._serializer = repo._serializer
561
 
        dummy_a._format.supports_tree_reference = repo._format.supports_tree_reference
562
 
        dummy_a._format.rich_root_data = repo._format.rich_root_data
563
546
        dummy_b._serializer = repo._serializer
564
 
        dummy_b._format.supports_tree_reference = repo._format.supports_tree_reference
565
 
        dummy_b._format.rich_root_data = repo._format.rich_root_data
566
547
        repository.InterRepository.register_optimiser(InterDummy)
567
548
        try:
568
549
            # we should get the default for something InterDummy returns False
612
593
                                                        repo_b).__class__)
613
594
 
614
595
 
 
596
class TestInterRemoteToOther(TestCaseWithTransport):
 
597
 
 
598
    def make_remote_repository(self, path, backing_format=None):
 
599
        """Make a RemoteRepository object backed by a real repository that will
 
600
        be created at the given path."""
 
601
        self.make_repository(path, format=backing_format)
 
602
        smart_server = server.SmartTCPServer_for_testing()
 
603
        smart_server.setUp()
 
604
        remote_transport = get_transport(smart_server.get_url()).clone(path)
 
605
        self.addCleanup(smart_server.tearDown)
 
606
        remote_bzrdir = bzrdir.BzrDir.open_from_transport(remote_transport)
 
607
        remote_repo = remote_bzrdir.open_repository()
 
608
        return remote_repo
 
609
 
 
610
    def test_is_compatible_same_format(self):
 
611
        """InterRemoteToOther is compatible with a remote repository and a
 
612
        second repository that have the same format."""
 
613
        local_repo = self.make_repository('local')
 
614
        remote_repo = self.make_remote_repository('remote')
 
615
        is_compatible = repository.InterRemoteToOther.is_compatible
 
616
        self.assertTrue(
 
617
            is_compatible(remote_repo, local_repo),
 
618
            "InterRemoteToOther(%r, %r) is false" % (remote_repo, local_repo))
 
619
          
 
620
    def test_is_incompatible_different_format(self):
 
621
        local_repo = self.make_repository('local', 'dirstate')
 
622
        remote_repo = self.make_remote_repository('a', 'dirstate-with-subtree')
 
623
        is_compatible = repository.InterRemoteToOther.is_compatible
 
624
        self.assertFalse(
 
625
            is_compatible(remote_repo, local_repo),
 
626
            "InterRemoteToOther(%r, %r) is true" % (local_repo, remote_repo))
 
627
 
 
628
    def test_is_incompatible_different_format_both_remote(self):
 
629
        remote_repo_a = self.make_remote_repository(
 
630
            'a', 'dirstate-with-subtree')
 
631
        remote_repo_b = self.make_remote_repository('b', 'dirstate')
 
632
        is_compatible = repository.InterRemoteToOther.is_compatible
 
633
        self.assertFalse(
 
634
            is_compatible(remote_repo_a, remote_repo_b),
 
635
            "InterRemoteToOther(%r, %r) is true"
 
636
            % (remote_repo_a, remote_repo_b))
 
637
 
 
638
 
615
639
class TestRepositoryConverter(TestCaseWithTransport):
616
640
 
617
641
    def test_convert_empty(self):
631
655
 
632
656
 
633
657
class TestMisc(TestCase):
634
 
 
 
658
    
635
659
    def test_unescape_xml(self):
636
660
        """We get some kind of error when malformed entities are passed"""
637
 
        self.assertRaises(KeyError, repository._unescape_xml, 'foo&bar;')
 
661
        self.assertRaises(KeyError, repository._unescape_xml, 'foo&bar;') 
638
662
 
639
663
 
640
664
class TestRepositoryFormatKnit3(TestCaseWithTransport):
641
665
 
642
 
    def test_attribute__fetch_order(self):
643
 
        """Knits need topological data insertion."""
644
 
        format = bzrdir.BzrDirMetaFormat1()
645
 
        format.repository_format = knitrepo.RepositoryFormatKnit3()
646
 
        repo = self.make_repository('.', format=format)
647
 
        self.assertEqual('topological', repo._format._fetch_order)
648
 
 
649
 
    def test_attribute__fetch_uses_deltas(self):
650
 
        """Knits reuse deltas."""
651
 
        format = bzrdir.BzrDirMetaFormat1()
652
 
        format.repository_format = knitrepo.RepositoryFormatKnit3()
653
 
        repo = self.make_repository('.', format=format)
654
 
        self.assertEqual(True, repo._format._fetch_uses_deltas)
655
 
 
656
666
    def test_convert(self):
657
667
        """Ensure the upgrade adds weaves for roots"""
658
668
        format = bzrdir.BzrDirMetaFormat1()
660
670
        tree = self.make_branch_and_tree('.', format)
661
671
        tree.commit("Dull commit", rev_id="dull")
662
672
        revision_tree = tree.branch.repository.revision_tree('dull')
663
 
        revision_tree.lock_read()
664
 
        try:
665
 
            self.assertRaises(errors.NoSuchFile, revision_tree.get_file_lines,
666
 
                revision_tree.inventory.root.file_id)
667
 
        finally:
668
 
            revision_tree.unlock()
 
673
        self.assertRaises(errors.NoSuchFile, revision_tree.get_file_lines,
 
674
            revision_tree.inventory.root.file_id)
669
675
        format = bzrdir.BzrDirMetaFormat1()
670
676
        format.repository_format = knitrepo.RepositoryFormatKnit3()
671
677
        upgrade.Convert('.', format)
672
678
        tree = workingtree.WorkingTree.open('.')
673
679
        revision_tree = tree.branch.repository.revision_tree('dull')
674
 
        revision_tree.lock_read()
675
 
        try:
676
 
            revision_tree.get_file_lines(revision_tree.inventory.root.file_id)
677
 
        finally:
678
 
            revision_tree.unlock()
 
680
        revision_tree.get_file_lines(revision_tree.inventory.root.file_id)
679
681
        tree.commit("Another dull commit", rev_id='dull2')
680
682
        revision_tree = tree.branch.repository.revision_tree('dull2')
681
 
        revision_tree.lock_read()
682
 
        self.addCleanup(revision_tree.unlock)
683
683
        self.assertEqual('dull', revision_tree.inventory.root.revision)
684
684
 
 
685
    def test_exposed_versioned_files_are_marked_dirty(self):
 
686
        format = bzrdir.BzrDirMetaFormat1()
 
687
        format.repository_format = knitrepo.RepositoryFormatKnit3()
 
688
        repo = self.make_repository('.', format=format)
 
689
        repo.lock_write()
 
690
        inv = repo.get_inventory_weave()
 
691
        repo.unlock()
 
692
        self.assertRaises(errors.OutSideTransaction,
 
693
            inv.add_lines, 'foo', [], [])
 
694
 
685
695
    def test_supports_external_lookups(self):
686
696
        format = bzrdir.BzrDirMetaFormat1()
687
697
        format.repository_format = knitrepo.RepositoryFormatKnit3()
689
699
        self.assertFalse(repo._format.supports_external_lookups)
690
700
 
691
701
 
692
 
class Test2a(tests.TestCaseWithMemoryTransport):
693
 
 
694
 
    def test_fetch_combines_groups(self):
695
 
        builder = self.make_branch_builder('source', format='2a')
696
 
        builder.start_series()
697
 
        builder.build_snapshot('1', None, [
698
 
            ('add', ('', 'root-id', 'directory', '')),
699
 
            ('add', ('file', 'file-id', 'file', 'content\n'))])
700
 
        builder.build_snapshot('2', ['1'], [
701
 
            ('modify', ('file-id', 'content-2\n'))])
702
 
        builder.finish_series()
703
 
        source = builder.get_branch()
704
 
        target = self.make_repository('target', format='2a')
705
 
        target.fetch(source.repository)
706
 
        target.lock_read()
707
 
        self.addCleanup(target.unlock)
708
 
        details = target.texts._index.get_build_details(
709
 
            [('file-id', '1',), ('file-id', '2',)])
710
 
        file_1_details = details[('file-id', '1')]
711
 
        file_2_details = details[('file-id', '2')]
712
 
        # The index, and what to read off disk, should be the same for both
713
 
        # versions of the file.
714
 
        self.assertEqual(file_1_details[0][:3], file_2_details[0][:3])
715
 
 
716
 
    def test_fetch_combines_groups(self):
717
 
        builder = self.make_branch_builder('source', format='2a')
718
 
        builder.start_series()
719
 
        builder.build_snapshot('1', None, [
720
 
            ('add', ('', 'root-id', 'directory', '')),
721
 
            ('add', ('file', 'file-id', 'file', 'content\n'))])
722
 
        builder.build_snapshot('2', ['1'], [
723
 
            ('modify', ('file-id', 'content-2\n'))])
724
 
        builder.finish_series()
725
 
        source = builder.get_branch()
726
 
        target = self.make_repository('target', format='2a')
727
 
        target.fetch(source.repository)
728
 
        target.lock_read()
729
 
        self.addCleanup(target.unlock)
730
 
        details = target.texts._index.get_build_details(
731
 
            [('file-id', '1',), ('file-id', '2',)])
732
 
        file_1_details = details[('file-id', '1')]
733
 
        file_2_details = details[('file-id', '2')]
734
 
        # The index, and what to read off disk, should be the same for both
735
 
        # versions of the file.
736
 
        self.assertEqual(file_1_details[0][:3], file_2_details[0][:3])
737
 
 
738
 
    def test_fetch_combines_groups(self):
739
 
        builder = self.make_branch_builder('source', format='2a')
740
 
        builder.start_series()
741
 
        builder.build_snapshot('1', None, [
742
 
            ('add', ('', 'root-id', 'directory', '')),
743
 
            ('add', ('file', 'file-id', 'file', 'content\n'))])
744
 
        builder.build_snapshot('2', ['1'], [
745
 
            ('modify', ('file-id', 'content-2\n'))])
746
 
        builder.finish_series()
747
 
        source = builder.get_branch()
748
 
        target = self.make_repository('target', format='2a')
749
 
        target.fetch(source.repository)
750
 
        target.lock_read()
751
 
        self.addCleanup(target.unlock)
752
 
        details = target.texts._index.get_build_details(
753
 
            [('file-id', '1',), ('file-id', '2',)])
754
 
        file_1_details = details[('file-id', '1')]
755
 
        file_2_details = details[('file-id', '2')]
756
 
        # The index, and what to read off disk, should be the same for both
757
 
        # versions of the file.
758
 
        self.assertEqual(file_1_details[0][:3], file_2_details[0][:3])
759
 
 
760
 
    def test_format_pack_compresses_True(self):
761
 
        repo = self.make_repository('repo', format='2a')
762
 
        self.assertTrue(repo._format.pack_compresses)
763
 
 
764
 
    def test_inventories_use_chk_map_with_parent_base_dict(self):
765
 
        tree = self.make_branch_and_memory_tree('repo', format="2a")
766
 
        tree.lock_write()
767
 
        tree.add([''], ['TREE_ROOT'])
768
 
        revid = tree.commit("foo")
769
 
        tree.unlock()
770
 
        tree.lock_read()
771
 
        self.addCleanup(tree.unlock)
772
 
        inv = tree.branch.repository.get_inventory(revid)
773
 
        self.assertNotEqual(None, inv.parent_id_basename_to_file_id)
774
 
        inv.parent_id_basename_to_file_id._ensure_root()
775
 
        inv.id_to_entry._ensure_root()
776
 
        self.assertEqual(65536, inv.id_to_entry._root_node.maximum_size)
777
 
        self.assertEqual(65536,
778
 
            inv.parent_id_basename_to_file_id._root_node.maximum_size)
779
 
 
780
 
    def test_autopack_unchanged_chk_nodes(self):
781
 
        # at 20 unchanged commits, chk pages are packed that are split into
782
 
        # two groups such that the new pack being made doesn't have all its
783
 
        # pages in the source packs (though they are in the repository).
784
 
        # Use a memory backed repository, we don't need to hit disk for this
785
 
        tree = self.make_branch_and_memory_tree('tree', format='2a')
786
 
        tree.lock_write()
787
 
        self.addCleanup(tree.unlock)
788
 
        tree.add([''], ['TREE_ROOT'])
789
 
        for pos in range(20):
790
 
            tree.commit(str(pos))
791
 
 
792
 
    def test_pack_with_hint(self):
793
 
        tree = self.make_branch_and_memory_tree('tree', format='2a')
794
 
        tree.lock_write()
795
 
        self.addCleanup(tree.unlock)
796
 
        tree.add([''], ['TREE_ROOT'])
797
 
        # 1 commit to leave untouched
798
 
        tree.commit('1')
799
 
        to_keep = tree.branch.repository._pack_collection.names()
800
 
        # 2 to combine
801
 
        tree.commit('2')
802
 
        tree.commit('3')
803
 
        all = tree.branch.repository._pack_collection.names()
804
 
        combine = list(set(all) - set(to_keep))
805
 
        self.assertLength(3, all)
806
 
        self.assertLength(2, combine)
807
 
        tree.branch.repository.pack(hint=combine)
808
 
        final = tree.branch.repository._pack_collection.names()
809
 
        self.assertLength(2, final)
810
 
        self.assertFalse(combine[0] in final)
811
 
        self.assertFalse(combine[1] in final)
812
 
        self.assertSubset(to_keep, final)
813
 
 
814
 
    def test_stream_source_to_gc(self):
815
 
        source = self.make_repository('source', format='2a')
816
 
        target = self.make_repository('target', format='2a')
817
 
        stream = source._get_source(target._format)
818
 
        self.assertIsInstance(stream, groupcompress_repo.GroupCHKStreamSource)
819
 
 
820
 
    def test_stream_source_to_non_gc(self):
821
 
        source = self.make_repository('source', format='2a')
822
 
        target = self.make_repository('target', format='rich-root-pack')
823
 
        stream = source._get_source(target._format)
824
 
        # We don't want the child GroupCHKStreamSource
825
 
        self.assertIs(type(stream), repository.StreamSource)
826
 
 
827
 
    def test_get_stream_for_missing_keys_includes_all_chk_refs(self):
828
 
        source_builder = self.make_branch_builder('source',
829
 
                            format='2a')
830
 
        # We have to build a fairly large tree, so that we are sure the chk
831
 
        # pages will have split into multiple pages.
832
 
        entries = [('add', ('', 'a-root-id', 'directory', None))]
833
 
        for i in 'abcdefghijklmnopqrstuvwxyz123456789':
834
 
            for j in 'abcdefghijklmnopqrstuvwxyz123456789':
835
 
                fname = i + j
836
 
                fid = fname + '-id'
837
 
                content = 'content for %s\n' % (fname,)
838
 
                entries.append(('add', (fname, fid, 'file', content)))
839
 
        source_builder.start_series()
840
 
        source_builder.build_snapshot('rev-1', None, entries)
841
 
        # Now change a few of them, so we get a few new pages for the second
842
 
        # revision
843
 
        source_builder.build_snapshot('rev-2', ['rev-1'], [
844
 
            ('modify', ('aa-id', 'new content for aa-id\n')),
845
 
            ('modify', ('cc-id', 'new content for cc-id\n')),
846
 
            ('modify', ('zz-id', 'new content for zz-id\n')),
847
 
            ])
848
 
        source_builder.finish_series()
849
 
        source_branch = source_builder.get_branch()
850
 
        source_branch.lock_read()
851
 
        self.addCleanup(source_branch.unlock)
852
 
        target = self.make_repository('target', format='2a')
853
 
        source = source_branch.repository._get_source(target._format)
854
 
        self.assertIsInstance(source, groupcompress_repo.GroupCHKStreamSource)
855
 
 
856
 
        # On a regular pass, getting the inventories and chk pages for rev-2
857
 
        # would only get the newly created chk pages
858
 
        search = graph.SearchResult(set(['rev-2']), set(['rev-1']), 1,
859
 
                                    set(['rev-2']))
860
 
        simple_chk_records = []
861
 
        for vf_name, substream in source.get_stream(search):
862
 
            if vf_name == 'chk_bytes':
863
 
                for record in substream:
864
 
                    simple_chk_records.append(record.key)
865
 
            else:
866
 
                for _ in substream:
867
 
                    continue
868
 
        # 3 pages, the root (InternalNode), + 2 pages which actually changed
869
 
        self.assertEqual([('sha1:91481f539e802c76542ea5e4c83ad416bf219f73',),
870
 
                          ('sha1:4ff91971043668583985aec83f4f0ab10a907d3f',),
871
 
                          ('sha1:81e7324507c5ca132eedaf2d8414ee4bb2226187',),
872
 
                          ('sha1:b101b7da280596c71a4540e9a1eeba8045985ee0',)],
873
 
                         simple_chk_records)
874
 
        # Now, when we do a similar call using 'get_stream_for_missing_keys'
875
 
        # we should get a much larger set of pages.
876
 
        missing = [('inventories', 'rev-2')]
877
 
        full_chk_records = []
878
 
        for vf_name, substream in source.get_stream_for_missing_keys(missing):
879
 
            if vf_name == 'inventories':
880
 
                for record in substream:
881
 
                    self.assertEqual(('rev-2',), record.key)
882
 
            elif vf_name == 'chk_bytes':
883
 
                for record in substream:
884
 
                    full_chk_records.append(record.key)
885
 
            else:
886
 
                self.fail('Should not be getting a stream of %s' % (vf_name,))
887
 
        # We have 257 records now. This is because we have 1 root page, and 256
888
 
        # leaf pages in a complete listing.
889
 
        self.assertEqual(257, len(full_chk_records))
890
 
        self.assertSubset(simple_chk_records, full_chk_records)
891
 
 
892
 
    def test_inconsistency_fatal(self):
893
 
        repo = self.make_repository('repo', format='2a')
894
 
        self.assertTrue(repo.revisions._index._inconsistency_fatal)
895
 
        self.assertFalse(repo.texts._index._inconsistency_fatal)
896
 
        self.assertFalse(repo.inventories._index._inconsistency_fatal)
897
 
        self.assertFalse(repo.signatures._index._inconsistency_fatal)
898
 
        self.assertFalse(repo.chk_bytes._index._inconsistency_fatal)
899
 
 
900
 
 
901
 
class TestKnitPackStreamSource(tests.TestCaseWithMemoryTransport):
902
 
 
903
 
    def test_source_to_exact_pack_092(self):
904
 
        source = self.make_repository('source', format='pack-0.92')
905
 
        target = self.make_repository('target', format='pack-0.92')
906
 
        stream_source = source._get_source(target._format)
907
 
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
908
 
 
909
 
    def test_source_to_exact_pack_rich_root_pack(self):
910
 
        source = self.make_repository('source', format='rich-root-pack')
911
 
        target = self.make_repository('target', format='rich-root-pack')
912
 
        stream_source = source._get_source(target._format)
913
 
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
914
 
 
915
 
    def test_source_to_exact_pack_19(self):
916
 
        source = self.make_repository('source', format='1.9')
917
 
        target = self.make_repository('target', format='1.9')
918
 
        stream_source = source._get_source(target._format)
919
 
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
920
 
 
921
 
    def test_source_to_exact_pack_19_rich_root(self):
922
 
        source = self.make_repository('source', format='1.9-rich-root')
923
 
        target = self.make_repository('target', format='1.9-rich-root')
924
 
        stream_source = source._get_source(target._format)
925
 
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
926
 
 
927
 
    def test_source_to_remote_exact_pack_19(self):
928
 
        trans = self.make_smart_server('target')
929
 
        trans.ensure_base()
930
 
        source = self.make_repository('source', format='1.9')
931
 
        target = self.make_repository('target', format='1.9')
932
 
        target = repository.Repository.open(trans.base)
933
 
        stream_source = source._get_source(target._format)
934
 
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
935
 
 
936
 
    def test_stream_source_to_non_exact(self):
937
 
        source = self.make_repository('source', format='pack-0.92')
938
 
        target = self.make_repository('target', format='1.9')
939
 
        stream = source._get_source(target._format)
940
 
        self.assertIs(type(stream), repository.StreamSource)
941
 
 
942
 
    def test_stream_source_to_non_exact_rich_root(self):
943
 
        source = self.make_repository('source', format='1.9')
944
 
        target = self.make_repository('target', format='1.9-rich-root')
945
 
        stream = source._get_source(target._format)
946
 
        self.assertIs(type(stream), repository.StreamSource)
947
 
 
948
 
    def test_source_to_remote_non_exact_pack_19(self):
949
 
        trans = self.make_smart_server('target')
950
 
        trans.ensure_base()
951
 
        source = self.make_repository('source', format='1.9')
952
 
        target = self.make_repository('target', format='1.6')
953
 
        target = repository.Repository.open(trans.base)
954
 
        stream_source = source._get_source(target._format)
955
 
        self.assertIs(type(stream_source), repository.StreamSource)
956
 
 
957
 
    def test_stream_source_to_knit(self):
958
 
        source = self.make_repository('source', format='pack-0.92')
959
 
        target = self.make_repository('target', format='dirstate')
960
 
        stream = source._get_source(target._format)
961
 
        self.assertIs(type(stream), repository.StreamSource)
962
 
 
963
 
 
964
 
class TestDevelopment6FindParentIdsOfRevisions(TestCaseWithTransport):
965
 
    """Tests for _find_parent_ids_of_revisions."""
966
 
 
967
 
    def setUp(self):
968
 
        super(TestDevelopment6FindParentIdsOfRevisions, self).setUp()
969
 
        self.builder = self.make_branch_builder('source',
970
 
            format='development6-rich-root')
971
 
        self.builder.start_series()
972
 
        self.builder.build_snapshot('initial', None,
973
 
            [('add', ('', 'tree-root', 'directory', None))])
974
 
        self.repo = self.builder.get_branch().repository
975
 
        self.addCleanup(self.builder.finish_series)
976
 
 
977
 
    def assertParentIds(self, expected_result, rev_set):
978
 
        self.assertEqual(sorted(expected_result),
979
 
            sorted(self.repo._find_parent_ids_of_revisions(rev_set)))
980
 
 
981
 
    def test_simple(self):
982
 
        self.builder.build_snapshot('revid1', None, [])
983
 
        self.builder.build_snapshot('revid2', ['revid1'], [])
984
 
        rev_set = ['revid2']
985
 
        self.assertParentIds(['revid1'], rev_set)
986
 
 
987
 
    def test_not_first_parent(self):
988
 
        self.builder.build_snapshot('revid1', None, [])
989
 
        self.builder.build_snapshot('revid2', ['revid1'], [])
990
 
        self.builder.build_snapshot('revid3', ['revid2'], [])
991
 
        rev_set = ['revid3', 'revid2']
992
 
        self.assertParentIds(['revid1'], rev_set)
993
 
 
994
 
    def test_not_null(self):
995
 
        rev_set = ['initial']
996
 
        self.assertParentIds([], rev_set)
997
 
 
998
 
    def test_not_null_set(self):
999
 
        self.builder.build_snapshot('revid1', None, [])
1000
 
        rev_set = [_mod_revision.NULL_REVISION]
1001
 
        self.assertParentIds([], rev_set)
1002
 
 
1003
 
    def test_ghost(self):
1004
 
        self.builder.build_snapshot('revid1', None, [])
1005
 
        rev_set = ['ghost', 'revid1']
1006
 
        self.assertParentIds(['initial'], rev_set)
1007
 
 
1008
 
    def test_ghost_parent(self):
1009
 
        self.builder.build_snapshot('revid1', None, [])
1010
 
        self.builder.build_snapshot('revid2', ['revid1', 'ghost'], [])
1011
 
        rev_set = ['revid2', 'revid1']
1012
 
        self.assertParentIds(['ghost', 'initial'], rev_set)
1013
 
 
1014
 
    def test_righthand_parent(self):
1015
 
        self.builder.build_snapshot('revid1', None, [])
1016
 
        self.builder.build_snapshot('revid2a', ['revid1'], [])
1017
 
        self.builder.build_snapshot('revid2b', ['revid1'], [])
1018
 
        self.builder.build_snapshot('revid3', ['revid2a', 'revid2b'], [])
1019
 
        rev_set = ['revid3', 'revid2a']
1020
 
        self.assertParentIds(['revid1', 'revid2b'], rev_set)
1021
 
 
1022
 
 
1023
702
class TestWithBrokenRepo(TestCaseWithTransport):
1024
703
    """These tests seem to be more appropriate as interface tests?"""
1025
704
 
1038
717
            inv = inventory.Inventory(revision_id='rev1a')
1039
718
            inv.root.revision = 'rev1a'
1040
719
            self.add_file(repo, inv, 'file1', 'rev1a', [])
1041
 
            repo.texts.add_lines((inv.root.file_id, 'rev1a'), [], [])
1042
720
            repo.add_inventory('rev1a', inv, [])
1043
721
            revision = _mod_revision.Revision('rev1a',
1044
722
                committer='jrandom@example.com', timestamp=0,
1079
757
    def add_revision(self, repo, revision_id, inv, parent_ids):
1080
758
        inv.revision_id = revision_id
1081
759
        inv.root.revision = revision_id
1082
 
        repo.texts.add_lines((inv.root.file_id, revision_id), [], [])
1083
760
        repo.add_inventory(revision_id, inv, parent_ids)
1084
761
        revision = _mod_revision.Revision(revision_id,
1085
762
            committer='jrandom@example.com', timestamp=0, inventory_sha1='',
1092
769
        entry.revision = revision
1093
770
        entry.text_size = 0
1094
771
        inv.add(entry)
1095
 
        text_key = (file_id, revision)
1096
 
        parent_keys = [(file_id, parent) for parent in parents]
1097
 
        repo.texts.add_lines(text_key, parent_keys, ['line\n'])
 
772
        vf = repo.weave_store.get_weave_or_empty(file_id,
 
773
                                                 repo.get_transaction())
 
774
        vf.add_lines(revision, parents, ['line\n'])
1098
775
 
1099
776
    def test_insert_from_broken_repo(self):
1100
777
        """Inserting a data stream from a broken repository won't silently
1102
779
        """
1103
780
        broken_repo = self.make_broken_repository()
1104
781
        empty_repo = self.make_repository('empty-repo')
1105
 
        try:
1106
 
            empty_repo.fetch(broken_repo)
1107
 
        except (errors.RevisionNotPresent, errors.BzrCheckError):
1108
 
            # Test successful: compression parent not being copied leads to
1109
 
            # error.
1110
 
            return
1111
 
        empty_repo.lock_read()
 
782
        search = graph.SearchResult(set(['rev1a', 'rev2', 'rev3']),
 
783
            set(), 3, ['rev1a', 'rev2', 'rev3'])
 
784
        stream = broken_repo.get_data_stream_for_search(search)
 
785
        empty_repo.lock_write()
1112
786
        self.addCleanup(empty_repo.unlock)
1113
 
        text = empty_repo.texts.get_record_stream(
1114
 
            [('file2-id', 'rev3')], 'topological', True).next()
1115
 
        self.assertEqual('line\n', text.get_bytes_as('fulltext'))
 
787
        empty_repo.start_write_group()
 
788
        try:
 
789
            self.assertRaises(
 
790
                errors.KnitCorrupt, empty_repo.insert_data_stream, stream)
 
791
        finally:
 
792
            empty_repo.abort_write_group()
 
793
 
 
794
 
 
795
class TestKnitPackNoSubtrees(TestCaseWithTransport):
 
796
 
 
797
    def get_format(self):
 
798
        return bzrdir.format_registry.make_bzrdir('pack-0.92')
 
799
 
 
800
    def test_disk_layout(self):
 
801
        format = self.get_format()
 
802
        repo = self.make_repository('.', format=format)
 
803
        # in case of side effects of locking.
 
804
        repo.lock_write()
 
805
        repo.unlock()
 
806
        t = repo.bzrdir.get_repository_transport(None)
 
807
        self.check_format(t)
 
808
        # XXX: no locks left when unlocked at the moment
 
809
        # self.assertEqualDiff('', t.get('lock').read())
 
810
        self.check_databases(t)
 
811
 
 
812
    def check_format(self, t):
 
813
        self.assertEqualDiff(
 
814
            "Bazaar pack repository format 1 (needs bzr 0.92)\n",
 
815
                             t.get('format').read())
 
816
 
 
817
    def assertHasKndx(self, t, knit_name):
 
818
        """Assert that knit_name exists on t."""
 
819
        self.assertEqualDiff('# bzr knit index 8\n',
 
820
                             t.get(knit_name + '.kndx').read())
 
821
 
 
822
    def assertHasNoKndx(self, t, knit_name):
 
823
        """Assert that knit_name has no index on t."""
 
824
        self.assertFalse(t.has(knit_name + '.kndx'))
 
825
 
 
826
    def assertHasNoKnit(self, t, knit_name):
 
827
        """Assert that knit_name exists on t."""
 
828
        # no default content
 
829
        self.assertFalse(t.has(knit_name + '.knit'))
 
830
 
 
831
    def check_databases(self, t):
 
832
        """check knit content for a repository."""
 
833
        # check conversion worked
 
834
        self.assertHasNoKndx(t, 'inventory')
 
835
        self.assertHasNoKnit(t, 'inventory')
 
836
        self.assertHasNoKndx(t, 'revisions')
 
837
        self.assertHasNoKnit(t, 'revisions')
 
838
        self.assertHasNoKndx(t, 'signatures')
 
839
        self.assertHasNoKnit(t, 'signatures')
 
840
        self.assertFalse(t.has('knits'))
 
841
        # revision-indexes file-container directory
 
842
        self.assertEqual([],
 
843
            list(GraphIndex(t, 'pack-names', None).iter_all_entries()))
 
844
        self.assertTrue(S_ISDIR(t.stat('packs').st_mode))
 
845
        self.assertTrue(S_ISDIR(t.stat('upload').st_mode))
 
846
        self.assertTrue(S_ISDIR(t.stat('indices').st_mode))
 
847
        self.assertTrue(S_ISDIR(t.stat('obsolete_packs').st_mode))
 
848
 
 
849
    def test_shared_disk_layout(self):
 
850
        format = self.get_format()
 
851
        repo = self.make_repository('.', shared=True, format=format)
 
852
        # we want:
 
853
        t = repo.bzrdir.get_repository_transport(None)
 
854
        self.check_format(t)
 
855
        # XXX: no locks left when unlocked at the moment
 
856
        # self.assertEqualDiff('', t.get('lock').read())
 
857
        # We should have a 'shared-storage' marker file.
 
858
        self.assertEqualDiff('', t.get('shared-storage').read())
 
859
        self.check_databases(t)
 
860
 
 
861
    def test_shared_no_tree_disk_layout(self):
 
862
        format = self.get_format()
 
863
        repo = self.make_repository('.', shared=True, format=format)
 
864
        repo.set_make_working_trees(False)
 
865
        # we want:
 
866
        t = repo.bzrdir.get_repository_transport(None)
 
867
        self.check_format(t)
 
868
        # XXX: no locks left when unlocked at the moment
 
869
        # self.assertEqualDiff('', t.get('lock').read())
 
870
        # We should have a 'shared-storage' marker file.
 
871
        self.assertEqualDiff('', t.get('shared-storage').read())
 
872
        # We should have a marker for the no-working-trees flag.
 
873
        self.assertEqualDiff('', t.get('no-working-trees').read())
 
874
        # The marker should go when we toggle the setting.
 
875
        repo.set_make_working_trees(True)
 
876
        self.assertFalse(t.has('no-working-trees'))
 
877
        self.check_databases(t)
 
878
 
 
879
    def test_adding_revision_creates_pack_indices(self):
 
880
        format = self.get_format()
 
881
        tree = self.make_branch_and_tree('.', format=format)
 
882
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
 
883
        self.assertEqual([],
 
884
            list(GraphIndex(trans, 'pack-names', None).iter_all_entries()))
 
885
        tree.commit('foobarbaz')
 
886
        index = GraphIndex(trans, 'pack-names', None)
 
887
        index_nodes = list(index.iter_all_entries())
 
888
        self.assertEqual(1, len(index_nodes))
 
889
        node = index_nodes[0]
 
890
        name = node[1][0]
 
891
        # the pack sizes should be listed in the index
 
892
        pack_value = node[2]
 
893
        sizes = [int(digits) for digits in pack_value.split(' ')]
 
894
        for size, suffix in zip(sizes, ['.rix', '.iix', '.tix', '.six']):
 
895
            stat = trans.stat('indices/%s%s' % (name, suffix))
 
896
            self.assertEqual(size, stat.st_size)
 
897
 
 
898
    def test_pulling_nothing_leads_to_no_new_names(self):
 
899
        format = self.get_format()
 
900
        tree1 = self.make_branch_and_tree('1', format=format)
 
901
        tree2 = self.make_branch_and_tree('2', format=format)
 
902
        tree1.branch.repository.fetch(tree2.branch.repository)
 
903
        trans = tree1.branch.repository.bzrdir.get_repository_transport(None)
 
904
        self.assertEqual([],
 
905
            list(GraphIndex(trans, 'pack-names', None).iter_all_entries()))
 
906
 
 
907
    def test_commit_across_pack_shape_boundary_autopacks(self):
 
908
        format = self.get_format()
 
909
        tree = self.make_branch_and_tree('.', format=format)
 
910
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
 
911
        # This test could be a little cheaper by replacing the packs
 
912
        # attribute on the repository to allow a different pack distribution
 
913
        # and max packs policy - so we are checking the policy is honoured
 
914
        # in the test. But for now 11 commits is not a big deal in a single
 
915
        # test.
 
916
        for x in range(9):
 
917
            tree.commit('commit %s' % x)
 
918
        # there should be 9 packs:
 
919
        index = GraphIndex(trans, 'pack-names', None)
 
920
        self.assertEqual(9, len(list(index.iter_all_entries())))
 
921
        # insert some files in obsolete_packs which should be removed by pack.
 
922
        trans.put_bytes('obsolete_packs/foo', '123')
 
923
        trans.put_bytes('obsolete_packs/bar', '321')
 
924
        # committing one more should coalesce to 1 of 10.
 
925
        tree.commit('commit triggering pack')
 
926
        index = GraphIndex(trans, 'pack-names', None)
 
927
        self.assertEqual(1, len(list(index.iter_all_entries())))
 
928
        # packing should not damage data
 
929
        tree = tree.bzrdir.open_workingtree()
 
930
        check_result = tree.branch.repository.check(
 
931
            [tree.branch.last_revision()])
 
932
        # We should have 50 (10x5) files in the obsolete_packs directory.
 
933
        obsolete_files = list(trans.list_dir('obsolete_packs'))
 
934
        self.assertFalse('foo' in obsolete_files)
 
935
        self.assertFalse('bar' in obsolete_files)
 
936
        self.assertEqual(50, len(obsolete_files))
 
937
        # XXX: Todo check packs obsoleted correctly - old packs and indices
 
938
        # in the obsolete_packs directory.
 
939
        large_pack_name = list(index.iter_all_entries())[0][1][0]
 
940
        # finally, committing again should not touch the large pack.
 
941
        tree.commit('commit not triggering pack')
 
942
        index = GraphIndex(trans, 'pack-names', None)
 
943
        self.assertEqual(2, len(list(index.iter_all_entries())))
 
944
        pack_names = [node[1][0] for node in index.iter_all_entries()]
 
945
        self.assertTrue(large_pack_name in pack_names)
 
946
 
 
947
    def test_pack_after_two_commits_packs_everything(self):
 
948
        format = self.get_format()
 
949
        tree = self.make_branch_and_tree('.', format=format)
 
950
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
 
951
        tree.commit('start')
 
952
        tree.commit('more work')
 
953
        tree.branch.repository.pack()
 
954
        # there should be 1 pack:
 
955
        index = GraphIndex(trans, 'pack-names', None)
 
956
        self.assertEqual(1, len(list(index.iter_all_entries())))
 
957
        self.assertEqual(2, len(tree.branch.repository.all_revision_ids()))
 
958
 
 
959
    def test_pack_layout(self):
 
960
        format = self.get_format()
 
961
        tree = self.make_branch_and_tree('.', format=format)
 
962
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
 
963
        tree.commit('start', rev_id='1')
 
964
        tree.commit('more work', rev_id='2')
 
965
        tree.branch.repository.pack()
 
966
        tree.lock_read()
 
967
        self.addCleanup(tree.unlock)
 
968
        pack = tree.branch.repository._pack_collection.get_pack_by_name(
 
969
            tree.branch.repository._pack_collection.names()[0])
 
970
        # revision access tends to be tip->ancestor, so ordering that way on 
 
971
        # disk is a good idea.
 
972
        for _1, key, val, refs in pack.revision_index.iter_all_entries():
 
973
            if key == ('1',):
 
974
                pos_1 = int(val[1:].split()[0])
 
975
            else:
 
976
                pos_2 = int(val[1:].split()[0])
 
977
        self.assertTrue(pos_2 < pos_1)
 
978
 
 
979
    def test_pack_repositories_support_multiple_write_locks(self):
 
980
        format = self.get_format()
 
981
        self.make_repository('.', shared=True, format=format)
 
982
        r1 = repository.Repository.open('.')
 
983
        r2 = repository.Repository.open('.')
 
984
        r1.lock_write()
 
985
        self.addCleanup(r1.unlock)
 
986
        r2.lock_write()
 
987
        r2.unlock()
 
988
 
 
989
    def _add_text(self, repo, fileid):
 
990
        """Add a text to the repository within a write group."""
 
991
        vf =repo.weave_store.get_weave(fileid, repo.get_transaction())
 
992
        vf.add_lines('samplerev+' + fileid, [], [])
 
993
 
 
994
    def test_concurrent_writers_merge_new_packs(self):
 
995
        format = self.get_format()
 
996
        self.make_repository('.', shared=True, format=format)
 
997
        r1 = repository.Repository.open('.')
 
998
        r2 = repository.Repository.open('.')
 
999
        r1.lock_write()
 
1000
        try:
 
1001
            # access enough data to load the names list
 
1002
            list(r1.all_revision_ids())
 
1003
            r2.lock_write()
 
1004
            try:
 
1005
                # access enough data to load the names list
 
1006
                list(r2.all_revision_ids())
 
1007
                r1.start_write_group()
 
1008
                try:
 
1009
                    r2.start_write_group()
 
1010
                    try:
 
1011
                        self._add_text(r1, 'fileidr1')
 
1012
                        self._add_text(r2, 'fileidr2')
 
1013
                    except:
 
1014
                        r2.abort_write_group()
 
1015
                        raise
 
1016
                except:
 
1017
                    r1.abort_write_group()
 
1018
                    raise
 
1019
                # both r1 and r2 have open write groups with data in them
 
1020
                # created while the other's write group was open.
 
1021
                # Commit both which requires a merge to the pack-names.
 
1022
                try:
 
1023
                    r1.commit_write_group()
 
1024
                except:
 
1025
                    r1.abort_write_group()
 
1026
                    r2.abort_write_group()
 
1027
                    raise
 
1028
                r2.commit_write_group()
 
1029
                # tell r1 to reload from disk
 
1030
                r1._pack_collection.reset()
 
1031
                # Now both repositories should know about both names
 
1032
                r1._pack_collection.ensure_loaded()
 
1033
                r2._pack_collection.ensure_loaded()
 
1034
                self.assertEqual(r1._pack_collection.names(), r2._pack_collection.names())
 
1035
                self.assertEqual(2, len(r1._pack_collection.names()))
 
1036
            finally:
 
1037
                r2.unlock()
 
1038
        finally:
 
1039
            r1.unlock()
 
1040
 
 
1041
    def test_concurrent_writer_second_preserves_dropping_a_pack(self):
 
1042
        format = self.get_format()
 
1043
        self.make_repository('.', shared=True, format=format)
 
1044
        r1 = repository.Repository.open('.')
 
1045
        r2 = repository.Repository.open('.')
 
1046
        # add a pack to drop
 
1047
        r1.lock_write()
 
1048
        try:
 
1049
            r1.start_write_group()
 
1050
            try:
 
1051
                self._add_text(r1, 'fileidr1')
 
1052
            except:
 
1053
                r1.abort_write_group()
 
1054
                raise
 
1055
            else:
 
1056
                r1.commit_write_group()
 
1057
            r1._pack_collection.ensure_loaded()
 
1058
            name_to_drop = r1._pack_collection.all_packs()[0].name
 
1059
        finally:
 
1060
            r1.unlock()
 
1061
        r1.lock_write()
 
1062
        try:
 
1063
            # access enough data to load the names list
 
1064
            list(r1.all_revision_ids())
 
1065
            r2.lock_write()
 
1066
            try:
 
1067
                # access enough data to load the names list
 
1068
                list(r2.all_revision_ids())
 
1069
                r1._pack_collection.ensure_loaded()
 
1070
                try:
 
1071
                    r2.start_write_group()
 
1072
                    try:
 
1073
                        # in r1, drop the pack
 
1074
                        r1._pack_collection._remove_pack_from_memory(
 
1075
                            r1._pack_collection.get_pack_by_name(name_to_drop))
 
1076
                        # in r2, add a pack
 
1077
                        self._add_text(r2, 'fileidr2')
 
1078
                    except:
 
1079
                        r2.abort_write_group()
 
1080
                        raise
 
1081
                except:
 
1082
                    r1._pack_collection.reset()
 
1083
                    raise
 
1084
                # r1 has a changed names list, and r2 an open write groups with
 
1085
                # changes.
 
1086
                # save r1, and then commit the r2 write group, which requires a
 
1087
                # merge to the pack-names, which should not reinstate
 
1088
                # name_to_drop
 
1089
                try:
 
1090
                    r1._pack_collection._save_pack_names()
 
1091
                    r1._pack_collection.reset()
 
1092
                except:
 
1093
                    r2.abort_write_group()
 
1094
                    raise
 
1095
                try:
 
1096
                    r2.commit_write_group()
 
1097
                except:
 
1098
                    r2.abort_write_group()
 
1099
                    raise
 
1100
                # Now both repositories should now about just one name.
 
1101
                r1._pack_collection.ensure_loaded()
 
1102
                r2._pack_collection.ensure_loaded()
 
1103
                self.assertEqual(r1._pack_collection.names(), r2._pack_collection.names())
 
1104
                self.assertEqual(1, len(r1._pack_collection.names()))
 
1105
                self.assertFalse(name_to_drop in r1._pack_collection.names())
 
1106
            finally:
 
1107
                r2.unlock()
 
1108
        finally:
 
1109
            r1.unlock()
 
1110
 
 
1111
    def test_lock_write_does_not_physically_lock(self):
 
1112
        repo = self.make_repository('.', format=self.get_format())
 
1113
        repo.lock_write()
 
1114
        self.addCleanup(repo.unlock)
 
1115
        self.assertFalse(repo.get_physical_lock_status())
 
1116
 
 
1117
    def prepare_for_break_lock(self):
 
1118
        # Setup the global ui factory state so that a break-lock method call
 
1119
        # will find usable input in the input stream.
 
1120
        old_factory = bzrlib.ui.ui_factory
 
1121
        def restoreFactory():
 
1122
            bzrlib.ui.ui_factory = old_factory
 
1123
        self.addCleanup(restoreFactory)
 
1124
        bzrlib.ui.ui_factory = bzrlib.ui.SilentUIFactory()
 
1125
        bzrlib.ui.ui_factory.stdin = StringIO("y\n")
 
1126
 
 
1127
    def test_break_lock_breaks_physical_lock(self):
 
1128
        repo = self.make_repository('.', format=self.get_format())
 
1129
        repo._pack_collection.lock_names()
 
1130
        repo2 = repository.Repository.open('.')
 
1131
        self.assertTrue(repo.get_physical_lock_status())
 
1132
        self.prepare_for_break_lock()
 
1133
        repo2.break_lock()
 
1134
        self.assertFalse(repo.get_physical_lock_status())
 
1135
 
 
1136
    def test_broken_physical_locks_error_on__unlock_names_lock(self):
 
1137
        repo = self.make_repository('.', format=self.get_format())
 
1138
        repo._pack_collection.lock_names()
 
1139
        self.assertTrue(repo.get_physical_lock_status())
 
1140
        repo2 = repository.Repository.open('.')
 
1141
        self.prepare_for_break_lock()
 
1142
        repo2.break_lock()
 
1143
        self.assertRaises(errors.LockBroken, repo._pack_collection._unlock_names)
 
1144
 
 
1145
    def test_fetch_without_find_ghosts_ignores_ghosts(self):
 
1146
        # we want two repositories at this point:
 
1147
        # one with a revision that is a ghost in the other
 
1148
        # repository.
 
1149
        # 'ghost' is present in has_ghost, 'ghost' is absent in 'missing_ghost'.
 
1150
        # 'references' is present in both repositories, and 'tip' is present
 
1151
        # just in has_ghost.
 
1152
        # has_ghost       missing_ghost
 
1153
        #------------------------------
 
1154
        # 'ghost'             -
 
1155
        # 'references'    'references'
 
1156
        # 'tip'               -
 
1157
        # In this test we fetch 'tip' which should not fetch 'ghost'
 
1158
        has_ghost = self.make_repository('has_ghost', format=self.get_format())
 
1159
        missing_ghost = self.make_repository('missing_ghost',
 
1160
            format=self.get_format())
 
1161
 
 
1162
        def add_commit(repo, revision_id, parent_ids):
 
1163
            repo.lock_write()
 
1164
            repo.start_write_group()
 
1165
            inv = inventory.Inventory(revision_id=revision_id)
 
1166
            inv.root.revision = revision_id
 
1167
            root_id = inv.root.file_id
 
1168
            sha1 = repo.add_inventory(revision_id, inv, [])
 
1169
            vf = repo.weave_store.get_weave_or_empty(root_id,
 
1170
                repo.get_transaction())
 
1171
            vf.add_lines(revision_id, [], [])
 
1172
            rev = bzrlib.revision.Revision(timestamp=0,
 
1173
                                           timezone=None,
 
1174
                                           committer="Foo Bar <foo@example.com>",
 
1175
                                           message="Message",
 
1176
                                           inventory_sha1=sha1,
 
1177
                                           revision_id=revision_id)
 
1178
            rev.parent_ids = parent_ids
 
1179
            repo.add_revision(revision_id, rev)
 
1180
            repo.commit_write_group()
 
1181
            repo.unlock()
 
1182
        add_commit(has_ghost, 'ghost', [])
 
1183
        add_commit(has_ghost, 'references', ['ghost'])
 
1184
        add_commit(missing_ghost, 'references', ['ghost'])
 
1185
        add_commit(has_ghost, 'tip', ['references'])
 
1186
        missing_ghost.fetch(has_ghost, 'tip')
 
1187
        # missing ghost now has tip and not ghost.
 
1188
        rev = missing_ghost.get_revision('tip')
 
1189
        inv = missing_ghost.get_inventory('tip')
 
1190
        self.assertRaises(errors.NoSuchRevision,
 
1191
            missing_ghost.get_revision, 'ghost')
 
1192
        self.assertRaises(errors.RevisionNotPresent,
 
1193
            missing_ghost.get_inventory, 'ghost')
 
1194
 
 
1195
    def test_supports_external_lookups(self):
 
1196
        repo = self.make_repository('.', format=self.get_format())
 
1197
        self.assertFalse(repo._format.supports_external_lookups)
 
1198
 
 
1199
 
 
1200
class TestKnitPackSubtrees(TestKnitPackNoSubtrees):
 
1201
 
 
1202
    def get_format(self):
 
1203
        return bzrdir.format_registry.make_bzrdir(
 
1204
            'pack-0.92-subtree')
 
1205
 
 
1206
    def check_format(self, t):
 
1207
        self.assertEqualDiff(
 
1208
            "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n",
 
1209
            t.get('format').read())
 
1210
 
 
1211
 
 
1212
class TestDevelopment0(TestKnitPackNoSubtrees):
 
1213
 
 
1214
    def get_format(self):
 
1215
        return bzrdir.format_registry.make_bzrdir(
 
1216
            'development')
 
1217
 
 
1218
    def check_format(self, t):
 
1219
        self.assertEqualDiff(
 
1220
            "Bazaar development format 0 (needs bzr.dev from before 1.3)\n",
 
1221
            t.get('format').read())
 
1222
 
 
1223
 
 
1224
class TestDevelopment0Subtree(TestKnitPackNoSubtrees):
 
1225
 
 
1226
    def get_format(self):
 
1227
        return bzrdir.format_registry.make_bzrdir(
 
1228
            'development-subtree')
 
1229
 
 
1230
    def check_format(self, t):
 
1231
        self.assertEqualDiff(
 
1232
            "Bazaar development format 0 with subtree support "
 
1233
            "(needs bzr.dev from before 1.3)\n",
 
1234
            t.get('format').read())
1116
1235
 
1117
1236
 
1118
1237
class TestRepositoryPackCollection(TestCaseWithTransport):
1120
1239
    def get_format(self):
1121
1240
        return bzrdir.format_registry.make_bzrdir('pack-0.92')
1122
1241
 
1123
 
    def get_packs(self):
1124
 
        format = self.get_format()
1125
 
        repo = self.make_repository('.', format=format)
1126
 
        return repo._pack_collection
1127
 
 
1128
 
    def make_packs_and_alt_repo(self, write_lock=False):
1129
 
        """Create a pack repo with 3 packs, and access it via a second repo."""
1130
 
        tree = self.make_branch_and_tree('.', format=self.get_format())
1131
 
        tree.lock_write()
1132
 
        self.addCleanup(tree.unlock)
1133
 
        rev1 = tree.commit('one')
1134
 
        rev2 = tree.commit('two')
1135
 
        rev3 = tree.commit('three')
1136
 
        r = repository.Repository.open('.')
1137
 
        if write_lock:
1138
 
            r.lock_write()
1139
 
        else:
1140
 
            r.lock_read()
1141
 
        self.addCleanup(r.unlock)
1142
 
        packs = r._pack_collection
1143
 
        packs.ensure_loaded()
1144
 
        return tree, r, packs, [rev1, rev2, rev3]
1145
 
 
1146
 
    def test__clear_obsolete_packs(self):
1147
 
        packs = self.get_packs()
1148
 
        obsolete_pack_trans = packs.transport.clone('obsolete_packs')
1149
 
        obsolete_pack_trans.put_bytes('a-pack.pack', 'content\n')
1150
 
        obsolete_pack_trans.put_bytes('a-pack.rix', 'content\n')
1151
 
        obsolete_pack_trans.put_bytes('a-pack.iix', 'content\n')
1152
 
        obsolete_pack_trans.put_bytes('another-pack.pack', 'foo\n')
1153
 
        obsolete_pack_trans.put_bytes('not-a-pack.rix', 'foo\n')
1154
 
        res = packs._clear_obsolete_packs()
1155
 
        self.assertEqual(['a-pack', 'another-pack'], sorted(res))
1156
 
        self.assertEqual([], obsolete_pack_trans.list_dir('.'))
1157
 
 
1158
 
    def test__clear_obsolete_packs_preserve(self):
1159
 
        packs = self.get_packs()
1160
 
        obsolete_pack_trans = packs.transport.clone('obsolete_packs')
1161
 
        obsolete_pack_trans.put_bytes('a-pack.pack', 'content\n')
1162
 
        obsolete_pack_trans.put_bytes('a-pack.rix', 'content\n')
1163
 
        obsolete_pack_trans.put_bytes('a-pack.iix', 'content\n')
1164
 
        obsolete_pack_trans.put_bytes('another-pack.pack', 'foo\n')
1165
 
        obsolete_pack_trans.put_bytes('not-a-pack.rix', 'foo\n')
1166
 
        res = packs._clear_obsolete_packs(preserve=set(['a-pack']))
1167
 
        self.assertEqual(['a-pack', 'another-pack'], sorted(res))
1168
 
        self.assertEqual(['a-pack.iix', 'a-pack.pack', 'a-pack.rix'],
1169
 
                         sorted(obsolete_pack_trans.list_dir('.')))
1170
 
 
1171
1242
    def test__max_pack_count(self):
1172
1243
        """The maximum pack count is a function of the number of revisions."""
 
1244
        format = self.get_format()
 
1245
        repo = self.make_repository('.', format=format)
 
1246
        packs = repo._pack_collection
1173
1247
        # no revisions - one pack, so that we can have a revision free repo
1174
1248
        # without it blowing up
1175
 
        packs = self.get_packs()
1176
1249
        self.assertEqual(1, packs._max_pack_count(0))
1177
1250
        # after that the sum of the digits, - check the first 1-9
1178
1251
        self.assertEqual(1, packs._max_pack_count(1))
1193
1266
        # check some arbitrary big numbers
1194
1267
        self.assertEqual(25, packs._max_pack_count(112894))
1195
1268
 
1196
 
    def test_repr(self):
1197
 
        packs = self.get_packs()
1198
 
        self.assertContainsRe(repr(packs),
1199
 
            'RepositoryPackCollection(.*Repository(.*))')
1200
 
 
1201
 
    def test__obsolete_packs(self):
1202
 
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
1203
 
        names = packs.names()
1204
 
        pack = packs.get_pack_by_name(names[0])
1205
 
        # Schedule this one for removal
1206
 
        packs._remove_pack_from_memory(pack)
1207
 
        # Simulate a concurrent update by renaming the .pack file and one of
1208
 
        # the indices
1209
 
        packs.transport.rename('packs/%s.pack' % (names[0],),
1210
 
                               'obsolete_packs/%s.pack' % (names[0],))
1211
 
        packs.transport.rename('indices/%s.iix' % (names[0],),
1212
 
                               'obsolete_packs/%s.iix' % (names[0],))
1213
 
        # Now trigger the obsoletion, and ensure that all the remaining files
1214
 
        # are still renamed
1215
 
        packs._obsolete_packs([pack])
1216
 
        self.assertEqual([n + '.pack' for n in names[1:]],
1217
 
                         sorted(packs._pack_transport.list_dir('.')))
1218
 
        # names[0] should not be present in the index anymore
1219
 
        self.assertEqual(names[1:],
1220
 
            sorted(set([osutils.splitext(n)[0] for n in
1221
 
                        packs._index_transport.list_dir('.')])))
1222
 
 
1223
1269
    def test_pack_distribution_zero(self):
1224
 
        packs = self.get_packs()
 
1270
        format = self.get_format()
 
1271
        repo = self.make_repository('.', format=format)
 
1272
        packs = repo._pack_collection
1225
1273
        self.assertEqual([0], packs.pack_distribution(0))
1226
1274
 
1227
1275
    def test_ensure_loaded_unlocked(self):
1228
 
        packs = self.get_packs()
 
1276
        format = self.get_format()
 
1277
        repo = self.make_repository('.', format=format)
1229
1278
        self.assertRaises(errors.ObjectNotLocked,
1230
 
                          packs.ensure_loaded)
 
1279
                          repo._pack_collection.ensure_loaded)
1231
1280
 
1232
1281
    def test_pack_distribution_one_to_nine(self):
1233
 
        packs = self.get_packs()
 
1282
        format = self.get_format()
 
1283
        repo = self.make_repository('.', format=format)
 
1284
        packs = repo._pack_collection
1234
1285
        self.assertEqual([1],
1235
1286
            packs.pack_distribution(1))
1236
1287
        self.assertEqual([1, 1],
1252
1303
 
1253
1304
    def test_pack_distribution_stable_at_boundaries(self):
1254
1305
        """When there are multi-rev packs the counts are stable."""
1255
 
        packs = self.get_packs()
 
1306
        format = self.get_format()
 
1307
        repo = self.make_repository('.', format=format)
 
1308
        packs = repo._pack_collection
1256
1309
        # in 10s:
1257
1310
        self.assertEqual([10], packs.pack_distribution(10))
1258
1311
        self.assertEqual([10, 1], packs.pack_distribution(11))
1267
1320
        self.assertEqual([100, 100, 10, 1], packs.pack_distribution(211))
1268
1321
 
1269
1322
    def test_plan_pack_operations_2009_revisions_skip_all_packs(self):
1270
 
        packs = self.get_packs()
 
1323
        format = self.get_format()
 
1324
        repo = self.make_repository('.', format=format)
 
1325
        packs = repo._pack_collection
1271
1326
        existing_packs = [(2000, "big"), (9, "medium")]
1272
1327
        # rev count - 2009 -> 2x1000 + 9x1
1273
1328
        pack_operations = packs.plan_autopack_combinations(
1275
1330
        self.assertEqual([], pack_operations)
1276
1331
 
1277
1332
    def test_plan_pack_operations_2010_revisions_skip_all_packs(self):
1278
 
        packs = self.get_packs()
 
1333
        format = self.get_format()
 
1334
        repo = self.make_repository('.', format=format)
 
1335
        packs = repo._pack_collection
1279
1336
        existing_packs = [(2000, "big"), (9, "medium"), (1, "single")]
1280
1337
        # rev count - 2010 -> 2x1000 + 1x10
1281
1338
        pack_operations = packs.plan_autopack_combinations(
1283
1340
        self.assertEqual([], pack_operations)
1284
1341
 
1285
1342
    def test_plan_pack_operations_2010_combines_smallest_two(self):
1286
 
        packs = self.get_packs()
 
1343
        format = self.get_format()
 
1344
        repo = self.make_repository('.', format=format)
 
1345
        packs = repo._pack_collection
1287
1346
        existing_packs = [(1999, "big"), (9, "medium"), (1, "single2"),
1288
1347
            (1, "single1")]
1289
1348
        # rev count - 2010 -> 2x1000 + 1x10 (3)
1290
1349
        pack_operations = packs.plan_autopack_combinations(
1291
1350
            existing_packs, [1000, 1000, 10])
1292
 
        self.assertEqual([[2, ["single2", "single1"]]], pack_operations)
1293
 
 
1294
 
    def test_plan_pack_operations_creates_a_single_op(self):
1295
 
        packs = self.get_packs()
1296
 
        existing_packs = [(50, 'a'), (40, 'b'), (30, 'c'), (10, 'd'),
1297
 
                          (10, 'e'), (6, 'f'), (4, 'g')]
1298
 
        # rev count 150 -> 1x100 and 5x10
1299
 
        # The two size 10 packs do not need to be touched. The 50, 40, 30 would
1300
 
        # be combined into a single 120 size pack, and the 6 & 4 would
1301
 
        # becombined into a size 10 pack. However, if we have to rewrite them,
1302
 
        # we save a pack file with no increased I/O by putting them into the
1303
 
        # same file.
1304
 
        distribution = packs.pack_distribution(150)
1305
 
        pack_operations = packs.plan_autopack_combinations(existing_packs,
1306
 
                                                           distribution)
1307
 
        self.assertEqual([[130, ['a', 'b', 'c', 'f', 'g']]], pack_operations)
 
1351
        self.assertEqual([[2, ["single2", "single1"]], [0, []]], pack_operations)
1308
1352
 
1309
1353
    def test_all_packs_none(self):
1310
1354
        format = self.get_format()
1348
1392
        tree.lock_read()
1349
1393
        self.addCleanup(tree.unlock)
1350
1394
        packs = tree.branch.repository._pack_collection
1351
 
        packs.reset()
1352
1395
        packs.ensure_loaded()
1353
1396
        name = packs.names()[0]
1354
1397
        pack_1 = packs.get_pack_by_name(name)
1355
1398
        # the pack should be correctly initialised
1356
 
        sizes = packs._names[name]
1357
 
        rev_index = GraphIndex(packs._index_transport, name + '.rix', sizes[0])
1358
 
        inv_index = GraphIndex(packs._index_transport, name + '.iix', sizes[1])
1359
 
        txt_index = GraphIndex(packs._index_transport, name + '.tix', sizes[2])
1360
 
        sig_index = GraphIndex(packs._index_transport, name + '.six', sizes[3])
 
1399
        rev_index = GraphIndex(packs._index_transport, name + '.rix',
 
1400
            packs._names[name][0])
 
1401
        inv_index = GraphIndex(packs._index_transport, name + '.iix',
 
1402
            packs._names[name][1])
 
1403
        txt_index = GraphIndex(packs._index_transport, name + '.tix',
 
1404
            packs._names[name][2])
 
1405
        sig_index = GraphIndex(packs._index_transport, name + '.six',
 
1406
            packs._names[name][3])
1361
1407
        self.assertEqual(pack_repo.ExistingPack(packs._pack_transport,
1362
1408
            name, rev_index, inv_index, txt_index, sig_index), pack_1)
1363
1409
        # and the same instance should be returned on successive calls.
1364
1410
        self.assertTrue(pack_1 is packs.get_pack_by_name(name))
1365
1411
 
1366
 
    def test_reload_pack_names_new_entry(self):
1367
 
        tree, r, packs, revs = self.make_packs_and_alt_repo()
1368
 
        names = packs.names()
1369
 
        # Add a new pack file into the repository
1370
 
        rev4 = tree.commit('four')
1371
 
        new_names = tree.branch.repository._pack_collection.names()
1372
 
        new_name = set(new_names).difference(names)
1373
 
        self.assertEqual(1, len(new_name))
1374
 
        new_name = new_name.pop()
1375
 
        # The old collection hasn't noticed yet
1376
 
        self.assertEqual(names, packs.names())
1377
 
        self.assertTrue(packs.reload_pack_names())
1378
 
        self.assertEqual(new_names, packs.names())
1379
 
        # And the repository can access the new revision
1380
 
        self.assertEqual({rev4:(revs[-1],)}, r.get_parent_map([rev4]))
1381
 
        self.assertFalse(packs.reload_pack_names())
1382
 
 
1383
 
    def test_reload_pack_names_added_and_removed(self):
1384
 
        tree, r, packs, revs = self.make_packs_and_alt_repo()
1385
 
        names = packs.names()
1386
 
        # Now repack the whole thing
1387
 
        tree.branch.repository.pack()
1388
 
        new_names = tree.branch.repository._pack_collection.names()
1389
 
        # The other collection hasn't noticed yet
1390
 
        self.assertEqual(names, packs.names())
1391
 
        self.assertTrue(packs.reload_pack_names())
1392
 
        self.assertEqual(new_names, packs.names())
1393
 
        self.assertEqual({revs[-1]:(revs[-2],)}, r.get_parent_map([revs[-1]]))
1394
 
        self.assertFalse(packs.reload_pack_names())
1395
 
 
1396
 
    def test_reload_pack_names_preserves_pending(self):
1397
 
        # TODO: Update this to also test for pending-deleted names
1398
 
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
1399
 
        # We will add one pack (via start_write_group + insert_record_stream),
1400
 
        # and remove another pack (via _remove_pack_from_memory)
1401
 
        orig_names = packs.names()
1402
 
        orig_at_load = packs._packs_at_load
1403
 
        to_remove_name = iter(orig_names).next()
1404
 
        r.start_write_group()
1405
 
        self.addCleanup(r.abort_write_group)
1406
 
        r.texts.insert_record_stream([versionedfile.FulltextContentFactory(
1407
 
            ('text', 'rev'), (), None, 'content\n')])
1408
 
        new_pack = packs._new_pack
1409
 
        self.assertTrue(new_pack.data_inserted())
1410
 
        new_pack.finish()
1411
 
        packs.allocate(new_pack)
1412
 
        packs._new_pack = None
1413
 
        removed_pack = packs.get_pack_by_name(to_remove_name)
1414
 
        packs._remove_pack_from_memory(removed_pack)
1415
 
        names = packs.names()
1416
 
        all_nodes, deleted_nodes, new_nodes, _ = packs._diff_pack_names()
1417
 
        new_names = set([x[0][0] for x in new_nodes])
1418
 
        self.assertEqual(names, sorted([x[0][0] for x in all_nodes]))
1419
 
        self.assertEqual(set(names) - set(orig_names), new_names)
1420
 
        self.assertEqual(set([new_pack.name]), new_names)
1421
 
        self.assertEqual([to_remove_name],
1422
 
                         sorted([x[0][0] for x in deleted_nodes]))
1423
 
        packs.reload_pack_names()
1424
 
        reloaded_names = packs.names()
1425
 
        self.assertEqual(orig_at_load, packs._packs_at_load)
1426
 
        self.assertEqual(names, reloaded_names)
1427
 
        all_nodes, deleted_nodes, new_nodes, _ = packs._diff_pack_names()
1428
 
        new_names = set([x[0][0] for x in new_nodes])
1429
 
        self.assertEqual(names, sorted([x[0][0] for x in all_nodes]))
1430
 
        self.assertEqual(set(names) - set(orig_names), new_names)
1431
 
        self.assertEqual(set([new_pack.name]), new_names)
1432
 
        self.assertEqual([to_remove_name],
1433
 
                         sorted([x[0][0] for x in deleted_nodes]))
1434
 
 
1435
 
    def test_autopack_obsoletes_new_pack(self):
1436
 
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
1437
 
        packs._max_pack_count = lambda x: 1
1438
 
        packs.pack_distribution = lambda x: [10]
1439
 
        r.start_write_group()
1440
 
        r.revisions.insert_record_stream([versionedfile.FulltextContentFactory(
1441
 
            ('bogus-rev',), (), None, 'bogus-content\n')])
1442
 
        # This should trigger an autopack, which will combine everything into a
1443
 
        # single pack file.
1444
 
        new_names = r.commit_write_group()
1445
 
        names = packs.names()
1446
 
        self.assertEqual(1, len(names))
1447
 
        self.assertEqual([names[0] + '.pack'],
1448
 
                         packs._pack_transport.list_dir('.'))
1449
 
 
1450
 
    def test_autopack_reloads_and_stops(self):
1451
 
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
1452
 
        # After we have determined what needs to be autopacked, trigger a
1453
 
        # full-pack via the other repo which will cause us to re-evaluate and
1454
 
        # decide we don't need to do anything
1455
 
        orig_execute = packs._execute_pack_operations
1456
 
        def _munged_execute_pack_ops(*args, **kwargs):
1457
 
            tree.branch.repository.pack()
1458
 
            return orig_execute(*args, **kwargs)
1459
 
        packs._execute_pack_operations = _munged_execute_pack_ops
1460
 
        packs._max_pack_count = lambda x: 1
1461
 
        packs.pack_distribution = lambda x: [10]
1462
 
        self.assertFalse(packs.autopack())
1463
 
        self.assertEqual(1, len(packs.names()))
1464
 
        self.assertEqual(tree.branch.repository._pack_collection.names(),
1465
 
                         packs.names())
1466
 
 
1467
 
    def test__save_pack_names(self):
1468
 
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
1469
 
        names = packs.names()
1470
 
        pack = packs.get_pack_by_name(names[0])
1471
 
        packs._remove_pack_from_memory(pack)
1472
 
        packs._save_pack_names(obsolete_packs=[pack])
1473
 
        cur_packs = packs._pack_transport.list_dir('.')
1474
 
        self.assertEqual([n + '.pack' for n in names[1:]], sorted(cur_packs))
1475
 
        # obsolete_packs will also have stuff like .rix and .iix present.
1476
 
        obsolete_packs = packs.transport.list_dir('obsolete_packs')
1477
 
        obsolete_names = set([osutils.splitext(n)[0] for n in obsolete_packs])
1478
 
        self.assertEqual([pack.name], sorted(obsolete_names))
1479
 
 
1480
 
    def test__save_pack_names_already_obsoleted(self):
1481
 
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
1482
 
        names = packs.names()
1483
 
        pack = packs.get_pack_by_name(names[0])
1484
 
        packs._remove_pack_from_memory(pack)
1485
 
        # We are going to simulate a concurrent autopack by manually obsoleting
1486
 
        # the pack directly.
1487
 
        packs._obsolete_packs([pack])
1488
 
        packs._save_pack_names(clear_obsolete_packs=True,
1489
 
                               obsolete_packs=[pack])
1490
 
        cur_packs = packs._pack_transport.list_dir('.')
1491
 
        self.assertEqual([n + '.pack' for n in names[1:]], sorted(cur_packs))
1492
 
        # Note that while we set clear_obsolete_packs=True, it should not
1493
 
        # delete a pack file that we have also scheduled for obsoletion.
1494
 
        obsolete_packs = packs.transport.list_dir('obsolete_packs')
1495
 
        obsolete_names = set([osutils.splitext(n)[0] for n in obsolete_packs])
1496
 
        self.assertEqual([pack.name], sorted(obsolete_names))
1497
 
 
1498
 
 
1499
1412
 
1500
1413
class TestPack(TestCaseWithTransport):
1501
1414
    """Tests for the Pack object."""
1555
1468
        pack_transport = self.get_transport('pack')
1556
1469
        index_transport = self.get_transport('index')
1557
1470
        upload_transport.mkdir('.')
1558
 
        collection = pack_repo.RepositoryPackCollection(
1559
 
            repo=None,
1560
 
            transport=self.get_transport('.'),
1561
 
            index_transport=index_transport,
1562
 
            upload_transport=upload_transport,
1563
 
            pack_transport=pack_transport,
1564
 
            index_builder_class=BTreeBuilder,
1565
 
            index_class=BTreeGraphIndex,
1566
 
            use_chk_index=False)
1567
 
        pack = pack_repo.NewPack(collection)
1568
 
        self.addCleanup(pack.abort) # Make sure the write stream gets closed
1569
 
        self.assertIsInstance(pack.revision_index, BTreeBuilder)
1570
 
        self.assertIsInstance(pack.inventory_index, BTreeBuilder)
1571
 
        self.assertIsInstance(pack._hash, type(osutils.md5()))
 
1471
        pack = pack_repo.NewPack(upload_transport, index_transport,
 
1472
            pack_transport)
 
1473
        self.assertIsInstance(pack.revision_index, InMemoryGraphIndex)
 
1474
        self.assertIsInstance(pack.inventory_index, InMemoryGraphIndex)
 
1475
        self.assertIsInstance(pack._hash, type(md5.new()))
1572
1476
        self.assertTrue(pack.upload_transport is upload_transport)
1573
1477
        self.assertTrue(pack.index_transport is index_transport)
1574
1478
        self.assertTrue(pack.pack_transport is pack_transport)
1581
1485
class TestPacker(TestCaseWithTransport):
1582
1486
    """Tests for the packs repository Packer class."""
1583
1487
 
1584
 
    def test_pack_optimizes_pack_order(self):
1585
 
        builder = self.make_branch_builder('.', format="1.9")
1586
 
        builder.start_series()
1587
 
        builder.build_snapshot('A', None, [
1588
 
            ('add', ('', 'root-id', 'directory', None)),
1589
 
            ('add', ('f', 'f-id', 'file', 'content\n'))])
1590
 
        builder.build_snapshot('B', ['A'],
1591
 
            [('modify', ('f-id', 'new-content\n'))])
1592
 
        builder.build_snapshot('C', ['B'],
1593
 
            [('modify', ('f-id', 'third-content\n'))])
1594
 
        builder.build_snapshot('D', ['C'],
1595
 
            [('modify', ('f-id', 'fourth-content\n'))])
1596
 
        b = builder.get_branch()
1597
 
        b.lock_read()
1598
 
        builder.finish_series()
1599
 
        self.addCleanup(b.unlock)
1600
 
        # At this point, we should have 4 pack files available
1601
 
        # Because of how they were built, they correspond to
1602
 
        # ['D', 'C', 'B', 'A']
1603
 
        packs = b.repository._pack_collection.packs
1604
 
        packer = pack_repo.Packer(b.repository._pack_collection,
1605
 
                                  packs, 'testing',
1606
 
                                  revision_ids=['B', 'C'])
1607
 
        # Now, when we are copying the B & C revisions, their pack files should
1608
 
        # be moved to the front of the stack
1609
 
        # The new ordering moves B & C to the front of the .packs attribute,
1610
 
        # and leaves the others in the original order.
1611
 
        new_packs = [packs[1], packs[2], packs[0], packs[3]]
1612
 
        new_pack = packer.pack()
1613
 
        self.assertEqual(new_packs, packer.packs)
1614
 
 
1615
 
 
1616
 
class TestOptimisingPacker(TestCaseWithTransport):
1617
 
    """Tests for the OptimisingPacker class."""
1618
 
 
1619
 
    def get_pack_collection(self):
1620
 
        repo = self.make_repository('.')
1621
 
        return repo._pack_collection
1622
 
 
1623
 
    def test_open_pack_will_optimise(self):
1624
 
        packer = pack_repo.OptimisingPacker(self.get_pack_collection(),
1625
 
                                            [], '.test')
1626
 
        new_pack = packer.open_pack()
1627
 
        self.addCleanup(new_pack.abort) # ensure cleanup
1628
 
        self.assertIsInstance(new_pack, pack_repo.NewPack)
1629
 
        self.assertTrue(new_pack.revision_index._optimize_for_size)
1630
 
        self.assertTrue(new_pack.inventory_index._optimize_for_size)
1631
 
        self.assertTrue(new_pack.text_index._optimize_for_size)
1632
 
        self.assertTrue(new_pack.signature_index._optimize_for_size)
1633
 
 
1634
 
 
1635
 
class TestCrossFormatPacks(TestCaseWithTransport):
1636
 
 
1637
 
    def log_pack(self, hint=None):
1638
 
        self.calls.append(('pack', hint))
1639
 
        self.orig_pack(hint=hint)
1640
 
        if self.expect_hint:
1641
 
            self.assertTrue(hint)
1642
 
 
1643
 
    def run_stream(self, src_fmt, target_fmt, expect_pack_called):
1644
 
        self.expect_hint = expect_pack_called
1645
 
        self.calls = []
1646
 
        source_tree = self.make_branch_and_tree('src', format=src_fmt)
1647
 
        source_tree.lock_write()
1648
 
        self.addCleanup(source_tree.unlock)
1649
 
        tip = source_tree.commit('foo')
1650
 
        target = self.make_repository('target', format=target_fmt)
1651
 
        target.lock_write()
1652
 
        self.addCleanup(target.unlock)
1653
 
        source = source_tree.branch.repository._get_source(target._format)
1654
 
        self.orig_pack = target.pack
1655
 
        target.pack = self.log_pack
1656
 
        search = target.search_missing_revision_ids(
1657
 
            source_tree.branch.repository, tip)
1658
 
        stream = source.get_stream(search)
1659
 
        from_format = source_tree.branch.repository._format
1660
 
        sink = target._get_sink()
1661
 
        sink.insert_stream(stream, from_format, [])
1662
 
        if expect_pack_called:
1663
 
            self.assertLength(1, self.calls)
1664
 
        else:
1665
 
            self.assertLength(0, self.calls)
1666
 
 
1667
 
    def run_fetch(self, src_fmt, target_fmt, expect_pack_called):
1668
 
        self.expect_hint = expect_pack_called
1669
 
        self.calls = []
1670
 
        source_tree = self.make_branch_and_tree('src', format=src_fmt)
1671
 
        source_tree.lock_write()
1672
 
        self.addCleanup(source_tree.unlock)
1673
 
        tip = source_tree.commit('foo')
1674
 
        target = self.make_repository('target', format=target_fmt)
1675
 
        target.lock_write()
1676
 
        self.addCleanup(target.unlock)
1677
 
        source = source_tree.branch.repository
1678
 
        self.orig_pack = target.pack
1679
 
        target.pack = self.log_pack
1680
 
        target.fetch(source)
1681
 
        if expect_pack_called:
1682
 
            self.assertLength(1, self.calls)
1683
 
        else:
1684
 
            self.assertLength(0, self.calls)
1685
 
 
1686
 
    def test_sink_format_hint_no(self):
1687
 
        # When the target format says packing makes no difference, pack is not
1688
 
        # called.
1689
 
        self.run_stream('1.9', 'rich-root-pack', False)
1690
 
 
1691
 
    def test_sink_format_hint_yes(self):
1692
 
        # When the target format says packing makes a difference, pack is
1693
 
        # called.
1694
 
        self.run_stream('1.9', '2a', True)
1695
 
 
1696
 
    def test_sink_format_same_no(self):
1697
 
        # When the formats are the same, pack is not called.
1698
 
        self.run_stream('2a', '2a', False)
1699
 
 
1700
 
    def test_IDS_format_hint_no(self):
1701
 
        # When the target format says packing makes no difference, pack is not
1702
 
        # called.
1703
 
        self.run_fetch('1.9', 'rich-root-pack', False)
1704
 
 
1705
 
    def test_IDS_format_hint_yes(self):
1706
 
        # When the target format says packing makes a difference, pack is
1707
 
        # called.
1708
 
        self.run_fetch('1.9', '2a', True)
1709
 
 
1710
 
    def test_IDS_format_same_no(self):
1711
 
        # When the formats are the same, pack is not called.
1712
 
        self.run_fetch('2a', '2a', False)
 
1488
    # To date, this class has been factored out and nothing new added to it;
 
1489
    # thus there are not yet any tests.
 
1490
 
 
1491
 
 
1492
class TestInterDifferingSerializer(TestCaseWithTransport):
 
1493
 
 
1494
    def test_progress_bar(self):
 
1495
        tree = self.make_branch_and_tree('tree')
 
1496
        tree.commit('rev1', rev_id='rev-1')
 
1497
        tree.commit('rev2', rev_id='rev-2')
 
1498
        tree.commit('rev3', rev_id='rev-3')
 
1499
        repo = self.make_repository('repo')
 
1500
        inter_repo = repository.InterDifferingSerializer(
 
1501
            tree.branch.repository, repo)
 
1502
        pb = progress.InstrumentedProgress(to_file=StringIO())
 
1503
        pb.never_throttle = True
 
1504
        inter_repo.fetch('rev-1', pb)
 
1505
        self.assertEqual('Transferring revisions', pb.last_msg)
 
1506
        self.assertEqual(1, pb.last_cnt)
 
1507
        self.assertEqual(1, pb.last_total)
 
1508
        inter_repo.fetch('rev-3', pb)
 
1509
        self.assertEqual(2, pb.last_cnt)
 
1510
        self.assertEqual(2, pb.last_total)