~bzr-pqm/bzr/bzr.dev

# Copyright (C) 2006-2010 Canonical Ltd

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

"""Weave-era BzrDir formats."""

from __future__ import absolute_import

from bzrlib.bzrdir import (

    BzrDir,

    BzrDirFormat,

    BzrDirMetaFormat1,

    )

from bzrlib.controldir import (

    ControlDir,

    Converter,

    format_registry,

    )

from bzrlib.lazy_import import lazy_import

lazy_import(globals(), """

import os

import warnings

from bzrlib import (

    errors,

    graph,

    lockable_files,

    lockdir,

    osutils,

    revision as _mod_revision,

    trace,

    ui,

    urlutils,

    versionedfile,

    weave,

    xml5,

    )

from bzrlib.i18n import gettext

from bzrlib.store.versioned import VersionedFileStore

from bzrlib.transactions import WriteTransaction

from bzrlib.transport import (

    get_transport,

    local,

    )

from bzrlib.plugins.weave_fmt import xml4

""")

class BzrDirFormatAllInOne(BzrDirFormat):

    """Common class for formats before meta-dirs."""

    fixed_components = True

    def initialize_on_transport_ex(self, transport, use_existing_dir=False,

        create_prefix=False, force_new_repo=False, stacked_on=None,

        stack_on_pwd=None, repo_format_name=None, make_working_trees=None,

        shared_repo=False):

        """See ControlDir.initialize_on_transport_ex."""

        require_stacking = (stacked_on is not None)

        # Format 5 cannot stack, but we've been asked to - actually init

        # a Meta1Dir

        if require_stacking:

            format = BzrDirMetaFormat1()

            return format.initialize_on_transport_ex(transport,

                use_existing_dir=use_existing_dir, create_prefix=create_prefix,

                force_new_repo=force_new_repo, stacked_on=stacked_on,

                stack_on_pwd=stack_on_pwd, repo_format_name=repo_format_name,

                make_working_trees=make_working_trees, shared_repo=shared_repo)

        return BzrDirFormat.initialize_on_transport_ex(self, transport,

            use_existing_dir=use_existing_dir, create_prefix=create_prefix,

            force_new_repo=force_new_repo, stacked_on=stacked_on,

            stack_on_pwd=stack_on_pwd, repo_format_name=repo_format_name,

            make_working_trees=make_working_trees, shared_repo=shared_repo)

    @classmethod

    def from_string(cls, format_string):

        if format_string != cls.get_format_string():

            raise AssertionError("unexpected format string %r" % format_string)

        return cls()

class BzrDirFormat5(BzrDirFormatAllInOne):

    """Bzr control format 5.

    This format is a combined format for working tree, branch and repository.

    It has:

     - Format 2 working trees [always]

     - Format 4 branches [always]

     - Format 5 repositories [always]

       Unhashed stores in the repository.

    """

    _lock_class = lockable_files.TransportLock

    def __eq__(self, other):

        return type(self) == type(other)

    @classmethod

    def get_format_string(cls):

        """See BzrDirFormat.get_format_string()."""

        return "Bazaar-NG branch, format 5\n"

    def get_branch_format(self):

        from bzrlib.plugins.weave_fmt.branch import BzrBranchFormat4

        return BzrBranchFormat4()

    def get_format_description(self):

        """See ControlDirFormat.get_format_description()."""

        return "All-in-one format 5"

    def get_converter(self, format=None):

        """See ControlDirFormat.get_converter()."""

        # there is one and only one upgrade path here.

        return ConvertBzrDir5To6()

    def _initialize_for_clone(self, url):

        return self.initialize_on_transport(get_transport(url), _cloning=True)

    def initialize_on_transport(self, transport, _cloning=False):

        """Format 5 dirs always have working tree, branch and repository.

        Except when they are being cloned.

        """

        from bzrlib.plugins.weave_fmt.branch import BzrBranchFormat4

        from bzrlib.plugins.weave_fmt.repository import RepositoryFormat5

        result = (super(BzrDirFormat5, self).initialize_on_transport(transport))

        RepositoryFormat5().initialize(result, _internal=True)

        if not _cloning:

            branch = BzrBranchFormat4().initialize(result)

            result._init_workingtree()

        return result

    def network_name(self):

        return self.get_format_string()

    def _open(self, transport):

        """See BzrDirFormat._open."""

        return BzrDir5(transport, self)

    def __return_repository_format(self):

        """Circular import protection."""

        from bzrlib.plugins.weave_fmt.repository import RepositoryFormat5

        return RepositoryFormat5()

    repository_format = property(__return_repository_format)

class BzrDirFormat6(BzrDirFormatAllInOne):

    """Bzr control format 6.

    This format is a combined format for working tree, branch and repository.

    It has:

     - Format 2 working trees [always]

     - Format 4 branches [always]

     - Format 6 repositories [always]

    """

    _lock_class = lockable_files.TransportLock

    def __eq__(self, other):

        return type(self) == type(other)

    @classmethod

    def get_format_string(cls):

        """See BzrDirFormat.get_format_string()."""

        return "Bazaar-NG branch, format 6\n"

    def get_format_description(self):

        """See ControlDirFormat.get_format_description()."""

        return "All-in-one format 6"

    def get_branch_format(self):

        from bzrlib.plugins.weave_fmt.branch import BzrBranchFormat4

        return BzrBranchFormat4()

    def get_converter(self, format=None):

        """See ControlDirFormat.get_converter()."""

        # there is one and only one upgrade path here.

        return ConvertBzrDir6ToMeta()

    def _initialize_for_clone(self, url):

        return self.initialize_on_transport(get_transport(url), _cloning=True)

    def initialize_on_transport(self, transport, _cloning=False):

        """Format 6 dirs always have working tree, branch and repository.

        Except when they are being cloned.

        """

        from bzrlib.plugins.weave_fmt.branch import BzrBranchFormat4

        from bzrlib.plugins.weave_fmt.repository import RepositoryFormat6

        result = super(BzrDirFormat6, self).initialize_on_transport(transport)

        RepositoryFormat6().initialize(result, _internal=True)

        if not _cloning:

            branch = BzrBranchFormat4().initialize(result)

            result._init_workingtree()

        return result

    def network_name(self):

        return self.get_format_string()

    def _open(self, transport):

        """See BzrDirFormat._open."""

        return BzrDir6(transport, self)

    def __return_repository_format(self):

        """Circular import protection."""

        from bzrlib.plugins.weave_fmt.repository import RepositoryFormat6

        return RepositoryFormat6()

    repository_format = property(__return_repository_format)

class ConvertBzrDir4To5(Converter):

    """Converts format 4 bzr dirs to format 5."""

    def __init__(self):

        super(ConvertBzrDir4To5, self).__init__()

        self.converted_revs = set()

        self.absent_revisions = set()

        self.text_count = 0

        self.revisions = {}

    def convert(self, to_convert, pb):

        """See Converter.convert()."""

        self.bzrdir = to_convert

        if pb is not None:

            warnings.warn(gettext("pb parameter to convert() is deprecated"))

        self.pb = ui.ui_factory.nested_progress_bar()

        try:

            ui.ui_factory.note(gettext('starting upgrade from format 4 to 5'))

            if isinstance(self.bzrdir.transport, local.LocalTransport):

                self.bzrdir.get_workingtree_transport(None).delete('stat-cache')

            self._convert_to_weaves()

            return ControlDir.open(self.bzrdir.user_url)

        finally:

            self.pb.finished()

    def _convert_to_weaves(self):

        ui.ui_factory.note(gettext(

          'note: upgrade may be faster if all store files are ungzipped first'))

        try:

            # TODO permissions

            stat = self.bzrdir.transport.stat('weaves')

            if not S_ISDIR(stat.st_mode):

                self.bzrdir.transport.delete('weaves')

                self.bzrdir.transport.mkdir('weaves')

        except errors.NoSuchFile:

            self.bzrdir.transport.mkdir('weaves')

        # deliberately not a WeaveFile as we want to build it up slowly.

        self.inv_weave = weave.Weave('inventory')

        # holds in-memory weaves for all files

        self.text_weaves = {}

        self.bzrdir.transport.delete('branch-format')

        self.branch = self.bzrdir.open_branch()

        self._convert_working_inv()

        rev_history = self.branch._revision_history()

        # to_read is a stack holding the revisions we still need to process;

        # appending to it adds new highest-priority revisions

        self.known_revisions = set(rev_history)

        self.to_read = rev_history[-1:]

        while self.to_read:

            rev_id = self.to_read.pop()

            if (rev_id not in self.revisions

                and rev_id not in self.absent_revisions):

                self._load_one_rev(rev_id)

        self.pb.clear()

        to_import = self._make_order()

        for i, rev_id in enumerate(to_import):

            self.pb.update(gettext('converting revision'), i, len(to_import))

            self._convert_one_rev(rev_id)

        self.pb.clear()

        self._write_all_weaves()

        self._write_all_revs()

        ui.ui_factory.note(gettext('upgraded to weaves:'))

        ui.ui_factory.note('  ' + gettext('%6d revisions and inventories') %

                                                        len(self.revisions))

        ui.ui_factory.note('  ' + gettext('%6d revisions not present') %

                                                    len(self.absent_revisions))

        ui.ui_factory.note('  ' + gettext('%6d texts') % self.text_count)

        self._cleanup_spare_files_after_format4()

        self.branch._transport.put_bytes(

            'branch-format',

            BzrDirFormat5().get_format_string(),

            mode=self.bzrdir._get_file_mode())

    def _cleanup_spare_files_after_format4(self):

        # FIXME working tree upgrade foo.

        for n in 'merged-patches', 'pending-merged-patches':

            try:

                ## assert os.path.getsize(p) == 0

                self.bzrdir.transport.delete(n)

            except errors.NoSuchFile:

                pass

        self.bzrdir.transport.delete_tree('inventory-store')

        self.bzrdir.transport.delete_tree('text-store')

    def _convert_working_inv(self):

        inv = xml4.serializer_v4.read_inventory(

                self.branch._transport.get('inventory'))

        new_inv_xml = xml5.serializer_v5.write_inventory_to_string(inv, working=True)

        self.branch._transport.put_bytes('inventory', new_inv_xml,

            mode=self.bzrdir._get_file_mode())

    def _write_all_weaves(self):

        controlweaves = VersionedFileStore(self.bzrdir.transport, prefixed=False,

            versionedfile_class=weave.WeaveFile)

        weave_transport = self.bzrdir.transport.clone('weaves')

        weaves = VersionedFileStore(weave_transport, prefixed=False,

                versionedfile_class=weave.WeaveFile)

        transaction = WriteTransaction()

        try:

            i = 0

            for file_id, file_weave in self.text_weaves.items():

                self.pb.update(gettext('writing weave'), i,

                                                        len(self.text_weaves))

                weaves._put_weave(file_id, file_weave, transaction)

                i += 1

            self.pb.update(gettext('inventory'), 0, 1)

            controlweaves._put_weave('inventory', self.inv_weave, transaction)

            self.pb.update(gettext('inventory'), 1, 1)

        finally:

            self.pb.clear()

    def _write_all_revs(self):

        """Write all revisions out in new form."""

        self.bzrdir.transport.delete_tree('revision-store')

        self.bzrdir.transport.mkdir('revision-store')

        revision_transport = self.bzrdir.transport.clone('revision-store')

        # TODO permissions

        from bzrlib.xml5 import serializer_v5

        from bzrlib.plugins.weave_fmt.repository import RevisionTextStore

        revision_store = RevisionTextStore(revision_transport,

            serializer_v5, False, versionedfile.PrefixMapper(),

            lambda:True, lambda:True)

        try:

            for i, rev_id in enumerate(self.converted_revs):

                self.pb.update(gettext('write revision'), i,

                                                len(self.converted_revs))

                text = serializer_v5.write_revision_to_string(

                    self.revisions[rev_id])

                key = (rev_id,)

                revision_store.add_lines(key, None, osutils.split_lines(text))

        finally:

            self.pb.clear()

    def _load_one_rev(self, rev_id):

        """Load a revision object into memory.

        Any parents not either loaded or abandoned get queued to be

        loaded."""

        self.pb.update(gettext('loading revision'),

                       len(self.revisions),

                       len(self.known_revisions))

        if not self.branch.repository.has_revision(rev_id):

            self.pb.clear()

            ui.ui_factory.note(gettext('revision {%s} not present in branch; '

                         'will be converted as a ghost') %

                         rev_id)

            self.absent_revisions.add(rev_id)

        else:

            rev = self.branch.repository.get_revision(rev_id)

            for parent_id in rev.parent_ids:

                self.known_revisions.add(parent_id)

                self.to_read.append(parent_id)

            self.revisions[rev_id] = rev

    def _load_old_inventory(self, rev_id):

        f = self.branch.repository.inventory_store.get(rev_id)

        try:

            old_inv_xml = f.read()

        finally:

            f.close()

        inv = xml4.serializer_v4.read_inventory_from_string(old_inv_xml)

        inv.revision_id = rev_id

        rev = self.revisions[rev_id]

        return inv

    def _load_updated_inventory(self, rev_id):

        inv_xml = self.inv_weave.get_text(rev_id)

        inv = xml5.serializer_v5.read_inventory_from_string(inv_xml, rev_id)

        return inv

    def _convert_one_rev(self, rev_id):

        """Convert revision and all referenced objects to new format."""

        rev = self.revisions[rev_id]

        inv = self._load_old_inventory(rev_id)

        present_parents = [p for p in rev.parent_ids

                           if p not in self.absent_revisions]

        self._convert_revision_contents(rev, inv, present_parents)

        self._store_new_inv(rev, inv, present_parents)

        self.converted_revs.add(rev_id)

    def _store_new_inv(self, rev, inv, present_parents):

        new_inv_xml = xml5.serializer_v5.write_inventory_to_string(inv)

        new_inv_sha1 = osutils.sha_string(new_inv_xml)

        self.inv_weave.add_lines(rev.revision_id,

                                 present_parents,

                                 new_inv_xml.splitlines(True))

        rev.inventory_sha1 = new_inv_sha1

    def _convert_revision_contents(self, rev, inv, present_parents):

        """Convert all the files within a revision.

        Also upgrade the inventory to refer to the text revision ids."""

        rev_id = rev.revision_id

        trace.mutter('converting texts of revision {%s}', rev_id)

        parent_invs = map(self._load_updated_inventory, present_parents)

        entries = inv.iter_entries()

        entries.next()

        for path, ie in entries:

            self._convert_file_version(rev, ie, parent_invs)

    def _convert_file_version(self, rev, ie, parent_invs):

        """Convert one version of one file.

        The file needs to be added into the weave if it is a merge

        of >=2 parents or if it's changed from its parent.

        """

        file_id = ie.file_id

        rev_id = rev.revision_id

        w = self.text_weaves.get(file_id)

        if w is None:

            w = weave.Weave(file_id)

            self.text_weaves[file_id] = w

        text_changed = False

        parent_candiate_entries = ie.parent_candidates(parent_invs)

        heads = graph.Graph(self).heads(parent_candiate_entries.keys())

        # XXX: Note that this is unordered - and this is tolerable because

        # the previous code was also unordered.

        previous_entries = dict((head, parent_candiate_entries[head]) for head

            in heads)

        self.snapshot_ie(previous_entries, ie, w, rev_id)

    def get_parent_map(self, revision_ids):

        """See graph.StackedParentsProvider.get_parent_map"""

        return dict((revision_id, self.revisions[revision_id])

                    for revision_id in revision_ids

                     if revision_id in self.revisions)

    def snapshot_ie(self, previous_revisions, ie, w, rev_id):

        # TODO: convert this logic, which is ~= snapshot to

        # a call to:. This needs the path figured out. rather than a work_tree

        # a v4 revision_tree can be given, or something that looks enough like

        # one to give the file content to the entry if it needs it.

        # and we need something that looks like a weave store for snapshot to

        # save against.

        #ie.snapshot(rev, PATH, previous_revisions, REVISION_TREE, InMemoryWeaveStore(self.text_weaves))

        if len(previous_revisions) == 1:

            previous_ie = previous_revisions.values()[0]

            if ie._unchanged(previous_ie):

                ie.revision = previous_ie.revision

                return

        if ie.has_text():

            f = self.branch.repository._text_store.get(ie.text_id)

            try:

                file_lines = f.readlines()

            finally:

                f.close()

            w.add_lines(rev_id, previous_revisions, file_lines)

            self.text_count += 1

        else:

            w.add_lines(rev_id, previous_revisions, [])

        ie.revision = rev_id

    def _make_order(self):

        """Return a suitable order for importing revisions.

        The order must be such that an revision is imported after all

        its (present) parents.

        """

        todo = set(self.revisions.keys())

        done = self.absent_revisions.copy()

        order = []

        while todo:

            # scan through looking for a revision whose parents

            # are all done

            for rev_id in sorted(list(todo)):

                rev = self.revisions[rev_id]

                parent_ids = set(rev.parent_ids)

                if parent_ids.issubset(done):

                    # can take this one now

                    order.append(rev_id)

                    todo.remove(rev_id)

                    done.add(rev_id)

        return order

class ConvertBzrDir5To6(Converter):

    """Converts format 5 bzr dirs to format 6."""

    def convert(self, to_convert, pb):

        """See Converter.convert()."""

        self.bzrdir = to_convert

        pb = ui.ui_factory.nested_progress_bar()

        try:

            ui.ui_factory.note(gettext('starting upgrade from format 5 to 6'))

            self._convert_to_prefixed()

            return ControlDir.open(self.bzrdir.user_url)

        finally:

            pb.finished()

    def _convert_to_prefixed(self):

        from bzrlib.store import TransportStore

        self.bzrdir.transport.delete('branch-format')

        for store_name in ["weaves", "revision-store"]:

            ui.ui_factory.note(gettext("adding prefixes to %s") % store_name)

            store_transport = self.bzrdir.transport.clone(store_name)

            store = TransportStore(store_transport, prefixed=True)

            for urlfilename in store_transport.list_dir('.'):

                filename = urlutils.unescape(urlfilename)

                if (filename.endswith(".weave") or

                    filename.endswith(".gz") or

                    filename.endswith(".sig")):

                    file_id, suffix = os.path.splitext(filename)

                else:

                    file_id = filename

                    suffix = ''

                new_name = store._mapper.map((file_id,)) + suffix

                # FIXME keep track of the dirs made RBC 20060121

                try:

                    store_transport.move(filename, new_name)

                except errors.NoSuchFile: # catches missing dirs strangely enough

                    store_transport.mkdir(osutils.dirname(new_name))

                    store_transport.move(filename, new_name)

        self.bzrdir.transport.put_bytes(

            'branch-format',

            BzrDirFormat6().get_format_string(),

            mode=self.bzrdir._get_file_mode())

class ConvertBzrDir6ToMeta(Converter):

    """Converts format 6 bzr dirs to metadirs."""

    def convert(self, to_convert, pb):

        """See Converter.convert()."""

        from bzrlib.plugins.weave_fmt.repository import RepositoryFormat7

        from bzrlib.branchfmt.fullhistory import BzrBranchFormat5

        self.bzrdir = to_convert

        self.pb = ui.ui_factory.nested_progress_bar()

        self.count = 0

        self.total = 20 # the steps we know about

        self.garbage_inventories = []

        self.dir_mode = self.bzrdir._get_dir_mode()

        self.file_mode = self.bzrdir._get_file_mode()

        ui.ui_factory.note(gettext('starting upgrade from format 6 to metadir'))

        self.bzrdir.transport.put_bytes(

                'branch-format',

                "Converting to format 6",

                mode=self.file_mode)

        # its faster to move specific files around than to open and use the apis...

        # first off, nuke ancestry.weave, it was never used.

        try:

            self.step(gettext('Removing ancestry.weave'))

            self.bzrdir.transport.delete('ancestry.weave')

        except errors.NoSuchFile:

            pass

        # find out whats there

        self.step(gettext('Finding branch files'))

        last_revision = self.bzrdir.open_branch().last_revision()

        bzrcontents = self.bzrdir.transport.list_dir('.')

        for name in bzrcontents:

            if name.startswith('basis-inventory.'):

                self.garbage_inventories.append(name)

        # create new directories for repository, working tree and branch

        repository_names = [('inventory.weave', True),

                            ('revision-store', True),

                            ('weaves', True)]

        self.step(gettext('Upgrading repository') + '  ')

        self.bzrdir.transport.mkdir('repository', mode=self.dir_mode)

        self.make_lock('repository')

        # we hard code the formats here because we are converting into

        # the meta format. The meta format upgrader can take this to a

        # future format within each component.

        self.put_format('repository', RepositoryFormat7())

        for entry in repository_names:

            self.move_entry('repository', entry)

        self.step(gettext('Upgrading branch') + '      ')

        self.bzrdir.transport.mkdir('branch', mode=self.dir_mode)

        self.make_lock('branch')

        self.put_format('branch', BzrBranchFormat5())

        branch_files = [('revision-history', True),

                        ('branch-name', True),

                        ('parent', False)]

        for entry in branch_files:

            self.move_entry('branch', entry)

        checkout_files = [('pending-merges', True),

                          ('inventory', True),

                          ('stat-cache', False)]

        # If a mandatory checkout file is not present, the branch does not have

        # a functional checkout. Do not create a checkout in the converted

        # branch.

        for name, mandatory in checkout_files:

            if mandatory and name not in bzrcontents:

                has_checkout = False

                break

        else:

            has_checkout = True

        if not has_checkout:

            ui.ui_factory.note(gettext('No working tree.'))

            # If some checkout files are there, we may as well get rid of them.

            for name, mandatory in checkout_files:

                if name in bzrcontents:

                    self.bzrdir.transport.delete(name)

        else:

            from bzrlib.workingtree_3 import WorkingTreeFormat3

            self.step(gettext('Upgrading working tree'))

            self.bzrdir.transport.mkdir('checkout', mode=self.dir_mode)

            self.make_lock('checkout')

            self.put_format(

                'checkout', WorkingTreeFormat3())

            self.bzrdir.transport.delete_multi(

                self.garbage_inventories, self.pb)

            for entry in checkout_files:

                self.move_entry('checkout', entry)

            if last_revision is not None:

                self.bzrdir.transport.put_bytes(

                    'checkout/last-revision', last_revision)

        self.bzrdir.transport.put_bytes(

            'branch-format',

            BzrDirMetaFormat1().get_format_string(),

            mode=self.file_mode)

        self.pb.finished()

        return ControlDir.open(self.bzrdir.user_url)

    def make_lock(self, name):

        """Make a lock for the new control dir name."""

        self.step(gettext('Make %s lock') % name)

        ld = lockdir.LockDir(self.bzrdir.transport,

                             '%s/lock' % name,

                             file_modebits=self.file_mode,

                             dir_modebits=self.dir_mode)

        ld.create()

    def move_entry(self, new_dir, entry):

        """Move then entry name into new_dir."""

        name = entry[0]

        mandatory = entry[1]

        self.step(gettext('Moving %s') % name)

        try:

            self.bzrdir.transport.move(name, '%s/%s' % (new_dir, name))

        except errors.NoSuchFile:

            if mandatory:

                raise

    def put_format(self, dirname, format):

        self.bzrdir.transport.put_bytes('%s/format' % dirname,

            format.get_format_string(),

            self.file_mode)

class BzrDirFormat4(BzrDirFormat):

    """Bzr dir format 4.

    This format is a combined format for working tree, branch and repository.

    It has:

     - Format 1 working trees [always]

     - Format 4 branches [always]

     - Format 4 repositories [always]

    This format is deprecated: it indexes texts using a text it which is

    removed in format 5; write support for this format has been removed.

    """

    _lock_class = lockable_files.TransportLock

    def __eq__(self, other):

        return type(self) == type(other)

    @classmethod

    def get_format_string(cls):

        """See BzrDirFormat.get_format_string()."""

        return "Bazaar-NG branch, format 0.0.4\n"

    def get_format_description(self):

        """See ControlDirFormat.get_format_description()."""