~bzr-pqm/bzr/bzr.dev

# Copyright (C) 2005, 2006, 2007 Canonical Ltd

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

from cStringIO import StringIO

from bzrlib.lazy_import import lazy_import

lazy_import(globals(), """

import re

import time

import unittest

from bzrlib import (

    bzrdir,

    check,

    errors,

    generate_ids,

    gpg,

    graph,

    lazy_regex,

    lockable_files,

    lockdir,

    osutils,

    registry,

    revision as _mod_revision,

    symbol_versioning,

    transactions,

    ui,

    )

from bzrlib.revisiontree import RevisionTree

from bzrlib.store.versioned import VersionedFileStore

from bzrlib.store.text import TextStore

from bzrlib.testament import Testament

""")

from bzrlib.decorators import needs_read_lock, needs_write_lock

from bzrlib.inter import InterObject

from bzrlib.inventory import Inventory, InventoryDirectory, ROOT_ID

from bzrlib.symbol_versioning import (

        deprecated_method,

        zero_nine,

        )

from bzrlib.trace import mutter, note, warning

# Old formats display a warning, but only once

_deprecation_warning_done = False

######################################################################

# Repositories

class Repository(object):

    """Repository holding history for one or more branches.

    The repository holds and retrieves historical information including

    revisions and file history.  It's normally accessed only by the Branch,

    which views a particular line of development through that history.

    The Repository builds on top of Stores and a Transport, which respectively 

    describe the disk data format and the way of accessing the (possibly 

    remote) disk.

    """

    _file_ids_altered_regex = lazy_regex.lazy_compile(

        r'file_id="(?P<file_id>[^"]+)"'

        r'.*revision="(?P<revision_id>[^"]+)"'

        )

    @needs_write_lock

    def add_inventory(self, revision_id, inv, parents):

        """Add the inventory inv to the repository as revision_id.

        :param parents: The revision ids of the parents that revision_id

                        is known to have and are in the repository already.

        returns the sha1 of the serialized inventory.

        """

        revision_id = osutils.safe_revision_id(revision_id)

        _mod_revision.check_not_reserved_id(revision_id)

        assert inv.revision_id is None or inv.revision_id == revision_id, \

            "Mismatch between inventory revision" \

            " id and insertion revid (%r, %r)" % (inv.revision_id, revision_id)

        assert inv.root is not None

        inv_text = self.serialise_inventory(inv)

        inv_sha1 = osutils.sha_string(inv_text)

        inv_vf = self.control_weaves.get_weave('inventory',

                                               self.get_transaction())

        self._inventory_add_lines(inv_vf, revision_id, parents,

                                  osutils.split_lines(inv_text))

        return inv_sha1

    def _inventory_add_lines(self, inv_vf, revision_id, parents, lines):

        final_parents = []

        for parent in parents:

            if parent in inv_vf:

                final_parents.append(parent)

        inv_vf.add_lines(revision_id, final_parents, lines)

    @needs_write_lock

    def add_revision(self, revision_id, rev, inv=None, config=None):

        """Add rev to the revision store as revision_id.

        :param revision_id: the revision id to use.

        :param rev: The revision object.

        :param inv: The inventory for the revision. if None, it will be looked

                    up in the inventory storer

        :param config: If None no digital signature will be created.

                       If supplied its signature_needed method will be used

                       to determine if a signature should be made.

        """

        revision_id = osutils.safe_revision_id(revision_id)

        # TODO: jam 20070210 Shouldn't we check rev.revision_id and

        #       rev.parent_ids?

        _mod_revision.check_not_reserved_id(revision_id)

        if config is not None and config.signature_needed():

            if inv is None:

                inv = self.get_inventory(revision_id)

            plaintext = Testament(rev, inv).as_short_text()

            self.store_revision_signature(

                gpg.GPGStrategy(config), plaintext, revision_id)

        if not revision_id in self.get_inventory_weave():

            if inv is None:

                raise errors.WeaveRevisionNotPresent(revision_id,

                                                     self.get_inventory_weave())

            else:

                # yes, this is not suitable for adding with ghosts.

                self.add_inventory(revision_id, inv, rev.parent_ids)

        self._revision_store.add_revision(rev, self.get_transaction())

    @needs_read_lock

    def _all_possible_ids(self):

        """Return all the possible revisions that we could find."""

        return self.get_inventory_weave().versions()

    def all_revision_ids(self):

        """Returns a list of all the revision ids in the repository. 

        This is deprecated because code should generally work on the graph

        reachable from a particular revision, and ignore any other revisions

        that might be present.  There is no direct replacement method.

        """

        return self._all_revision_ids()

    @needs_read_lock

    def _all_revision_ids(self):

        """Returns a list of all the revision ids in the repository. 

        These are in as much topological order as the underlying store can 

        present: for weaves ghosts may lead to a lack of correctness until

        the reweave updates the parents list.

        """

        if self._revision_store.text_store.listable():

            return self._revision_store.all_revision_ids(self.get_transaction())

        result = self._all_possible_ids()

        # TODO: jam 20070210 Ensure that _all_possible_ids returns non-unicode

        #       ids. (It should, since _revision_store's API should change to

        #       return utf8 revision_ids)

        return self._eliminate_revisions_not_present(result)

    def break_lock(self):

        """Break a lock if one is present from another instance.

        Uses the ui factory to ask for confirmation if the lock may be from

        an active process.

        """

        self.control_files.break_lock()

    @needs_read_lock

    def _eliminate_revisions_not_present(self, revision_ids):

        """Check every revision id in revision_ids to see if we have it.

        Returns a set of the present revisions.

        """

        result = []

        for id in revision_ids:

            if self.has_revision(id):

               result.append(id)

        return result

    @staticmethod

    def create(a_bzrdir):

        """Construct the current default format repository in a_bzrdir."""

        return RepositoryFormat.get_default_format().initialize(a_bzrdir)

    def __init__(self, _format, a_bzrdir, control_files, _revision_store, control_store, text_store):

        """instantiate a Repository.

        :param _format: The format of the repository on disk.

        :param a_bzrdir: The BzrDir of the repository.

        In the future we will have a single api for all stores for

        getting file texts, inventories and revisions, then

        this construct will accept instances of those things.

        """

        super(Repository, self).__init__()

        self._format = _format

        # the following are part of the public API for Repository:

        self.bzrdir = a_bzrdir

        self.control_files = control_files

        self._revision_store = _revision_store

        self.text_store = text_store

        # backwards compatibility

        self.weave_store = text_store

        # not right yet - should be more semantically clear ? 

        # 

        self.control_store = control_store

        self.control_weaves = control_store

        # TODO: make sure to construct the right store classes, etc, depending

        # on whether escaping is required.

        self._warn_if_deprecated()

    def __repr__(self):

        return '%s(%r)' % (self.__class__.__name__, 

                           self.bzrdir.transport.base)

    def is_locked(self):

        return self.control_files.is_locked()

    def lock_write(self):

        self.control_files.lock_write()

    def lock_read(self):

        self.control_files.lock_read()

    def get_physical_lock_status(self):

        return self.control_files.get_physical_lock_status()

    @needs_read_lock

    def gather_stats(self, revid=None, committers=None):

        """Gather statistics from a revision id.

        :param revid: The revision id to gather statistics from, if None, then

            no revision specific statistics are gathered.

        :param committers: Optional parameter controlling whether to grab

            a count of committers from the revision specific statistics.

        :return: A dictionary of statistics. Currently this contains:

            committers: The number of committers if requested.

            firstrev: A tuple with timestamp, timezone for the penultimate left

                most ancestor of revid, if revid is not the NULL_REVISION.

            latestrev: A tuple with timestamp, timezone for revid, if revid is

                not the NULL_REVISION.

            revisions: The total revision count in the repository.

            size: An estimate disk size of the repository in bytes.

        """

        result = {}

        if revid and committers:

            result['committers'] = 0

        if revid and revid != _mod_revision.NULL_REVISION:

            if committers:

                all_committers = set()

            revisions = self.get_ancestry(revid)

            # pop the leading None

            revisions.pop(0)

            first_revision = None

            if not committers:

                # ignore the revisions in the middle - just grab first and last

                revisions = revisions[0], revisions[-1]

            for revision in self.get_revisions(revisions):

                if not first_revision:

                    first_revision = revision

                if committers:

                    all_committers.add(revision.committer)

            last_revision = revision

            if committers:

                result['committers'] = len(all_committers)

            result['firstrev'] = (first_revision.timestamp,

                first_revision.timezone)

            result['latestrev'] = (last_revision.timestamp,

                last_revision.timezone)

        # now gather global repository information

        if self.bzrdir.root_transport.listable():

            c, t = self._revision_store.total_size(self.get_transaction())

            result['revisions'] = c

            result['size'] = t

        return result

    @needs_read_lock

    def missing_revision_ids(self, other, revision_id=None):

        """Return the revision ids that other has that this does not.

        These are returned in topological order.

        revision_id: only return revision ids included by revision_id.

        """

        revision_id = osutils.safe_revision_id(revision_id)

        return InterRepository.get(other, self).missing_revision_ids(revision_id)

    @staticmethod

    def open(base):

        """Open the repository rooted at base.

        For instance, if the repository is at URL/.bzr/repository,

        Repository.open(URL) -> a Repository instance.

        """

        control = bzrdir.BzrDir.open(base)

        return control.open_repository()

    def copy_content_into(self, destination, revision_id=None, basis=None):

        """Make a complete copy of the content in self into destination.

        This is a destructive operation! Do not use it on existing 

        repositories.

        """

        revision_id = osutils.safe_revision_id(revision_id)

        return InterRepository.get(self, destination).copy_content(revision_id, basis)

    def fetch(self, source, revision_id=None, pb=None):

        """Fetch the content required to construct revision_id from source.

        If revision_id is None all content is copied.

        """

        revision_id = osutils.safe_revision_id(revision_id)

        return InterRepository.get(source, self).fetch(revision_id=revision_id,

                                                       pb=pb)

    def get_commit_builder(self, branch, parents, config, timestamp=None, 

                           timezone=None, committer=None, revprops=None, 

                           revision_id=None):

        """Obtain a CommitBuilder for this repository.

        :param branch: Branch to commit to.

        :param parents: Revision ids of the parents of the new revision.

        :param config: Configuration to use.

        :param timestamp: Optional timestamp recorded for commit.

        :param timezone: Optional timezone for timestamp.

        :param committer: Optional committer to set for commit.

        :param revprops: Optional dictionary of revision properties.

        :param revision_id: Optional revision id.

        """

        revision_id = osutils.safe_revision_id(revision_id)

        return _CommitBuilder(self, parents, config, timestamp, timezone,

                              committer, revprops, revision_id)

    def unlock(self):

        self.control_files.unlock()

    @needs_read_lock

    def clone(self, a_bzrdir, revision_id=None, basis=None):

        """Clone this repository into a_bzrdir using the current format.

        Currently no check is made that the format of this repository and

        the bzrdir format are compatible. FIXME RBC 20060201.

        :return: The newly created destination repository.

        """

        if not isinstance(a_bzrdir._format, self.bzrdir._format.__class__):

            # use target default format.

            dest_repo = a_bzrdir.create_repository()

        else:

            # Most control formats need the repository to be specifically

            # created, but on some old all-in-one formats it's not needed

            try:

                dest_repo = self._format.initialize(a_bzrdir, shared=self.is_shared())

            except errors.UninitializableFormat:

                dest_repo = a_bzrdir.open_repository()

        self.copy_content_into(dest_repo, revision_id, basis)

        return dest_repo

    @needs_read_lock

    def has_revision(self, revision_id):

        """True if this repository has a copy of the revision."""

        revision_id = osutils.safe_revision_id(revision_id)

        return self._revision_store.has_revision_id(revision_id,

                                                    self.get_transaction())

    @needs_read_lock

    def get_revision_reconcile(self, revision_id):

        """'reconcile' helper routine that allows access to a revision always.

        This variant of get_revision does not cross check the weave graph

        against the revision one as get_revision does: but it should only

        be used by reconcile, or reconcile-alike commands that are correcting

        or testing the revision graph.

        """

        if not revision_id or not isinstance(revision_id, basestring):

            raise errors.InvalidRevisionId(revision_id=revision_id,

                                           branch=self)

        return self.get_revisions([revision_id])[0]

    @needs_read_lock

    def get_revisions(self, revision_ids):

        revision_ids = [osutils.safe_revision_id(r) for r in revision_ids]

        revs = self._revision_store.get_revisions(revision_ids,

                                                  self.get_transaction())

        for rev in revs:

            assert not isinstance(rev.revision_id, unicode)

            for parent_id in rev.parent_ids:

                assert not isinstance(parent_id, unicode)

        return revs

    @needs_read_lock

    def get_revision_xml(self, revision_id):

        # TODO: jam 20070210 This shouldn't be necessary since get_revision

        #       would have already do it.

        # TODO: jam 20070210 Just use _serializer.write_revision_to_string()

        revision_id = osutils.safe_revision_id(revision_id)

        rev = self.get_revision(revision_id)

        rev_tmp = StringIO()

        # the current serializer..

        self._revision_store._serializer.write_revision(rev, rev_tmp)

        rev_tmp.seek(0)

        return rev_tmp.getvalue()

    @needs_read_lock

    def get_revision(self, revision_id):

        """Return the Revision object for a named revision"""

        # TODO: jam 20070210 get_revision_reconcile should do this for us

        revision_id = osutils.safe_revision_id(revision_id)

        r = self.get_revision_reconcile(revision_id)

        # weave corruption can lead to absent revision markers that should be

        # present.

        # the following test is reasonably cheap (it needs a single weave read)

        # and the weave is cached in read transactions. In write transactions

        # it is not cached but typically we only read a small number of

        # revisions. For knits when they are introduced we will probably want

        # to ensure that caching write transactions are in use.

        inv = self.get_inventory_weave()

        self._check_revision_parents(r, inv)

        return r

    @needs_read_lock

    def get_deltas_for_revisions(self, revisions):

        """Produce a generator of revision deltas.

        Note that the input is a sequence of REVISIONS, not revision_ids.

        Trees will be held in memory until the generator exits.

        Each delta is relative to the revision's lefthand predecessor.

        """

        required_trees = set()

        for revision in revisions:

            required_trees.add(revision.revision_id)

            required_trees.update(revision.parent_ids[:1])

        trees = dict((t.get_revision_id(), t) for 

                     t in self.revision_trees(required_trees))

        for revision in revisions:

            if not revision.parent_ids:

                old_tree = self.revision_tree(None)

            else:

                old_tree = trees[revision.parent_ids[0]]

            yield trees[revision.revision_id].changes_from(old_tree)

    @needs_read_lock

    def get_revision_delta(self, revision_id):

        """Return the delta for one revision.

        The delta is relative to the left-hand predecessor of the

        revision.

        """

        r = self.get_revision(revision_id)

        return list(self.get_deltas_for_revisions([r]))[0]

    def _check_revision_parents(self, revision, inventory):

        """Private to Repository and Fetch.

        This checks the parentage of revision in an inventory weave for 

        consistency and is only applicable to inventory-weave-for-ancestry

        using repository formats & fetchers.

        """

        weave_parents = inventory.get_parents(revision.revision_id)

        weave_names = inventory.versions()

        for parent_id in revision.parent_ids:

            if parent_id in weave_names:

                # this parent must not be a ghost.

                if not parent_id in weave_parents:

                    # but it is a ghost

                    raise errors.CorruptRepository(self)

    @needs_write_lock

    def store_revision_signature(self, gpg_strategy, plaintext, revision_id):

        revision_id = osutils.safe_revision_id(revision_id)

        signature = gpg_strategy.sign(plaintext)

        self._revision_store.add_revision_signature_text(revision_id,

                                                         signature,

                                                         self.get_transaction())

    def fileids_altered_by_revision_ids(self, revision_ids):

        """Find the file ids and versions affected by revisions.

        :param revisions: an iterable containing revision ids.

        :return: a dictionary mapping altered file-ids to an iterable of

        revision_ids. Each altered file-ids has the exact revision_ids that

        altered it listed explicitly.

        """

        assert self._serializer.support_altered_by_hack, \

            ("fileids_altered_by_revision_ids only supported for branches " 

             "which store inventory as unnested xml, not on %r" % self)

        selected_revision_ids = set(osutils.safe_revision_id(r)

                                    for r in revision_ids)

        w = self.get_inventory_weave()

        result = {}

        # this code needs to read every new line in every inventory for the

        # inventories [revision_ids]. Seeing a line twice is ok. Seeing a line

        # not present in one of those inventories is unnecessary but not 

        # harmful because we are filtering by the revision id marker in the

        # inventory lines : we only select file ids altered in one of those  

        # revisions. We don't need to see all lines in the inventory because

        # only those added in an inventory in rev X can contain a revision=X

        # line.

        unescape_revid_cache = {}

        unescape_fileid_cache = {}

        # jam 20061218 In a big fetch, this handles hundreds of thousands

        # of lines, so it has had a lot of inlining and optimizing done.

        # Sorry that it is a little bit messy.

        # Move several functions to be local variables, since this is a long

        # running loop.

        search = self._file_ids_altered_regex.search

        unescape = _unescape_xml

        setdefault = result.setdefault

        pb = ui.ui_factory.nested_progress_bar()

        try:

            for line in w.iter_lines_added_or_present_in_versions(

                                        selected_revision_ids, pb=pb):

                match = search(line)

                if match is None:

                    continue

                # One call to match.group() returning multiple items is quite a

                # bit faster than 2 calls to match.group() each returning 1

                file_id, revision_id = match.group('file_id', 'revision_id')

                # Inlining the cache lookups helps a lot when you make 170,000

                # lines and 350k ids, versus 8.4 unique ids.

                # Using a cache helps in 2 ways:

                #   1) Avoids unnecessary decoding calls

                #   2) Re-uses cached strings, which helps in future set and

                #      equality checks.

                # (2) is enough that removing encoding entirely along with

                # the cache (so we are using plain strings) results in no

                # performance improvement.

                try:

                    revision_id = unescape_revid_cache[revision_id]

                except KeyError:

                    unescaped = unescape(revision_id)

                    unescape_revid_cache[revision_id] = unescaped

                    revision_id = unescaped

                if revision_id in selected_revision_ids:

                    try:

                        file_id = unescape_fileid_cache[file_id]

                    except KeyError:

                        unescaped = unescape(file_id)

                        unescape_fileid_cache[file_id] = unescaped

                        file_id = unescaped

                    setdefault(file_id, set()).add(revision_id)

        finally:

            pb.finished()

        return result

    @needs_read_lock

    def get_inventory_weave(self):

        return self.control_weaves.get_weave('inventory',

            self.get_transaction())

    @needs_read_lock

    def get_inventory(self, revision_id):

        """Get Inventory object by hash."""

        # TODO: jam 20070210 Technically we don't need to sanitize, since all

        #       called functions must sanitize.

        revision_id = osutils.safe_revision_id(revision_id)

        return self.deserialise_inventory(

            revision_id, self.get_inventory_xml(revision_id))

    def deserialise_inventory(self, revision_id, xml):

        """Transform the xml into an inventory object. 

        :param revision_id: The expected revision id of the inventory.

        :param xml: A serialised inventory.

        """

        revision_id = osutils.safe_revision_id(revision_id)

        result = self._serializer.read_inventory_from_string(xml)

        result.root.revision = revision_id

        return result

    def serialise_inventory(self, inv):

        return self._serializer.write_inventory_to_string(inv)

    @needs_read_lock

    def get_inventory_xml(self, revision_id):

        """Get inventory XML as a file object."""

        revision_id = osutils.safe_revision_id(revision_id)

        try:

            assert isinstance(revision_id, str), type(revision_id)

            iw = self.get_inventory_weave()

            return iw.get_text(revision_id)

        except IndexError:

            raise errors.HistoryMissing(self, 'inventory', revision_id)

    @needs_read_lock

    def get_inventory_sha1(self, revision_id):

        """Return the sha1 hash of the inventory entry

        """

        # TODO: jam 20070210 Shouldn't this be deprecated / removed?

        revision_id = osutils.safe_revision_id(revision_id)

        return self.get_revision(revision_id).inventory_sha1

    @needs_read_lock

    def get_revision_graph(self, revision_id=None):

        """Return a dictionary containing the revision graph.

        :param revision_id: The revision_id to get a graph from. If None, then

        the entire revision graph is returned. This is a deprecated mode of

        operation and will be removed in the future.

        :return: a dictionary of revision_id->revision_parents_list.

        """

        # special case NULL_REVISION

        if revision_id == _mod_revision.NULL_REVISION:

            return {}

        revision_id = osutils.safe_revision_id(revision_id)

        a_weave = self.get_inventory_weave()

        all_revisions = self._eliminate_revisions_not_present(

                                a_weave.versions())

        entire_graph = dict([(node, a_weave.get_parents(node)) for 

                             node in all_revisions])

        if revision_id is None:

            return entire_graph

        elif revision_id not in entire_graph:

            raise errors.NoSuchRevision(self, revision_id)

        else:

            # add what can be reached from revision_id

            result = {}

            pending = set([revision_id])

            while len(pending) > 0:

                node = pending.pop()

                result[node] = entire_graph[node]

                for revision_id in result[node]:

                    if revision_id not in result:

                        pending.add(revision_id)

            return result

    @needs_read_lock

    def get_revision_graph_with_ghosts(self, revision_ids=None):

        """Return a graph of the revisions with ghosts marked as applicable.

        :param revision_ids: an iterable of revisions to graph or None for all.

        :return: a Graph object with the graph reachable from revision_ids.

        """

        result = graph.Graph()

        if not revision_ids:

            pending = set(self.all_revision_ids())

            required = set([])

        else:

            pending = set(osutils.safe_revision_id(r) for r in revision_ids)

            # special case NULL_REVISION

            if _mod_revision.NULL_REVISION in pending:

                pending.remove(_mod_revision.NULL_REVISION)

            required = set(pending)

        done = set([])

        while len(pending):

            revision_id = pending.pop()

            try:

                rev = self.get_revision(revision_id)

            except errors.NoSuchRevision:

                if revision_id in required:

                    raise

                # a ghost

                result.add_ghost(revision_id)

                continue

            for parent_id in rev.parent_ids:

                # is this queued or done ?

                if (parent_id not in pending and

                    parent_id not in done):

                    # no, queue it.

                    pending.add(parent_id)

            result.add_node(revision_id, rev.parent_ids)

            done.add(revision_id)