~bzr-pqm/bzr/bzr.dev

# Copyright (C) 2005, 2006 Canonical Ltd

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

from binascii import hexlify

from copy import deepcopy

from cStringIO import StringIO

import re

import time

from unittest import TestSuite

from warnings import warn

from bzrlib import bzrdir, check, delta, gpg, errors, xml5, ui, transactions, osutils

from bzrlib.decorators import needs_read_lock, needs_write_lock

from bzrlib.errors import InvalidRevisionId

from bzrlib.graph import Graph

from bzrlib.inter import InterObject

from bzrlib.inventory import Inventory, InventoryDirectory, ROOT_ID

from bzrlib.knit import KnitVersionedFile, KnitPlainFactory

from bzrlib.lockable_files import LockableFiles, TransportLock

from bzrlib.lockdir import LockDir

from bzrlib.osutils import (safe_unicode, rand_bytes, compact_date, 

                            local_time_offset)

from bzrlib.revision import NULL_REVISION, Revision

from bzrlib.revisiontree import RevisionTree

from bzrlib.store.versioned import VersionedFileStore, WeaveStore

from bzrlib.store.text import TextStore

from bzrlib.symbol_versioning import (deprecated_method,

        zero_nine, 

        )

from bzrlib.testament import Testament

from bzrlib.trace import mutter, note, warning

from bzrlib.tsort import topo_sort

from bzrlib.weave import WeaveFile

# Old formats display a warning, but only once

_deprecation_warning_done = False

class Repository(object):

    """Repository holding history for one or more branches.

    The repository holds and retrieves historical information including

    revisions and file history.  It's normally accessed only by the Branch,

    which views a particular line of development through that history.

    The Repository builds on top of Stores and a Transport, which respectively 

    describe the disk data format and the way of accessing the (possibly 

    remote) disk.

    """

    @needs_write_lock

    def add_inventory(self, revid, inv, parents):

        """Add the inventory inv to the repository as revid.

        :param parents: The revision ids of the parents that revid

                        is known to have and are in the repository already.

        returns the sha1 of the serialized inventory.

        """

        assert inv.revision_id is None or inv.revision_id == revid, \

            "Mismatch between inventory revision" \

            " id and insertion revid (%r, %r)" % (inv.revision_id, revid)

        assert inv.root is not None

        inv_text = xml5.serializer_v5.write_inventory_to_string(inv)

        inv_sha1 = osutils.sha_string(inv_text)

        inv_vf = self.control_weaves.get_weave('inventory',

                                               self.get_transaction())

        self._inventory_add_lines(inv_vf, revid, parents, osutils.split_lines(inv_text))

        return inv_sha1

    def _inventory_add_lines(self, inv_vf, revid, parents, lines):

        final_parents = []

        for parent in parents:

            if parent in inv_vf:

                final_parents.append(parent)

        inv_vf.add_lines(revid, final_parents, lines)

    @needs_write_lock

    def add_revision(self, rev_id, rev, inv=None, config=None):

        """Add rev to the revision store as rev_id.

        :param rev_id: the revision id to use.

        :param rev: The revision object.

        :param inv: The inventory for the revision. if None, it will be looked

                    up in the inventory storer

        :param config: If None no digital signature will be created.

                       If supplied its signature_needed method will be used

                       to determine if a signature should be made.

        """

        if config is not None and config.signature_needed():

            if inv is None:

                inv = self.get_inventory(rev_id)

            plaintext = Testament(rev, inv).as_short_text()

            self.store_revision_signature(

                gpg.GPGStrategy(config), plaintext, rev_id)

        if not rev_id in self.get_inventory_weave():

            if inv is None:

                raise errors.WeaveRevisionNotPresent(rev_id,

                                                     self.get_inventory_weave())

            else:

                # yes, this is not suitable for adding with ghosts.

                self.add_inventory(rev_id, inv, rev.parent_ids)

        self._revision_store.add_revision(rev, self.get_transaction())

    @needs_read_lock

    def _all_possible_ids(self):

        """Return all the possible revisions that we could find."""

        return self.get_inventory_weave().versions()

    def all_revision_ids(self):

        """Returns a list of all the revision ids in the repository. 

        This is deprecated because code should generally work on the graph

        reachable from a particular revision, and ignore any other revisions

        that might be present.  There is no direct replacement method.

        """

        return self._all_revision_ids()

    @needs_read_lock

    def _all_revision_ids(self):

        """Returns a list of all the revision ids in the repository. 

        These are in as much topological order as the underlying store can 

        present: for weaves ghosts may lead to a lack of correctness until

        the reweave updates the parents list.

        """

        if self._revision_store.text_store.listable():

            return self._revision_store.all_revision_ids(self.get_transaction())

        result = self._all_possible_ids()

        return self._eliminate_revisions_not_present(result)

    def break_lock(self):

        """Break a lock if one is present from another instance.

        Uses the ui factory to ask for confirmation if the lock may be from

        an active process.

        """

        self.control_files.break_lock()

    @needs_read_lock

    def _eliminate_revisions_not_present(self, revision_ids):

        """Check every revision id in revision_ids to see if we have it.

        Returns a set of the present revisions.

        """

        result = []

        for id in revision_ids:

            if self.has_revision(id):

               result.append(id)

        return result

    @staticmethod

    def create(a_bzrdir):

        """Construct the current default format repository in a_bzrdir."""

        return RepositoryFormat.get_default_format().initialize(a_bzrdir)

    def __init__(self, _format, a_bzrdir, control_files, _revision_store, control_store, text_store):

        """instantiate a Repository.

        :param _format: The format of the repository on disk.

        :param a_bzrdir: The BzrDir of the repository.

        In the future we will have a single api for all stores for

        getting file texts, inventories and revisions, then

        this construct will accept instances of those things.

        """

        super(Repository, self).__init__()

        self._format = _format

        # the following are part of the public API for Repository:

        self.bzrdir = a_bzrdir

        self.control_files = control_files

        self._revision_store = _revision_store

        self.text_store = text_store

        # backwards compatibility

        self.weave_store = text_store

        # not right yet - should be more semantically clear ? 

        # 

        self.control_store = control_store

        self.control_weaves = control_store

        # TODO: make sure to construct the right store classes, etc, depending

        # on whether escaping is required.

        self._warn_if_deprecated()

    def __repr__(self):

        return '%s(%r)' % (self.__class__.__name__, 

                           self.bzrdir.transport.base)

    def is_locked(self):

        return self.control_files.is_locked()

    def lock_write(self):

        self.control_files.lock_write()

    def lock_read(self):

        self.control_files.lock_read()

    def get_physical_lock_status(self):

        return self.control_files.get_physical_lock_status()

    @needs_read_lock

    def missing_revision_ids(self, other, revision_id=None):

        """Return the revision ids that other has that this does not.

        These are returned in topological order.

        revision_id: only return revision ids included by revision_id.

        """

        return InterRepository.get(other, self).missing_revision_ids(revision_id)

    @staticmethod

    def open(base):

        """Open the repository rooted at base.

        For instance, if the repository is at URL/.bzr/repository,

        Repository.open(URL) -> a Repository instance.

        """

        control = bzrdir.BzrDir.open(base)

        return control.open_repository()

    def copy_content_into(self, destination, revision_id=None, basis=None):

        """Make a complete copy of the content in self into destination.

        This is a destructive operation! Do not use it on existing 

        repositories.

        """

        return InterRepository.get(self, destination).copy_content(revision_id, basis)

    def fetch(self, source, revision_id=None, pb=None):

        """Fetch the content required to construct revision_id from source.

        If revision_id is None all content is copied.

        """

        return InterRepository.get(source, self).fetch(revision_id=revision_id,

                                                       pb=pb)

    def get_commit_builder(self, branch, parents, config, timestamp=None, 

                           timezone=None, committer=None, revprops=None, 

                           revision_id=None):

        """Obtain a CommitBuilder for this repository.

        :param branch: Branch to commit to.

        :param parents: Revision ids of the parents of the new revision.

        :param config: Configuration to use.

        :param timestamp: Optional timestamp recorded for commit.

        :param timezone: Optional timezone for timestamp.

        :param committer: Optional committer to set for commit.

        :param revprops: Optional dictionary of revision properties.

        :param revision_id: Optional revision id.

        """

        return _CommitBuilder(self, parents, config, timestamp, timezone,

                              committer, revprops, revision_id)

    def unlock(self):

        self.control_files.unlock()

    @needs_read_lock

    def clone(self, a_bzrdir, revision_id=None, basis=None):

        """Clone this repository into a_bzrdir using the current format.

        Currently no check is made that the format of this repository and

        the bzrdir format are compatible. FIXME RBC 20060201.

        """

        if not isinstance(a_bzrdir._format, self.bzrdir._format.__class__):

            # use target default format.

            result = a_bzrdir.create_repository()

        # FIXME RBC 20060209 split out the repository type to avoid this check ?

        elif isinstance(a_bzrdir._format,

                      (bzrdir.BzrDirFormat4,

                       bzrdir.BzrDirFormat5,

                       bzrdir.BzrDirFormat6)):

            result = a_bzrdir.open_repository()

        else:

            result = self._format.initialize(a_bzrdir, shared=self.is_shared())

        self.copy_content_into(result, revision_id, basis)

        return result

    @needs_read_lock

    def has_revision(self, revision_id):

        """True if this repository has a copy of the revision."""

        return self._revision_store.has_revision_id(revision_id,

                                                    self.get_transaction())

    @needs_read_lock

    def get_revision_reconcile(self, revision_id):

        """'reconcile' helper routine that allows access to a revision always.

        This variant of get_revision does not cross check the weave graph

        against the revision one as get_revision does: but it should only

        be used by reconcile, or reconcile-alike commands that are correcting

        or testing the revision graph.

        """

        if not revision_id or not isinstance(revision_id, basestring):

            raise InvalidRevisionId(revision_id=revision_id, branch=self)

        return self._revision_store.get_revisions([revision_id],

                                                  self.get_transaction())[0]

    @needs_read_lock

    def get_revisions(self, revision_ids):

        return self._revision_store.get_revisions(revision_ids,

                                                  self.get_transaction())

    @needs_read_lock

    def get_revision_xml(self, revision_id):

        rev = self.get_revision(revision_id) 

        rev_tmp = StringIO()

        # the current serializer..

        self._revision_store._serializer.write_revision(rev, rev_tmp)

        rev_tmp.seek(0)

        return rev_tmp.getvalue()

    @needs_read_lock

    def get_revision(self, revision_id):

        """Return the Revision object for a named revision"""

        r = self.get_revision_reconcile(revision_id)

        # weave corruption can lead to absent revision markers that should be

        # present.

        # the following test is reasonably cheap (it needs a single weave read)

        # and the weave is cached in read transactions. In write transactions

        # it is not cached but typically we only read a small number of

        # revisions. For knits when they are introduced we will probably want

        # to ensure that caching write transactions are in use.

        inv = self.get_inventory_weave()

        self._check_revision_parents(r, inv)

        return r

    @needs_read_lock

    def get_deltas_for_revisions(self, revisions):

        """Produce a generator of revision deltas.

        Note that the input is a sequence of REVISIONS, not revision_ids.

        Trees will be held in memory until the generator exits.

        Each delta is relative to the revision's lefthand predecessor.

        """

        required_trees = set()

        for revision in revisions:

            required_trees.add(revision.revision_id)

            required_trees.update(revision.parent_ids[:1])

        trees = dict((t.get_revision_id(), t) for 

                     t in self.revision_trees(required_trees))

        for revision in revisions:

            if not revision.parent_ids:

                old_tree = self.revision_tree(None)

            else:

                old_tree = trees[revision.parent_ids[0]]

            yield trees[revision.revision_id].changes_from(old_tree)

    @needs_read_lock

    def get_revision_delta(self, revision_id):

        """Return the delta for one revision.

        The delta is relative to the left-hand predecessor of the

        revision.

        """

        r = self.get_revision(revision_id)

        return list(self.get_deltas_for_revisions([r]))[0]

    def _check_revision_parents(self, revision, inventory):

        """Private to Repository and Fetch.

        This checks the parentage of revision in an inventory weave for 

        consistency and is only applicable to inventory-weave-for-ancestry

        using repository formats & fetchers.

        """

        weave_parents = inventory.get_parents(revision.revision_id)

        weave_names = inventory.versions()

        for parent_id in revision.parent_ids:

            if parent_id in weave_names:

                # this parent must not be a ghost.

                if not parent_id in weave_parents:

                    # but it is a ghost

                    raise errors.CorruptRepository(self)

    @needs_write_lock

    def store_revision_signature(self, gpg_strategy, plaintext, revision_id):

        signature = gpg_strategy.sign(plaintext)

        self._revision_store.add_revision_signature_text(revision_id,

                                                         signature,

                                                         self.get_transaction())

    def fileids_altered_by_revision_ids(self, revision_ids):

        """Find the file ids and versions affected by revisions.

        :param revisions: an iterable containing revision ids.

        :return: a dictionary mapping altered file-ids to an iterable of

        revision_ids. Each altered file-ids has the exact revision_ids that

        altered it listed explicitly.

        """

        assert isinstance(self._format, (RepositoryFormat5,

                                         RepositoryFormat6,

                                         RepositoryFormat7,

                                         RepositoryFormatKnit1)), \

            ("fileids_altered_by_revision_ids only supported for branches " 

             "which store inventory as unnested xml, not on %r" % self)

        selected_revision_ids = set(revision_ids)

        w = self.get_inventory_weave()

        result = {}

        # this code needs to read every new line in every inventory for the

        # inventories [revision_ids]. Seeing a line twice is ok. Seeing a line

        # not present in one of those inventories is unnecessary but not 

        # harmful because we are filtering by the revision id marker in the

        # inventory lines : we only select file ids altered in one of those  

        # revisions. We don't need to see all lines in the inventory because

        # only those added in an inventory in rev X can contain a revision=X

        # line.

        for line in w.iter_lines_added_or_present_in_versions(selected_revision_ids):

            start = line.find('file_id="')+9

            if start < 9: continue

            end = line.find('"', start)

            assert end>= 0

            file_id = _unescape_xml(line[start:end])

            start = line.find('revision="')+10

            if start < 10: continue

            end = line.find('"', start)

            assert end>= 0

            revision_id = _unescape_xml(line[start:end])

            if revision_id in selected_revision_ids:

                result.setdefault(file_id, set()).add(revision_id)

        return result

    @needs_read_lock

    def get_inventory_weave(self):

        return self.control_weaves.get_weave('inventory',

            self.get_transaction())

    @needs_read_lock

    def get_inventory(self, revision_id):

        """Get Inventory object by hash."""

        return self.deserialise_inventory(

            revision_id, self.get_inventory_xml(revision_id))

    def deserialise_inventory(self, revision_id, xml):

        """Transform the xml into an inventory object. 

        :param revision_id: The expected revision id of the inventory.

        :param xml: A serialised inventory.

        """

        result = xml5.serializer_v5.read_inventory_from_string(xml)

        result.root.revision = revision_id

        return result

    @needs_read_lock

    def get_inventory_xml(self, revision_id):

        """Get inventory XML as a file object."""

        try:

            assert isinstance(revision_id, basestring), type(revision_id)

            iw = self.get_inventory_weave()

            return iw.get_text(revision_id)

        except IndexError:

            raise errors.HistoryMissing(self, 'inventory', revision_id)

    @needs_read_lock

    def get_inventory_sha1(self, revision_id):

        """Return the sha1 hash of the inventory entry

        """

        return self.get_revision(revision_id).inventory_sha1

    @needs_read_lock

    def get_revision_graph(self, revision_id=None):

        """Return a dictionary containing the revision graph.

        :param revision_id: The revision_id to get a graph from. If None, then

        the entire revision graph is returned. This is a deprecated mode of

        operation and will be removed in the future.

        :return: a dictionary of revision_id->revision_parents_list.

        """

        # special case NULL_REVISION

        if revision_id == NULL_REVISION:

            return {}

        weave = self.get_inventory_weave()

        all_revisions = self._eliminate_revisions_not_present(weave.versions())

        entire_graph = dict([(node, weave.get_parents(node)) for 

                             node in all_revisions])

        if revision_id is None:

            return entire_graph

        elif revision_id not in entire_graph:

            raise errors.NoSuchRevision(self, revision_id)

        else:

            # add what can be reached from revision_id

            result = {}

            pending = set([revision_id])

            while len(pending) > 0:

                node = pending.pop()

                result[node] = entire_graph[node]

                for revision_id in result[node]:

                    if revision_id not in result:

                        pending.add(revision_id)

            return result

    @needs_read_lock

    def get_revision_graph_with_ghosts(self, revision_ids=None):

        """Return a graph of the revisions with ghosts marked as applicable.

        :param revision_ids: an iterable of revisions to graph or None for all.

        :return: a Graph object with the graph reachable from revision_ids.

        """

        result = Graph()

        if not revision_ids:

            pending = set(self.all_revision_ids())

            required = set([])

        else:

            pending = set(revision_ids)

            # special case NULL_REVISION

            if NULL_REVISION in pending:

                pending.remove(NULL_REVISION)

            required = set(pending)

        done = set([])

        while len(pending):

            revision_id = pending.pop()

            try:

                rev = self.get_revision(revision_id)

            except errors.NoSuchRevision:

                if revision_id in required:

                    raise

                # a ghost

                result.add_ghost(revision_id)

                continue

            for parent_id in rev.parent_ids:

                # is this queued or done ?

                if (parent_id not in pending and

                    parent_id not in done):

                    # no, queue it.

                    pending.add(parent_id)

            result.add_node(revision_id, rev.parent_ids)

            done.add(revision_id)

        return result

    @needs_read_lock

    def get_revision_inventory(self, revision_id):

        """Return inventory of a past revision."""

        # TODO: Unify this with get_inventory()

        # bzr 0.0.6 and later imposes the constraint that the inventory_id

        # must be the same as its revision, so this is trivial.

        if revision_id is None:

            # This does not make sense: if there is no revision,

            # then it is the current tree inventory surely ?!

            # and thus get_root_id() is something that looks at the last

            # commit on the branch, and the get_root_id is an inventory check.

            raise NotImplementedError

            # return Inventory(self.get_root_id())

        else:

            return self.get_inventory(revision_id)

    @needs_read_lock

    def is_shared(self):

        """Return True if this repository is flagged as a shared repository."""

        raise NotImplementedError(self.is_shared)

    @needs_write_lock

    def reconcile(self, other=None, thorough=False):

        """Reconcile this repository."""

        from bzrlib.reconcile import RepoReconciler

        reconciler = RepoReconciler(self, thorough=thorough)

        reconciler.reconcile()

        return reconciler

    @needs_read_lock

    def revision_tree(self, revision_id):

        """Return Tree for a revision on this branch.

        `revision_id` may be None for the empty tree revision.

        """

        # TODO: refactor this to use an existing revision object

        # so we don't need to read it in twice.

        if revision_id is None or revision_id == NULL_REVISION:

            return RevisionTree(self, Inventory(), NULL_REVISION)

        else:

            inv = self.get_revision_inventory(revision_id)

            return RevisionTree(self, inv, revision_id)

    @needs_read_lock

    def revision_trees(self, revision_ids):

        """Return Tree for a revision on this branch.

        `revision_id` may not be None or 'null:'"""

        assert None not in revision_ids

        assert NULL_REVISION not in revision_ids

        texts = self.get_inventory_weave().get_texts(revision_ids)

        for text, revision_id in zip(texts, revision_ids):

            inv = self.deserialise_inventory(revision_id, text)

            yield RevisionTree(self, inv, revision_id)

    @needs_read_lock

    def get_ancestry(self, revision_id):

        """Return a list of revision-ids integrated by a revision.

        The first element of the list is always None, indicating the origin 

        revision.  This might change when we have history horizons, or 

        perhaps we should have a new API.

        This is topologically sorted.

        """

        if revision_id is None:

            return [None]

        if not self.has_revision(revision_id):

            raise errors.NoSuchRevision(self, revision_id)

        w = self.get_inventory_weave()

        candidates = w.get_ancestry(revision_id)

        return [None] + candidates # self._eliminate_revisions_not_present(candidates)

    @needs_read_lock

    def print_file(self, file, revision_id):

        """Print `file` to stdout.

        FIXME RBC 20060125 as John Meinel points out this is a bad api

        - it writes to stdout, it assumes that that is valid etc. Fix

        by creating a new more flexible convenience function.

        """

        tree = self.revision_tree(revision_id)

        # use inventory as it was in that revision

        file_id = tree.inventory.path2id(file)

        if not file_id:

            # TODO: jam 20060427 Write a test for this code path

            #       it had a bug in it, and was raising the wrong

            #       exception.

            raise errors.BzrError("%r is not present in revision %s" % (file, revision_id))

        tree.print_file(file_id)

    def get_transaction(self):

        return self.control_files.get_transaction()

    def revision_parents(self, revid):

        return self.get_inventory_weave().parent_names(revid)

    @needs_write_lock

    def set_make_working_trees(self, new_value):

        """Set the policy flag for making working trees when creating branches.

        This only applies to branches that use this repository.

        The default is 'True'.

        :param new_value: True to restore the default, False to disable making

                          working trees.

        """

        raise NotImplementedError(self.set_make_working_trees)

    def make_working_trees(self):

        """Returns the policy for making working trees on new branches."""

        raise NotImplementedError(self.make_working_trees)

    @needs_write_lock

    def sign_revision(self, revision_id, gpg_strategy):

        plaintext = Testament.from_revision(self, revision_id).as_short_text()

        self.store_revision_signature(gpg_strategy, plaintext, revision_id)

    @needs_read_lock

    def has_signature_for_revision_id(self, revision_id):

        """Query for a revision signature for revision_id in the repository."""

        return self._revision_store.has_signature(revision_id,

                                                  self.get_transaction())

    @needs_read_lock

    def get_signature_text(self, revision_id):

        """Return the text for a signature."""

        return self._revision_store.get_signature_text(revision_id,

                                                       self.get_transaction())

    @needs_read_lock

    def check(self, revision_ids):

        """Check consistency of all history of given revision_ids.

        Different repository implementations should override _check().

        :param revision_ids: A non-empty list of revision_ids whose ancestry

             will be checked.  Typically the last revision_id of a branch.

        """

        if not revision_ids:

            raise ValueError("revision_ids must be non-empty in %s.check" 

                    % (self,))

        return self._check(revision_ids)

    def _check(self, revision_ids):

        result = check.Check(self)

        result.check()

        return result