~bzr-pqm/bzr/bzr.dev

# Copyright (C) 2005, 2006 Canonical Ltd

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

from cStringIO import StringIO

from bzrlib.lazy_import import lazy_import

lazy_import(globals(), """

from binascii import hexlify

from copy import deepcopy

import re

import time

import unittest

from bzrlib import (

    bzrdir,

    check,

    delta,

    errors,

    generate_ids,

    gpg,

    graph,

    knit,

    lazy_regex,

    lockable_files,

    lockdir,

    osutils,

    revision as _mod_revision,

    symbol_versioning,

    transactions,

    ui,

    weave,

    weavefile,

    xml5,

    xml6,

    )

from bzrlib.osutils import (

    rand_bytes,

    compact_date, 

    local_time_offset,

    )

from bzrlib.revisiontree import RevisionTree

from bzrlib.store.versioned import VersionedFileStore

from bzrlib.store.text import TextStore

from bzrlib.testament import Testament

""")

from bzrlib.decorators import needs_read_lock, needs_write_lock

from bzrlib.inter import InterObject

from bzrlib.inventory import Inventory, InventoryDirectory, ROOT_ID

from bzrlib.symbol_versioning import (

        deprecated_method,

        zero_nine,

        )

from bzrlib.trace import mutter, note, warning

# Old formats display a warning, but only once

_deprecation_warning_done = False

class Repository(object):

    """Repository holding history for one or more branches.

    The repository holds and retrieves historical information including

    revisions and file history.  It's normally accessed only by the Branch,

    which views a particular line of development through that history.

    The Repository builds on top of Stores and a Transport, which respectively 

    describe the disk data format and the way of accessing the (possibly 

    remote) disk.

    """

    _file_ids_altered_regex = lazy_regex.lazy_compile(

        r'file_id="(?P<file_id>[^"]+)"'

        r'.*revision="(?P<revision_id>[^"]+)"'

        )

    @needs_write_lock

    def add_inventory(self, revid, inv, parents):

        """Add the inventory inv to the repository as revid.

        :param parents: The revision ids of the parents that revid

                        is known to have and are in the repository already.

        returns the sha1 of the serialized inventory.

        """

        assert inv.revision_id is None or inv.revision_id == revid, \

            "Mismatch between inventory revision" \

            " id and insertion revid (%r, %r)" % (inv.revision_id, revid)

        assert inv.root is not None

        inv_text = self.serialise_inventory(inv)

        inv_sha1 = osutils.sha_string(inv_text)

        inv_vf = self.control_weaves.get_weave('inventory',

                                               self.get_transaction())

        self._inventory_add_lines(inv_vf, revid, parents, osutils.split_lines(inv_text))

        return inv_sha1

    def _inventory_add_lines(self, inv_vf, revid, parents, lines):

        final_parents = []

        for parent in parents:

            if parent in inv_vf:

                final_parents.append(parent)

        inv_vf.add_lines(revid, final_parents, lines)

    @needs_write_lock

    def add_revision(self, rev_id, rev, inv=None, config=None):

        """Add rev to the revision store as rev_id.

        :param rev_id: the revision id to use.

        :param rev: The revision object.

        :param inv: The inventory for the revision. if None, it will be looked

                    up in the inventory storer

        :param config: If None no digital signature will be created.

                       If supplied its signature_needed method will be used

                       to determine if a signature should be made.

        """

        if config is not None and config.signature_needed():

            if inv is None:

                inv = self.get_inventory(rev_id)

            plaintext = Testament(rev, inv).as_short_text()

            self.store_revision_signature(

                gpg.GPGStrategy(config), plaintext, rev_id)

        if not rev_id in self.get_inventory_weave():

            if inv is None:

                raise errors.WeaveRevisionNotPresent(rev_id,

                                                     self.get_inventory_weave())

            else:

                # yes, this is not suitable for adding with ghosts.

                self.add_inventory(rev_id, inv, rev.parent_ids)

        self._revision_store.add_revision(rev, self.get_transaction())

    @needs_read_lock

    def _all_possible_ids(self):

        """Return all the possible revisions that we could find."""

        return self.get_inventory_weave().versions()

    def all_revision_ids(self):

        """Returns a list of all the revision ids in the repository. 

        This is deprecated because code should generally work on the graph

        reachable from a particular revision, and ignore any other revisions

        that might be present.  There is no direct replacement method.

        """

        return self._all_revision_ids()

    @needs_read_lock

    def _all_revision_ids(self):

        """Returns a list of all the revision ids in the repository. 

        These are in as much topological order as the underlying store can 

        present: for weaves ghosts may lead to a lack of correctness until

        the reweave updates the parents list.

        """

        if self._revision_store.text_store.listable():

            return self._revision_store.all_revision_ids(self.get_transaction())

        result = self._all_possible_ids()

        return self._eliminate_revisions_not_present(result)

    def break_lock(self):

        """Break a lock if one is present from another instance.

        Uses the ui factory to ask for confirmation if the lock may be from

        an active process.

        """

        self.control_files.break_lock()

    @needs_read_lock

    def _eliminate_revisions_not_present(self, revision_ids):

        """Check every revision id in revision_ids to see if we have it.

        Returns a set of the present revisions.

        """

        result = []

        for id in revision_ids:

            if self.has_revision(id):

               result.append(id)

        return result

    @staticmethod

    def create(a_bzrdir):

        """Construct the current default format repository in a_bzrdir."""

        return RepositoryFormat.get_default_format().initialize(a_bzrdir)

    def __init__(self, _format, a_bzrdir, control_files, _revision_store, control_store, text_store):

        """instantiate a Repository.

        :param _format: The format of the repository on disk.

        :param a_bzrdir: The BzrDir of the repository.

        In the future we will have a single api for all stores for

        getting file texts, inventories and revisions, then

        this construct will accept instances of those things.

        """

        super(Repository, self).__init__()

        self._format = _format

        # the following are part of the public API for Repository:

        self.bzrdir = a_bzrdir

        self.control_files = control_files

        self._revision_store = _revision_store

        self.text_store = text_store

        # backwards compatibility

        self.weave_store = text_store

        # not right yet - should be more semantically clear ? 

        # 

        self.control_store = control_store

        self.control_weaves = control_store

        # TODO: make sure to construct the right store classes, etc, depending

        # on whether escaping is required.

        self._warn_if_deprecated()

        self._serializer = xml5.serializer_v5

    def __repr__(self):

        return '%s(%r)' % (self.__class__.__name__, 

                           self.bzrdir.transport.base)

    def is_locked(self):

        return self.control_files.is_locked()

    def lock_write(self):

        self.control_files.lock_write()

    def lock_read(self):

        self.control_files.lock_read()

    def get_physical_lock_status(self):

        return self.control_files.get_physical_lock_status()

    @needs_read_lock

    def missing_revision_ids(self, other, revision_id=None):

        """Return the revision ids that other has that this does not.

        These are returned in topological order.

        revision_id: only return revision ids included by revision_id.

        """

        return InterRepository.get(other, self).missing_revision_ids(revision_id)

    @staticmethod

    def open(base):

        """Open the repository rooted at base.

        For instance, if the repository is at URL/.bzr/repository,

        Repository.open(URL) -> a Repository instance.

        """

        control = bzrdir.BzrDir.open(base)

        return control.open_repository()

    def copy_content_into(self, destination, revision_id=None, basis=None):

        """Make a complete copy of the content in self into destination.

        This is a destructive operation! Do not use it on existing 

        repositories.

        """

        return InterRepository.get(self, destination).copy_content(revision_id, basis)

    def fetch(self, source, revision_id=None, pb=None):

        """Fetch the content required to construct revision_id from source.

        If revision_id is None all content is copied.

        """

        return InterRepository.get(source, self).fetch(revision_id=revision_id,

                                                       pb=pb)

    def get_commit_builder(self, branch, parents, config, timestamp=None, 

                           timezone=None, committer=None, revprops=None, 

                           revision_id=None):

        """Obtain a CommitBuilder for this repository.

        :param branch: Branch to commit to.

        :param parents: Revision ids of the parents of the new revision.

        :param config: Configuration to use.

        :param timestamp: Optional timestamp recorded for commit.

        :param timezone: Optional timezone for timestamp.

        :param committer: Optional committer to set for commit.

        :param revprops: Optional dictionary of revision properties.

        :param revision_id: Optional revision id.

        """

        return _CommitBuilder(self, parents, config, timestamp, timezone,

                              committer, revprops, revision_id)

    def unlock(self):

        self.control_files.unlock()

    @needs_read_lock

    def clone(self, a_bzrdir, revision_id=None, basis=None):

        """Clone this repository into a_bzrdir using the current format.

        Currently no check is made that the format of this repository and

        the bzrdir format are compatible. FIXME RBC 20060201.

        """

        if not isinstance(a_bzrdir._format, self.bzrdir._format.__class__):

            # use target default format.

            result = a_bzrdir.create_repository()

        # FIXME RBC 20060209 split out the repository type to avoid this check ?

        elif isinstance(a_bzrdir._format,

                      (bzrdir.BzrDirFormat4,

                       bzrdir.BzrDirFormat5,

                       bzrdir.BzrDirFormat6)):

            result = a_bzrdir.open_repository()

        else:

            result = self._format.initialize(a_bzrdir, shared=self.is_shared())

        self.copy_content_into(result, revision_id, basis)

        return result

    @needs_read_lock

    def has_revision(self, revision_id):

        """True if this repository has a copy of the revision."""

        return self._revision_store.has_revision_id(revision_id,

                                                    self.get_transaction())

    @needs_read_lock

    def get_revision_reconcile(self, revision_id):

        """'reconcile' helper routine that allows access to a revision always.

        This variant of get_revision does not cross check the weave graph

        against the revision one as get_revision does: but it should only

        be used by reconcile, or reconcile-alike commands that are correcting

        or testing the revision graph.

        """

        if not revision_id or not isinstance(revision_id, basestring):

            raise errors.InvalidRevisionId(revision_id=revision_id,

                                           branch=self)

        return self._revision_store.get_revisions([revision_id],

                                                  self.get_transaction())[0]

    @needs_read_lock

    def get_revisions(self, revision_ids):

        return self._revision_store.get_revisions(revision_ids,

                                                  self.get_transaction())

    @needs_read_lock

    def get_revision_xml(self, revision_id):

        rev = self.get_revision(revision_id) 

        rev_tmp = StringIO()

        # the current serializer..

        self._revision_store._serializer.write_revision(rev, rev_tmp)

        rev_tmp.seek(0)

        return rev_tmp.getvalue()

    @needs_read_lock

    def get_revision(self, revision_id):

        """Return the Revision object for a named revision"""

        r = self.get_revision_reconcile(revision_id)

        # weave corruption can lead to absent revision markers that should be

        # present.

        # the following test is reasonably cheap (it needs a single weave read)

        # and the weave is cached in read transactions. In write transactions

        # it is not cached but typically we only read a small number of

        # revisions. For knits when they are introduced we will probably want

        # to ensure that caching write transactions are in use.

        inv = self.get_inventory_weave()

        self._check_revision_parents(r, inv)

        return r

    @needs_read_lock

    def get_deltas_for_revisions(self, revisions):

        """Produce a generator of revision deltas.

        Note that the input is a sequence of REVISIONS, not revision_ids.

        Trees will be held in memory until the generator exits.

        Each delta is relative to the revision's lefthand predecessor.

        """

        required_trees = set()

        for revision in revisions:

            required_trees.add(revision.revision_id)

            required_trees.update(revision.parent_ids[:1])

        trees = dict((t.get_revision_id(), t) for 

                     t in self.revision_trees(required_trees))

        for revision in revisions:

            if not revision.parent_ids:

                old_tree = self.revision_tree(None)

            else:

                old_tree = trees[revision.parent_ids[0]]

            yield trees[revision.revision_id].changes_from(old_tree)

    @needs_read_lock

    def get_revision_delta(self, revision_id):

        """Return the delta for one revision.

        The delta is relative to the left-hand predecessor of the

        revision.

        """

        r = self.get_revision(revision_id)

        return list(self.get_deltas_for_revisions([r]))[0]

    def _check_revision_parents(self, revision, inventory):

        """Private to Repository and Fetch.

        This checks the parentage of revision in an inventory weave for 

        consistency and is only applicable to inventory-weave-for-ancestry

        using repository formats & fetchers.

        """

        weave_parents = inventory.get_parents(revision.revision_id)

        weave_names = inventory.versions()

        for parent_id in revision.parent_ids:

            if parent_id in weave_names:

                # this parent must not be a ghost.

                if not parent_id in weave_parents:

                    # but it is a ghost

                    raise errors.CorruptRepository(self)

    @needs_write_lock

    def store_revision_signature(self, gpg_strategy, plaintext, revision_id):

        signature = gpg_strategy.sign(plaintext)

        self._revision_store.add_revision_signature_text(revision_id,

                                                         signature,

                                                         self.get_transaction())

    def fileids_altered_by_revision_ids(self, revision_ids):

        """Find the file ids and versions affected by revisions.

        :param revisions: an iterable containing revision ids.

        :return: a dictionary mapping altered file-ids to an iterable of

        revision_ids. Each altered file-ids has the exact revision_ids that

        altered it listed explicitly.

        """

        assert self._serializer.support_altered_by_hack, \

            ("fileids_altered_by_revision_ids only supported for branches " 

             "which store inventory as unnested xml, not on %r" % self)

        selected_revision_ids = set(revision_ids)

        w = self.get_inventory_weave()

        result = {}

        # this code needs to read every new line in every inventory for the

        # inventories [revision_ids]. Seeing a line twice is ok. Seeing a line

        # not present in one of those inventories is unnecessary but not 

        # harmful because we are filtering by the revision id marker in the

        # inventory lines : we only select file ids altered in one of those  

        # revisions. We don't need to see all lines in the inventory because

        # only those added in an inventory in rev X can contain a revision=X

        # line.

        unescape_revid_cache = {}

        unescape_fileid_cache = {}

        # jam 20061218 In a big fetch, this handles hundreds of thousands

        # of lines, so it has had a lot of inlining and optimizing done.

        # Sorry that it is a little bit messy.

        # Move several functions to be local variables, since this is a long

        # running loop.

        search = self._file_ids_altered_regex.search

        unescape = _unescape_xml

        setdefault = result.setdefault

        pb = ui.ui_factory.nested_progress_bar()

        try:

            for line in w.iter_lines_added_or_present_in_versions(

                                        selected_revision_ids, pb=pb):

                match = search(line)

                if match is None:

                    continue

                # One call to match.group() returning multiple items is quite a

                # bit faster than 2 calls to match.group() each returning 1

                file_id, revision_id = match.group('file_id', 'revision_id')

                # Inlining the cache lookups helps a lot when you make 170,000

                # lines and 350k ids, versus 8.4 unique ids.

                # Using a cache helps in 2 ways:

                #   1) Avoids unnecessary decoding calls

                #   2) Re-uses cached strings, which helps in future set and

                #      equality checks.

                # (2) is enough that removing encoding entirely along with

                # the cache (so we are using plain strings) results in no

                # performance improvement.

                try:

                    revision_id = unescape_revid_cache[revision_id]

                except KeyError:

                    unescaped = unescape(revision_id)

                    unescape_revid_cache[revision_id] = unescaped

                    revision_id = unescaped

                if revision_id in selected_revision_ids:

                    try:

                        file_id = unescape_fileid_cache[file_id]

                    except KeyError:

                        unescaped = unescape(file_id)

                        unescape_fileid_cache[file_id] = unescaped

                        file_id = unescaped

                    setdefault(file_id, set()).add(revision_id)

        finally:

            pb.finished()

        return result

    @needs_read_lock

    def get_inventory_weave(self):

        return self.control_weaves.get_weave('inventory',

            self.get_transaction())

    @needs_read_lock

    def get_inventory(self, revision_id):

        """Get Inventory object by hash."""

        return self.deserialise_inventory(

            revision_id, self.get_inventory_xml(revision_id))

    def deserialise_inventory(self, revision_id, xml):

        """Transform the xml into an inventory object. 

        :param revision_id: The expected revision id of the inventory.

        :param xml: A serialised inventory.

        """

        result = self._serializer.read_inventory_from_string(xml)

        result.root.revision = revision_id

        return result

    def serialise_inventory(self, inv):

        return self._serializer.write_inventory_to_string(inv)

    @needs_read_lock

    def get_inventory_xml(self, revision_id):

        """Get inventory XML as a file object."""

        try:

            assert isinstance(revision_id, basestring), type(revision_id)

            iw = self.get_inventory_weave()

            return iw.get_text(revision_id)

        except IndexError:

            raise errors.HistoryMissing(self, 'inventory', revision_id)

    @needs_read_lock

    def get_inventory_sha1(self, revision_id):

        """Return the sha1 hash of the inventory entry

        """

        return self.get_revision(revision_id).inventory_sha1

    @needs_read_lock

    def get_revision_graph(self, revision_id=None):

        """Return a dictionary containing the revision graph.

        :param revision_id: The revision_id to get a graph from. If None, then

        the entire revision graph is returned. This is a deprecated mode of

        operation and will be removed in the future.

        :return: a dictionary of revision_id->revision_parents_list.

        """

        # special case NULL_REVISION

        if revision_id == _mod_revision.NULL_REVISION:

            return {}

        a_weave = self.get_inventory_weave()

        all_revisions = self._eliminate_revisions_not_present(

                                a_weave.versions())

        entire_graph = dict([(node, a_weave.get_parents(node)) for 

                             node in all_revisions])

        if revision_id is None:

            return entire_graph

        elif revision_id not in entire_graph:

            raise errors.NoSuchRevision(self, revision_id)

        else:

            # add what can be reached from revision_id

            result = {}

            pending = set([revision_id])

            while len(pending) > 0:

                node = pending.pop()

                result[node] = entire_graph[node]

                for revision_id in result[node]:

                    if revision_id not in result:

                        pending.add(revision_id)

            return result

    @needs_read_lock

    def get_revision_graph_with_ghosts(self, revision_ids=None):

        """Return a graph of the revisions with ghosts marked as applicable.

        :param revision_ids: an iterable of revisions to graph or None for all.

        :return: a Graph object with the graph reachable from revision_ids.

        """

        result = graph.Graph()

        if not revision_ids:

            pending = set(self.all_revision_ids())

            required = set([])

        else:

            pending = set(revision_ids)

            # special case NULL_REVISION

            if _mod_revision.NULL_REVISION in pending:

                pending.remove(_mod_revision.NULL_REVISION)

            required = set(pending)

        done = set([])

        while len(pending):

            revision_id = pending.pop()

            try:

                rev = self.get_revision(revision_id)

            except errors.NoSuchRevision:

                if revision_id in required:

                    raise

                # a ghost

                result.add_ghost(revision_id)

                continue

            for parent_id in rev.parent_ids:

                # is this queued or done ?

                if (parent_id not in pending and

                    parent_id not in done):

                    # no, queue it.

                    pending.add(parent_id)

            result.add_node(revision_id, rev.parent_ids)

            done.add(revision_id)

        return result

    @needs_read_lock

    def get_revision_inventory(self, revision_id):

        """Return inventory of a past revision."""

        # TODO: Unify this with get_inventory()

        # bzr 0.0.6 and later imposes the constraint that the inventory_id

        # must be the same as its revision, so this is trivial.

        if revision_id is None:

            # This does not make sense: if there is no revision,

            # then it is the current tree inventory surely ?!

            # and thus get_root_id() is something that looks at the last

            # commit on the branch, and the get_root_id is an inventory check.

            raise NotImplementedError

            # return Inventory(self.get_root_id())

        else:

            return self.get_inventory(revision_id)

    @needs_read_lock

    def is_shared(self):

        """Return True if this repository is flagged as a shared repository."""

        raise NotImplementedError(self.is_shared)

    @needs_write_lock

    def reconcile(self, other=None, thorough=False):

        """Reconcile this repository."""

        from bzrlib.reconcile import RepoReconciler

        reconciler = RepoReconciler(self, thorough=thorough)

        reconciler.reconcile()

        return reconciler

    @needs_read_lock

    def revision_tree(self, revision_id):

        """Return Tree for a revision on this branch.

        `revision_id` may be None for the empty tree revision.

        """

        # TODO: refactor this to use an existing revision object

        # so we don't need to read it in twice.

        if revision_id is None or revision_id == _mod_revision.NULL_REVISION:

            return RevisionTree(self, Inventory(root_id=None), 

                                _mod_revision.NULL_REVISION)

        else:

            inv = self.get_revision_inventory(revision_id)

            return RevisionTree(self, inv, revision_id)

    @needs_read_lock

    def revision_trees(self, revision_ids):

        """Return Tree for a revision on this branch.

        `revision_id` may not be None or 'null:'"""

        assert None not in revision_ids

        assert _mod_revision.NULL_REVISION not in revision_ids

        texts = self.get_inventory_weave().get_texts(revision_ids)

        for text, revision_id in zip(texts, revision_ids):

            inv = self.deserialise_inventory(revision_id, text)

            yield RevisionTree(self, inv, revision_id)

    @needs_read_lock

    def get_ancestry(self, revision_id):

        """Return a list of revision-ids integrated by a revision.

        The first element of the list is always None, indicating the origin 

        revision.  This might change when we have history horizons, or 

        perhaps we should have a new API.

        This is topologically sorted.

        """

        if revision_id is None:

            return [None]

        if not self.has_revision(revision_id):

            raise errors.NoSuchRevision(self, revision_id)

        w = self.get_inventory_weave()

        candidates = w.get_ancestry(revision_id)

        return [None] + candidates # self._eliminate_revisions_not_present(candidates)

    @needs_read_lock

    def print_file(self, file, revision_id):

        """Print `file` to stdout.

        FIXME RBC 20060125 as John Meinel points out this is a bad api

        - it writes to stdout, it assumes that that is valid etc. Fix

        by creating a new more flexible convenience function.

        """

        tree = self.revision_tree(revision_id)

        # use inventory as it was in that revision

        file_id = tree.inventory.path2id(file)

        if not file_id:

            # TODO: jam 20060427 Write a test for this code path

            #       it had a bug in it, and was raising the wrong

            #       exception.

            raise errors.BzrError("%r is not present in revision %s" % (file, revision_id))

        tree.print_file(file_id)

    def get_transaction(self):

        return self.control_files.get_transaction()

    def revision_parents(self, revid):

        return self.get_inventory_weave().parent_names(revid)