~bzr-pqm/bzr/bzr.dev

1196 by Martin Pool
- [WIP] retrieve historical texts from weaves
1
#! /usr/bin/python
2
3
# Copyright (C) 2005 Canonical Ltd
4
5
# This program is free software; you can redistribute it and/or modify
6
# it under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 2 of the License, or
8
# (at your option) any later version.
9
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13
# GNU General Public License for more details.
14
15
# You should have received a copy of the GNU General Public License
16
# along with this program; if not, write to the Free Software
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18
1393.1.19 by Martin Pool
- add WeaveStore.__iter__, __contains__ and copy_multi()
19
# XXX: Some consideration of the problems that might occur if there are
20
# files whose id differs only in case.  That should probably be forbidden.
21
22
1471 by Robert Collins
Bugfix to previous url escaping patch - include weave stores
23
import errno
24
import os
1393.1.19 by Martin Pool
- add WeaveStore.__iter__, __contains__ and copy_multi()
25
from cStringIO import StringIO
1471 by Robert Collins
Bugfix to previous url escaping patch - include weave stores
26
import urllib
1223 by Martin Pool
- store inventories in weave
27
28
from bzrlib.weavefile import read_weave, write_weave_v5
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
29
from bzrlib.weave import WeaveFile
1430 by Robert Collins
touchup the prefixed-store patch
30
from bzrlib.store import TransportStore, hash_prefix
1223 by Martin Pool
- store inventories in weave
31
from bzrlib.atomicfile import AtomicFile
1429 by Robert Collins
merge in niemeyers prefixed-store patch
32
from bzrlib.errors import NoSuchFile, FileExists
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
33
from bzrlib.symbol_versioning import *
1393.1.19 by Martin Pool
- add WeaveStore.__iter__, __contains__ and copy_multi()
34
from bzrlib.trace import mutter
1393.2.2 by John Arbash Meinel
Updated stores to use Transport
35
36
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
37
class VersionedFileStore(TransportStore):
38
    """Collection of many versioned files in a transport."""
1393.1.19 by Martin Pool
- add WeaveStore.__iter__, __contains__ and copy_multi()
39
1185.58.4 by John Arbash Meinel
Added permission checking to Branch, and propogated that change into the stores.
40
    def __init__(self, transport, prefixed=False, precious=False,
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
41
                 dir_mode=None, file_mode=None,
42
                 versionedfile_class=WeaveFile):
1185.58.4 by John Arbash Meinel
Added permission checking to Branch, and propogated that change into the stores.
43
        super(WeaveStore, self).__init__(transport,
44
                dir_mode=dir_mode, file_mode=file_mode,
45
                prefixed=prefixed, compressed=False)
1417.1.10 by Robert Collins
add a cache bound to Transactions, and a precious facility, so that we keep inventory.weave in memory, but can discard weaves for other such files.
46
        self._precious = precious
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
47
        self._versionedfile_class = versionedfile_class
1196 by Martin Pool
- [WIP] retrieve historical texts from weaves
48
1223 by Martin Pool
- store inventories in weave
49
    def filename(self, file_id):
1393.2.2 by John Arbash Meinel
Updated stores to use Transport
50
        """Return the path relative to the transport root."""
1429 by Robert Collins
merge in niemeyers prefixed-store patch
51
        if self._prefixed:
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
52
            return hash_prefix(file_id) + urllib.quote(file_id)
1429 by Robert Collins
merge in niemeyers prefixed-store patch
53
        else:
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
54
            return urllib.quote(file_id)
1429 by Robert Collins
merge in niemeyers prefixed-store patch
55
1393.1.19 by Martin Pool
- add WeaveStore.__iter__, __contains__ and copy_multi()
56
    def __iter__(self):
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
57
        suffixes = self._versionedfile_class.get_suffixes()
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
58
        ids = set()
1479 by Robert Collins
More quoting at the transport layer bugfixes.
59
        for relpath in self._iter_files_recursive():
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
60
            for suffix in suffixes:
61
                if relpath.endswith(suffix):
62
                    id = os.path.basename(relpath[:-len(suffix)])
63
                    if not id in ids:
64
                        yield id
65
                        ids.add(id)
1393.1.19 by Martin Pool
- add WeaveStore.__iter__, __contains__ and copy_multi()
66
1442.1.45 by Robert Collins
replace __contains__ calls in stores with has_id
67
    def has_id(self, fileid):
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
68
        suffixes = self._versionedfile_class.get_suffixes()
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
69
        filename = self.filename(fileid)
70
        for suffix in suffixes:
71
            if not self._transport.has(filename + suffix):
72
                return False
73
        return True
1223 by Martin Pool
- store inventories in weave
74
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
75
    def get_weave(self, file_id, transaction):
76
        weave = transaction.map.find_weave(file_id)
77
        if weave:
78
            mutter("cache hit in %s for %s", self, file_id)
79
            return weave
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
80
        w = self._versionedfile_class(self.filename(file_id), self._transport, self._file_mode)
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
81
        transaction.map.add_weave(file_id, w)
1417.1.10 by Robert Collins
add a cache bound to Transactions, and a precious facility, so that we keep inventory.weave in memory, but can discard weaves for other such files.
82
        transaction.register_clean(w, precious=self._precious)
1363 by Martin Pool
- add quick-and-dirty cache for weaves to speed check command
83
        return w
1262 by Martin Pool
- fetch should also copy ancestry records
84
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
85
    @deprecated_method(zero_eight)
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
86
    def get_lines(self, file_id, rev_id, transaction):
1262 by Martin Pool
- fetch should also copy ancestry records
87
        """Return text from a particular version of a weave.
88
89
        Returned as a list of lines."""
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
90
        w = self.get_weave(file_id, transaction)
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
91
        return w.get_lines(rev_id)
1196 by Martin Pool
- [WIP] retrieve historical texts from weaves
92
    
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
93
    def _new_weave(self, file_id, transaction):
94
        """Make a new weave for file_id and return it."""
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
95
        weave = self._make_new_versionedfile(file_id)
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
96
        transaction.map.add_weave(file_id, weave)
97
        transaction.register_clean(weave, precious=self._precious)
98
        return weave
1185.33.88 by Martin Pool
[patch] read only the table-of-contents of weaves to speed up commit
99
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
100
    def _make_new_versionedfile(self, file_id):
101
        try:
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
102
            weave = self._versionedfile_class(self.filename(file_id), self._transport, self._file_mode)
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
103
        except NoSuchFile:
104
            if not self._prefixed:
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
105
                # unexpected error - NoSuchFile is raised on a missing dir only and that
106
                # only occurs when we are prefixed.
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
107
                raise
108
            self._transport.mkdir(hash_prefix(file_id), mode=self._dir_mode)
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
109
            weave = self._versionedfile_class(self.filename(file_id), self._transport, self._file_mode)
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
110
        return weave
111
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
112
    def get_weave_or_empty(self, file_id, transaction):
1224 by Martin Pool
- new method WeaveStore.get_weave_or_empty
113
        """Return a weave, or an empty one if it doesn't exist.""" 
114
        try:
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
115
            return self.get_weave(file_id, transaction)
1393.2.2 by John Arbash Meinel
Updated stores to use Transport
116
        except NoSuchFile:
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
117
            return self._new_weave(file_id, transaction)
1224 by Martin Pool
- new method WeaveStore.get_weave_or_empty
118
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
119
    @deprecated_method(zero_eight)
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
120
    def put_weave(self, file_id, weave, transaction):
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
121
        """This is a deprecated API: It writes an entire collection of ids out.
122
        
123
        This became inappropriate when we made a versioned file api which
124
        tracks the state of the collection of versions for a single id.
125
        
126
        Its maintained for backwards compatability but will only work on
127
        weave stores - pre 0.8 repositories.
128
        """
129
        self._put_weave(self, file_id, weave, transaction)
130
131
    def _put_weave(self, file_id, weave, transaction):
132
        """Preserved here for upgrades-to-weaves to use."""
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
133
        myweave = self._make_new_versionedfile(file_id)
134
        myweave.join(weave)
1223 by Martin Pool
- store inventories in weave
135
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
136
    @deprecated_method(zero_eight)
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
137
    def add_text(self, file_id, rev_id, new_lines, parents, transaction):
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
138
        """This method was a shorthand for 
139
140
        vfile = self.get_weave_or_empty(file_id, transaction)
141
        vfile.add_lines(rev_id, parents, new_lines)
142
        """
143
        vfile = self.get_weave_or_empty(file_id, transaction)
144
        vfile.add_lines(rev_id, parents, new_lines)
1223 by Martin Pool
- store inventories in weave
145
        
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
146
    @deprecated_method(zero_eight)
1417.1.8 by Robert Collins
use transactions in the weave store interface, which enables caching for log
147
    def add_identical_text(self, file_id, old_rev_id, new_rev_id, parents,
148
                           transaction):
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
149
        """This method was a shorthand for
150
151
        vfile = self.get_weave_or_empty(file_id, transaction)
152
        vfile.clone_text(new_rev_id, old_rev_id, parents)
153
        """
154
        vfile = self.get_weave_or_empty(file_id, transaction)
155
        vfile.clone_text(new_rev_id, old_rev_id, parents)
1223 by Martin Pool
- store inventories in weave
156
     
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
157
    def copy_all_ids(self, store_from, pb=None, from_transaction=None):
158
        """Copy all the file ids from store_from into self."""
159
        if from_transaction is None:
160
            warn("Please pase from_transaction into "
161
                 "versioned_store.copy_all_ids.", stacklevel=2)
162
        if not store_from.listable():
163
            raise UnlistableStore(store_from)
164
        ids = []
165
        for count, file_id in enumerate(store_from):
166
            if pb:
167
                pb.update('listing files', count, count)
168
            ids.append(file_id)
169
        if pb:
170
            pb.clear()
171
        mutter('copy_all ids: %r', ids)
172
        self.copy_multi(store_from, ids, pb=pb,
173
                        from_transaction=from_transaction)
174
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
175
    def copy_multi(self, from_store, file_ids, pb=None, from_transaction=None):
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
176
        """Copy all the versions for multiple file_ids from from_store.
177
        
178
        :param from_transaction: required current transaction in from_store.
179
        """
1393.1.19 by Martin Pool
- add WeaveStore.__iter__, __contains__ and copy_multi()
180
        assert isinstance(from_store, WeaveStore)
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
181
        if from_transaction is None:
182
            warn("WeaveStore.copy_multi without a from_transaction parameter "
183
                 "is deprecated. Please provide a from_transaction.",
184
                 DeprecationWarning,
185
                 stacklevel=2)
1185.79.2 by John Arbash Meinel
Adding progress bars to copy_all and copy_multi, fixing ordering of repository.clone() to pull inventories after weaves.
186
        for count, f in enumerate(file_ids):
1393.1.19 by Martin Pool
- add WeaveStore.__iter__, __contains__ and copy_multi()
187
            mutter("copy weave {%s} into %s", f, self)
1185.79.2 by John Arbash Meinel
Adding progress bars to copy_all and copy_multi, fixing ordering of repository.clone() to pull inventories after weaves.
188
            if pb:
189
                pb.update('copy', count, len(file_ids))
1563.2.10 by Robert Collins
Change weave store to be a versioned store, using WeaveFiles which maintain integrity without needing explicit 'put' operations.
190
            # if we have it in cache, its faster.
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
191
            if not from_transaction:
1563.2.14 by Robert Collins
Prepare weave store to delegate copy details to the versioned file.
192
                from bzrlib.transactions import PassThroughTransaction
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
193
                from_transaction = PassThroughTransaction()
194
            # joining is fast with knits, and bearable for weaves -
195
            # indeed the new case can be optimised
196
            target = self._make_new_versionedfile(f)
197
            target.join(from_store.get_weave(f, from_transaction))
1185.79.2 by John Arbash Meinel
Adding progress bars to copy_all and copy_multi, fixing ordering of repository.clone() to pull inventories after weaves.
198
        if pb:
199
            pb.clear()
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
200
201
202
WeaveStore = VersionedFileStore