23
25
from cStringIO import StringIO
25
from warnings import warn
31
28
from bzrlib.weavefile import read_weave, write_weave_v5
32
from bzrlib.weave import WeaveFile, Weave
33
from bzrlib.store import TransportStore
29
from bzrlib.weave import Weave
30
from bzrlib.store import TransportStore, hash_prefix
34
31
from bzrlib.atomicfile import AtomicFile
35
from bzrlib.symbol_versioning import (deprecated_method,
32
from bzrlib.errors import NoSuchFile, FileExists
37
33
from bzrlib.trace import mutter
41
class VersionedFileStore(TransportStore):
42
"""Collection of many versioned files in a transport."""
44
# TODO: Rather than passing versionedfile_kwargs, perhaps pass in a
45
# transport factory callable?
36
class WeaveStore(TransportStore):
37
"""Collection of several weave files in a directory.
39
This has some shortcuts for reading and writing them.
41
FILE_SUFFIX = '.weave'
46
43
def __init__(self, transport, prefixed=False, precious=False,
47
dir_mode=None, file_mode=None,
48
versionedfile_class=WeaveFile,
49
versionedfile_kwargs={},
51
super(VersionedFileStore, self).__init__(transport,
44
dir_mode=None, file_mode=None):
45
super(WeaveStore, self).__init__(transport,
52
46
dir_mode=dir_mode, file_mode=file_mode,
53
prefixed=prefixed, compressed=False, escaped=escaped)
47
prefixed=prefixed, compressed=False)
54
48
self._precious = precious
55
self._versionedfile_class = versionedfile_class
56
self._versionedfile_kwargs = versionedfile_kwargs
58
def _clear_cache_id(self, file_id, transaction):
59
"""WARNING may lead to inconsistent object references for file_id.
61
Remove file_id from the transaction map.
63
NOT in the transaction api because theres no reliable way to clear
64
callers. So its here for very specialised use rather than having an
67
weave = transaction.map.find_weave(file_id)
69
mutter("old data in transaction in %s for %s", self, file_id)
70
# FIXME abstraction violation - transaction now has stale data.
71
transaction.map.remove_object(weave)
73
50
def filename(self, file_id):
74
51
"""Return the path relative to the transport root."""
75
return self._relpath(file_id)
53
return hash_prefix(file_id) + urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
55
return urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
77
57
def __iter__(self):
78
suffixes = self._versionedfile_class.get_suffixes()
58
l = len(WeaveStore.FILE_SUFFIX)
80
59
for relpath in self._iter_files_recursive():
81
for suffix in suffixes:
82
if relpath.endswith(suffix):
83
# TODO: use standard remove_suffix function
84
escaped_id = os.path.basename(relpath[:-len(suffix)])
85
file_id = self._unescape(escaped_id)
86
if file_id not in ids:
89
break # only one suffix can match
91
def has_id(self, file_id):
92
suffixes = self._versionedfile_class.get_suffixes()
93
filename = self.filename(file_id)
94
for suffix in suffixes:
95
if not self._transport.has(filename + suffix):
99
def get_empty(self, file_id, transaction):
100
"""Get an empty weave, which implies deleting the existing one first."""
101
if self.has_id(file_id):
102
self.delete(file_id, transaction)
103
return self.get_weave_or_empty(file_id, transaction)
105
def delete(self, file_id, transaction):
106
"""Remove file_id from the store."""
107
suffixes = self._versionedfile_class.get_suffixes()
108
filename = self.filename(file_id)
109
for suffix in suffixes:
110
self._transport.delete(filename + suffix)
111
self._clear_cache_id(file_id, transaction)
60
if relpath.endswith(WeaveStore.FILE_SUFFIX):
61
yield os.path.basename(relpath[:-l])
63
def has_id(self, fileid):
64
return self._transport.has(self.filename(fileid))
113
66
def _get(self, file_id):
114
67
return self._transport.get(self.filename(file_id))
116
69
def _put(self, file_id, f):
117
fn = self.filename(file_id)
119
return self._transport.put_file(fn, f, mode=self._file_mode)
120
except errors.NoSuchFile:
121
if not self._prefixed:
123
self._transport.mkdir(os.path.dirname(fn), mode=self._dir_mode)
124
return self._transport.put_file(fn, f, mode=self._file_mode)
126
def get_weave(self, file_id, transaction, _filename=None):
127
"""Return the VersionedFile for file_id.
129
:param _filename: filename that would be returned from self.filename for
130
file_id. This is used to reduce duplicate filename calculations when
131
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
72
self._transport.mkdir(hash_prefix(file_id), mode=self._dir_mode)
75
return self._transport.put(self.filename(file_id), f, mode=self._file_mode)
77
def get_weave(self, file_id, transaction):
133
78
weave = transaction.map.find_weave(file_id)
134
if weave is not None:
135
#mutter("cache hit in %s for %s", self, file_id)
137
if _filename is None:
138
_filename = self.filename(file_id)
139
if transaction.writeable():
140
w = self._versionedfile_class(_filename, self._transport, self._file_mode,
141
**self._versionedfile_kwargs)
142
transaction.map.add_weave(file_id, w)
143
transaction.register_dirty(w)
145
w = self._versionedfile_class(_filename,
150
**self._versionedfile_kwargs)
151
transaction.map.add_weave(file_id, w)
152
transaction.register_clean(w, precious=self._precious)
155
def _make_new_versionedfile(self, file_id, transaction,
156
known_missing=False, _filename=None):
157
"""Make a new versioned file.
159
:param _filename: filename that would be returned from self.filename for
160
file_id. This is used to reduce duplicate filename calculations when
161
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
80
mutter("cache hit in %s for %s", self, file_id)
82
w = read_weave(self._get(file_id))
83
transaction.map.add_weave(file_id, w)
84
transaction.register_clean(w, precious=self._precious)
85
# TODO: jam 20051219 This should check if there is a prelude
86
# which is already cached, and if so, should remove it
87
# But transaction doesn't seem to have a 'remove'
88
# One workaround would be to re-add the object with
92
def get_weave_prelude(self, file_id, transaction):
94
weave = transaction.map.find_weave(weave_id)
96
mutter("cache hit in %s for %s", self, weave_id)
98
# We want transactions to also cache preludes if that
99
# is all that we are loading. So we need a unique
100
# identifier, so that someone who wants the whole text
101
# won't get just the prelude
102
weave_id = 'PRELUDE-' + file_id
103
weave = transaction.map.find_weave(weave_id)
105
mutter("cache hit in %s for %s", self, weave_id)
107
w = read_weave(self._get(file_id), prelude=True)
108
transaction.map.add_weave(weave_id, w)
109
transaction.register_clean(w, precious=self._precious)
112
def get_lines(self, file_id, rev_id, transaction):
113
"""Return text from a particular version of a weave.
115
Returned as a list of lines."""
116
w = self.get_weave(file_id, transaction)
117
return w.get(w.lookup(rev_id))
119
def get_weave_prelude_or_empty(self, file_id, transaction):
120
"""cheap version that reads the prelude but not the lines
163
if not known_missing and self.has_id(file_id):
164
self.delete(file_id, transaction)
165
if _filename is None:
166
_filename = self.filename(file_id)
168
# we try without making the directory first because thats optimising
169
# for the common case.
170
weave = self._versionedfile_class(_filename, self._transport, self._file_mode, create=True,
171
**self._versionedfile_kwargs)
172
except errors.NoSuchFile:
173
if not self._prefixed:
174
# unexpected error - NoSuchFile is expected to be raised on a
175
# missing dir only and that only occurs when we are prefixed.
177
self._transport.mkdir(self.hash_prefix(file_id), mode=self._dir_mode)
178
weave = self._versionedfile_class(_filename, self._transport,
179
self._file_mode, create=True,
180
**self._versionedfile_kwargs)
123
return self.get_weave_prelude(file_id, transaction)
125
# We can cache here, because we know that there
126
# is no complete object, since we got NoSuchFile
127
weave = Weave(weave_name=file_id)
128
transaction.map.add_weave(file_id, weave)
129
transaction.register_clean(weave, precious=self._precious)
183
132
def get_weave_or_empty(self, file_id, transaction):
184
"""Return a weave, or an empty one if it doesn't exist."""
185
# This is typically used from 'commit' and 'fetch/push/pull' where
186
# we scan across many versioned files once. As such the small overhead
187
# of calculating the filename before doing a cache lookup is more than
188
# compensated for by not calculating the filename when making new
190
_filename = self.filename(file_id)
133
"""Return a weave, or an empty one if it doesn't exist."""
192
return self.get_weave(file_id, transaction, _filename=_filename)
193
except errors.NoSuchFile:
194
weave = self._make_new_versionedfile(file_id, transaction,
195
known_missing=True, _filename=_filename)
135
return self.get_weave(file_id, transaction)
137
weave = Weave(weave_name=file_id)
196
138
transaction.map.add_weave(file_id, weave)
197
# has to be dirty - its able to mutate on its own.
198
transaction.register_dirty(weave)
139
transaction.register_clean(weave, precious=self._precious)
201
def _put_weave(self, file_id, weave, transaction):
202
"""Preserved here for upgrades-to-weaves to use."""
203
myweave = self._make_new_versionedfile(file_id, transaction)
206
def copy(self, source, result_id, transaction):
207
"""Copy the source versioned file to result_id in this store."""
208
self._clear_cache_id(result_id, transaction)
209
source.copy_to(self.filename(result_id), self._transport)
211
def copy_all_ids(self, store_from, pb=None, from_transaction=None,
212
to_transaction=None):
213
"""Copy all the file ids from store_from into self."""
214
if from_transaction is None:
215
warn("Please pass from_transaction into "
216
"versioned_store.copy_all_ids.", stacklevel=2)
217
if to_transaction is None:
218
warn("Please pass to_transaction into "
219
"versioned_store.copy_all_ids.", stacklevel=2)
220
if not store_from.listable():
221
raise errors.UnlistableStore(store_from)
223
for count, file_id in enumerate(store_from):
225
pb.update('listing files', count, count)
229
mutter('copy_all ids: %r', ids)
230
self.copy_multi(store_from, ids, pb=pb,
231
from_transaction=from_transaction,
232
to_transaction=to_transaction)
234
def copy_multi(self, from_store, file_ids, pb=None, from_transaction=None,
235
to_transaction=None):
236
"""Copy all the versions for multiple file_ids from from_store.
142
def put_weave(self, file_id, weave, transaction):
143
"""Write back a modified weave"""
144
transaction.register_dirty(weave)
145
# TODO FOR WRITE TRANSACTIONS: this should be done in a callback
146
# from the transaction, when it decides to save.
148
write_weave_v5(weave, sio)
150
self._put(file_id, sio)
152
def add_text(self, file_id, rev_id, new_lines, parents, transaction):
153
w = self.get_weave_or_empty(file_id, transaction)
154
parent_idxs = map(w.lookup, parents)
155
w.add(rev_id, parent_idxs, new_lines)
156
self.put_weave(file_id, w, transaction)
238
:param from_transaction: required current transaction in from_store.
240
from bzrlib.transactions import PassThroughTransaction
158
def add_identical_text(self, file_id, old_rev_id, new_rev_id, parents,
160
w = self.get_weave_or_empty(file_id, transaction)
161
parent_idxs = map(w.lookup, parents)
162
w.add_identical(old_rev_id, new_rev_id, parent_idxs)
163
self.put_weave(file_id, w, transaction)
165
def copy_multi(self, from_store, file_ids):
241
166
assert isinstance(from_store, WeaveStore)
242
if from_transaction is None:
243
warn("WeaveStore.copy_multi without a from_transaction parameter "
244
"is deprecated. Please provide a from_transaction.",
247
# we are reading one object - caching is irrelevant.
248
from_transaction = PassThroughTransaction()
249
if to_transaction is None:
250
warn("WeaveStore.copy_multi without a to_transaction parameter "
251
"is deprecated. Please provide a to_transaction.",
254
# we are copying single objects, and there may be open tranasactions
255
# so again with the passthrough
256
to_transaction = PassThroughTransaction()
257
pb = bzrlib.ui.ui_factory.nested_progress_bar()
259
for count, f in enumerate(file_ids):
260
mutter("copy weave {%s} into %s", f, self)
261
pb.update('copy', count, len(file_ids))
262
# if we have it in cache, its faster.
263
# joining is fast with knits, and bearable for weaves -
264
# indeed the new case can be optimised if needed.
265
target = self._make_new_versionedfile(f, to_transaction)
266
target.join(from_store.get_weave(f, from_transaction))
270
def total_size(self):
271
count, bytes = super(VersionedFileStore, self).total_size()
272
return (count / len(self._versionedfile_class.get_suffixes())), bytes
274
WeaveStore = VersionedFileStore
168
mutter("copy weave {%s} into %s", f, self)
169
self._put(f, from_store._get(f))
added
removed
bzrlib/store/weave.py
