23
25
from cStringIO import StringIO
25
from warnings import warn
27
from bzrlib import errors
28
28
from bzrlib.weavefile import read_weave, write_weave_v5
29
from bzrlib.weave import WeaveFile, Weave
30
from bzrlib.store import TransportStore
29
from bzrlib.weave import Weave
30
from bzrlib.store import TransportStore, hash_prefix
31
31
from bzrlib.atomicfile import AtomicFile
32
from bzrlib.symbol_versioning import (deprecated_method,
32
from bzrlib.errors import NoSuchFile, FileExists
34
33
from bzrlib.trace import mutter
38
class VersionedFileStore(TransportStore):
39
"""Collection of many versioned files in a transport."""
41
# TODO: Rather than passing versionedfile_kwargs, perhaps pass in a
42
# transport factory callable?
36
class WeaveStore(TransportStore):
37
"""Collection of several weave files in a directory.
39
This has some shortcuts for reading and writing them.
41
FILE_SUFFIX = '.weave'
43
43
def __init__(self, transport, prefixed=False, precious=False,
44
dir_mode=None, file_mode=None,
45
versionedfile_class=WeaveFile,
46
versionedfile_kwargs={},
48
super(VersionedFileStore, self).__init__(transport,
44
dir_mode=None, file_mode=None):
45
super(WeaveStore, self).__init__(transport,
49
46
dir_mode=dir_mode, file_mode=file_mode,
50
prefixed=prefixed, compressed=False, escaped=escaped)
47
prefixed=prefixed, compressed=False)
51
48
self._precious = precious
52
self._versionedfile_class = versionedfile_class
53
self._versionedfile_kwargs = versionedfile_kwargs
55
def _clear_cache_id(self, file_id, transaction):
56
"""WARNING may lead to inconsistent object references for file_id.
58
Remove file_id from the transaction map.
60
NOT in the transaction api because theres no reliable way to clear
61
callers. So its here for very specialised use rather than having an
64
weave = transaction.map.find_weave(file_id)
66
mutter("old data in transaction in %s for %s", self, file_id)
67
# FIXME abstraction violation - transaction now has stale data.
68
transaction.map.remove_object(weave)
70
50
def filename(self, file_id):
71
51
"""Return the path relative to the transport root."""
72
return self._relpath(file_id)
53
return hash_prefix(file_id) + urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
55
return urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
74
57
def __iter__(self):
75
suffixes = self._versionedfile_class.get_suffixes()
58
l = len(WeaveStore.FILE_SUFFIX)
77
59
for relpath in self._iter_files_recursive():
78
for suffix in suffixes:
79
if relpath.endswith(suffix):
80
# TODO: use standard remove_suffix function
81
escaped_id = os.path.basename(relpath[:-len(suffix)])
82
file_id = self._unescape(escaped_id)
83
if file_id not in ids:
86
break # only one suffix can match
60
if relpath.endswith(WeaveStore.FILE_SUFFIX):
61
yield os.path.basename(relpath[:-l])
88
63
def has_id(self, fileid):
89
suffixes = self._versionedfile_class.get_suffixes()
90
filename = self.filename(fileid)
91
for suffix in suffixes:
92
if not self._transport.has(filename + suffix):
96
def get_empty(self, file_id, transaction):
97
"""Get an empty weave, which implies deleting the existing one first."""
98
if self.has_id(file_id):
99
self.delete(file_id, transaction)
100
return self.get_weave_or_empty(file_id, transaction)
102
def delete(self, file_id, transaction):
103
"""Remove file_id from the store."""
104
suffixes = self._versionedfile_class.get_suffixes()
105
filename = self.filename(file_id)
106
for suffix in suffixes:
107
self._transport.delete(filename + suffix)
108
self._clear_cache_id(file_id, transaction)
64
return self._transport.has(self.filename(fileid))
110
66
def _get(self, file_id):
111
67
return self._transport.get(self.filename(file_id))
113
69
def _put(self, file_id, f):
114
fn = self.filename(file_id)
116
return self._transport.put(fn, f, mode=self._file_mode)
117
except errors.NoSuchFile:
118
if not self._prefixed:
120
self._transport.mkdir(os.path.dirname(fn), mode=self._dir_mode)
121
return self._transport.put(fn, f, mode=self._file_mode)
123
def get_weave(self, file_id, transaction, _filename=None):
124
"""Return the VersionedFile for file_id.
126
:param _filename: filename that would be returned from self.filename for
127
file_id. This is used to reduce duplicate filename calculations when
128
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
72
self._transport.mkdir(hash_prefix(file_id), mode=self._dir_mode)
75
return self._transport.put(self.filename(file_id), f, mode=self._file_mode)
77
def get_weave(self, file_id, transaction):
130
78
weave = transaction.map.find_weave(file_id)
131
if weave is not None:
132
#mutter("cache hit in %s for %s", self, file_id)
134
if _filename is None:
135
_filename = self.filename(file_id)
136
if transaction.writeable():
137
w = self._versionedfile_class(_filename, self._transport, self._file_mode,
138
**self._versionedfile_kwargs)
139
transaction.map.add_weave(file_id, w)
140
transaction.register_dirty(w)
142
w = self._versionedfile_class(_filename,
147
**self._versionedfile_kwargs)
148
transaction.map.add_weave(file_id, w)
149
transaction.register_clean(w, precious=self._precious)
152
@deprecated_method(zero_eight)
80
mutter("cache hit in %s for %s", self, file_id)
82
w = read_weave(self._get(file_id))
83
transaction.map.add_weave(file_id, w)
84
transaction.register_clean(w, precious=self._precious)
85
# TODO: jam 20051219 This should check if there is a prelude
86
# which is already cached, and if so, should remove it
87
# But transaction doesn't seem to have a 'remove'
88
# One workaround would be to re-add the object with
92
def get_weave_prelude(self, file_id, transaction):
94
weave = transaction.map.find_weave(weave_id)
96
mutter("cache hit in %s for %s", self, weave_id)
98
# We want transactions to also cache preludes if that
99
# is all that we are loading. So we need a unique
100
# identifier, so that someone who wants the whole text
101
# won't get just the prelude
102
weave_id = 'PRELUDE-' + file_id
103
weave = transaction.map.find_weave(weave_id)
105
mutter("cache hit in %s for %s", self, weave_id)
107
w = read_weave(self._get(file_id), prelude=True)
108
transaction.map.add_weave(weave_id, w)
109
transaction.register_clean(w, precious=self._precious)
153
112
def get_lines(self, file_id, rev_id, transaction):
154
113
"""Return text from a particular version of a weave.
156
Returned as a list of lines.
115
Returned as a list of lines."""
158
116
w = self.get_weave(file_id, transaction)
159
return w.get_lines(rev_id)
117
return w.get(w.lookup(rev_id))
161
def _make_new_versionedfile(self, file_id, transaction,
162
known_missing=False, _filename=None):
163
"""Make a new versioned file.
165
:param _filename: filename that would be returned from self.filename for
166
file_id. This is used to reduce duplicate filename calculations when
167
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
119
def get_weave_prelude_or_empty(self, file_id, transaction):
120
"""cheap version that reads the prelude but not the lines
169
if not known_missing and self.has_id(file_id):
170
self.delete(file_id, transaction)
171
if _filename is None:
172
_filename = self.filename(file_id)
174
# we try without making the directory first because thats optimising
175
# for the common case.
176
weave = self._versionedfile_class(_filename, self._transport, self._file_mode, create=True,
177
**self._versionedfile_kwargs)
178
except errors.NoSuchFile:
179
if not self._prefixed:
180
# unexpected error - NoSuchFile is expected to be raised on a
181
# missing dir only and that only occurs when we are prefixed.
183
self._transport.mkdir(self.hash_prefix(file_id), mode=self._dir_mode)
184
weave = self._versionedfile_class(_filename, self._transport,
185
self._file_mode, create=True,
186
**self._versionedfile_kwargs)
123
return self.get_weave_prelude(file_id, transaction)
125
# We can cache here, because we know that there
126
# is no complete object, since we got NoSuchFile
127
weave = Weave(weave_name=file_id)
128
transaction.map.add_weave(file_id, weave)
129
transaction.register_clean(weave, precious=self._precious)
189
132
def get_weave_or_empty(self, file_id, transaction):
190
"""Return a weave, or an empty one if it doesn't exist."""
191
# This is typically used from 'commit' and 'fetch/push/pull' where
192
# we scan across many versioned files once. As such the small overhead
193
# of calculating the filename before doing a cache lookup is more than
194
# compensated for by not calculating the filename when making new
196
_filename = self.filename(file_id)
133
"""Return a weave, or an empty one if it doesn't exist."""
198
return self.get_weave(file_id, transaction, _filename=_filename)
199
except errors.NoSuchFile:
200
weave = self._make_new_versionedfile(file_id, transaction,
201
known_missing=True, _filename=_filename)
135
return self.get_weave(file_id, transaction)
137
weave = Weave(weave_name=file_id)
202
138
transaction.map.add_weave(file_id, weave)
203
# has to be dirty - its able to mutate on its own.
204
transaction.register_dirty(weave)
139
transaction.register_clean(weave, precious=self._precious)
207
@deprecated_method(zero_eight)
208
142
def put_weave(self, file_id, weave, transaction):
209
"""This is a deprecated API: It writes an entire collection of ids out.
211
This became inappropriate when we made a versioned file api which
212
tracks the state of the collection of versions for a single id.
214
Its maintained for backwards compatability but will only work on
215
weave stores - pre 0.8 repositories.
217
self._put_weave(file_id, weave, transaction)
219
def _put_weave(self, file_id, weave, transaction):
220
"""Preserved here for upgrades-to-weaves to use."""
221
myweave = self._make_new_versionedfile(file_id, transaction)
224
@deprecated_method(zero_eight)
143
"""Write back a modified weave"""
144
transaction.register_dirty(weave)
145
# TODO FOR WRITE TRANSACTIONS: this should be done in a callback
146
# from the transaction, when it decides to save.
148
write_weave_v5(weave, sio)
150
self._put(file_id, sio)
225
152
def add_text(self, file_id, rev_id, new_lines, parents, transaction):
226
"""This method was a shorthand for
228
vfile = self.get_weave_or_empty(file_id, transaction)
229
vfile.add_lines(rev_id, parents, new_lines)
231
vfile = self.get_weave_or_empty(file_id, transaction)
232
vfile.add_lines(rev_id, parents, new_lines)
153
w = self.get_weave_or_empty(file_id, transaction)
154
parent_idxs = map(w.lookup, parents)
155
w.add(rev_id, parent_idxs, new_lines)
156
self.put_weave(file_id, w, transaction)
234
@deprecated_method(zero_eight)
235
158
def add_identical_text(self, file_id, old_rev_id, new_rev_id, parents,
237
"""This method was a shorthand for
239
vfile = self.get_weave_or_empty(file_id, transaction)
240
vfile.clone_text(new_rev_id, old_rev_id, parents)
242
vfile = self.get_weave_or_empty(file_id, transaction)
243
vfile.clone_text(new_rev_id, old_rev_id, parents)
245
def copy(self, source, result_id, transaction):
246
"""Copy the source versioned file to result_id in this store."""
247
self._clear_cache_id(result_id, transaction)
248
source.copy_to(self.filename(result_id), self._transport)
250
def copy_all_ids(self, store_from, pb=None, from_transaction=None,
251
to_transaction=None):
252
"""Copy all the file ids from store_from into self."""
253
if from_transaction is None:
254
warn("Please pass from_transaction into "
255
"versioned_store.copy_all_ids.", stacklevel=2)
256
if to_transaction is None:
257
warn("Please pass to_transaction into "
258
"versioned_store.copy_all_ids.", stacklevel=2)
259
if not store_from.listable():
260
raise errors.UnlistableStore(store_from)
262
for count, file_id in enumerate(store_from):
264
pb.update('listing files', count, count)
268
mutter('copy_all ids: %r', ids)
269
self.copy_multi(store_from, ids, pb=pb,
270
from_transaction=from_transaction,
271
to_transaction=to_transaction)
273
def copy_multi(self, from_store, file_ids, pb=None, from_transaction=None,
274
to_transaction=None):
275
"""Copy all the versions for multiple file_ids from from_store.
277
:param from_transaction: required current transaction in from_store.
279
from bzrlib.transactions import PassThroughTransaction
160
w = self.get_weave_or_empty(file_id, transaction)
161
parent_idxs = map(w.lookup, parents)
162
w.add_identical(old_rev_id, new_rev_id, parent_idxs)
163
self.put_weave(file_id, w, transaction)
165
def copy_multi(self, from_store, file_ids):
280
166
assert isinstance(from_store, WeaveStore)
281
if from_transaction is None:
282
warn("WeaveStore.copy_multi without a from_transaction parameter "
283
"is deprecated. Please provide a from_transaction.",
286
# we are reading one object - caching is irrelevant.
287
from_transaction = PassThroughTransaction()
288
if to_transaction is None:
289
warn("WeaveStore.copy_multi without a to_transaction parameter "
290
"is deprecated. Please provide a to_transaction.",
293
# we are copying single objects, and there may be open tranasactions
294
# so again with the passthrough
295
to_transaction = PassThroughTransaction()
296
pb = bzrlib.ui.ui_factory.nested_progress_bar()
297
for count, f in enumerate(file_ids):
298
168
mutter("copy weave {%s} into %s", f, self)
299
pb.update('copy', count, len(file_ids))
300
# if we have it in cache, its faster.
301
# joining is fast with knits, and bearable for weaves -
302
# indeed the new case can be optimised if needed.
303
target = self._make_new_versionedfile(f, to_transaction)
304
target.join(from_store.get_weave(f, from_transaction))
307
def total_size(self):
308
count, bytes = super(VersionedFileStore, self).total_size()
309
return (count / len(self._versionedfile_class.get_suffixes())), bytes
311
WeaveStore = VersionedFileStore
169
self._put(f, from_store._get(f))
added
removed
bzrlib/store/weave.py
