23
25
from cStringIO import StringIO
25
from warnings import warn
27
from bzrlib import errors
28
28
from bzrlib.weavefile import read_weave, write_weave_v5
29
from bzrlib.weave import WeaveFile, Weave
30
from bzrlib.store import TransportStore
29
from bzrlib.weave import Weave
30
from bzrlib.store import TransportStore, hash_prefix
31
31
from bzrlib.atomicfile import AtomicFile
32
from bzrlib.symbol_versioning import (deprecated_method,
32
from bzrlib.errors import NoSuchFile, FileExists
34
33
from bzrlib.trace import mutter
38
class VersionedFileStore(TransportStore):
39
"""Collection of many versioned files in a transport."""
41
# TODO: Rather than passing versionedfile_kwargs, perhaps pass in a
42
# transport factory callable?
43
def __init__(self, transport, prefixed=False, precious=False,
44
dir_mode=None, file_mode=None,
45
versionedfile_class=WeaveFile,
46
versionedfile_kwargs={},
48
super(VersionedFileStore, self).__init__(transport,
49
dir_mode=dir_mode, file_mode=file_mode,
50
prefixed=prefixed, compressed=False, escaped=escaped)
36
class WeaveStore(TransportStore):
37
"""Collection of several weave files in a directory.
39
This has some shortcuts for reading and writing them.
41
FILE_SUFFIX = '.weave'
43
def __init__(self, transport, prefixed=False, precious=False):
44
self._transport = transport
45
self._prefixed = prefixed
51
46
self._precious = precious
52
self._versionedfile_class = versionedfile_class
53
self._versionedfile_kwargs = versionedfile_kwargs
55
def _clear_cache_id(self, file_id, transaction):
56
"""WARNING may lead to inconsistent object references for file_id.
58
Remove file_id from the transaction map.
60
NOT in the transaction api because theres no reliable way to clear
61
callers. So its here for very specialised use rather than having an
64
weave = transaction.map.find_weave(file_id)
66
mutter("old data in transaction in %s for %s", self, file_id)
67
# FIXME abstraction violation - transaction now has stale data.
68
transaction.map.remove_object(weave)
70
48
def filename(self, file_id):
71
49
"""Return the path relative to the transport root."""
72
return self._relpath(file_id)
51
return hash_prefix(file_id) + urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
53
return urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
74
55
def __iter__(self):
75
suffixes = self._versionedfile_class.get_suffixes()
56
l = len(WeaveStore.FILE_SUFFIX)
77
57
for relpath in self._iter_files_recursive():
78
for suffix in suffixes:
79
if relpath.endswith(suffix):
80
# TODO: use standard remove_suffix function
81
escaped_id = os.path.basename(relpath[:-len(suffix)])
82
file_id = self._unescape(escaped_id)
83
if file_id not in ids:
86
break # only one suffix can match
58
if relpath.endswith(WeaveStore.FILE_SUFFIX):
59
yield os.path.basename(relpath[:-l])
88
61
def has_id(self, fileid):
89
suffixes = self._versionedfile_class.get_suffixes()
90
filename = self.filename(fileid)
91
for suffix in suffixes:
92
if not self._transport.has(filename + suffix):
96
def get_empty(self, file_id, transaction):
97
"""Get an empty weave, which implies deleting the existing one first."""
98
if self.has_id(file_id):
99
self.delete(file_id, transaction)
100
return self.get_weave_or_empty(file_id, transaction)
102
def delete(self, file_id, transaction):
103
"""Remove file_id from the store."""
104
suffixes = self._versionedfile_class.get_suffixes()
105
filename = self.filename(file_id)
106
for suffix in suffixes:
107
self._transport.delete(filename + suffix)
108
self._clear_cache_id(file_id, transaction)
62
return self._transport.has(self.filename(fileid))
110
64
def _get(self, file_id):
111
65
return self._transport.get(self.filename(file_id))
113
67
def _put(self, file_id, f):
114
fn = self.filename(file_id)
116
return self._transport.put(fn, f, mode=self._file_mode)
117
except errors.NoSuchFile:
118
if not self._prefixed:
120
self._transport.mkdir(os.path.dirname(fn), mode=self._dir_mode)
121
return self._transport.put(fn, f, mode=self._file_mode)
123
def get_weave(self, file_id, transaction, _filename=None):
124
"""Return the VersionedFile for file_id.
126
:param _filename: filename that would be returned from self.filename for
127
file_id. This is used to reduce duplicate filename calculations when
128
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
70
self._transport.mkdir(hash_prefix(file_id))
73
return self._transport.put(self.filename(file_id), f)
75
def get_weave(self, file_id, transaction):
130
76
weave = transaction.map.find_weave(file_id)
131
if weave is not None:
132
#mutter("cache hit in %s for %s", self, file_id)
78
mutter("cache hit in %s for %s", self, file_id)
134
if _filename is None:
135
_filename = self.filename(file_id)
136
if transaction.writeable():
137
w = self._versionedfile_class(_filename, self._transport, self._file_mode,
138
**self._versionedfile_kwargs)
139
transaction.map.add_weave(file_id, w)
140
transaction.register_dirty(w)
142
w = self._versionedfile_class(_filename,
147
**self._versionedfile_kwargs)
148
transaction.map.add_weave(file_id, w)
149
transaction.register_clean(w, precious=self._precious)
80
w = read_weave(self._get(file_id))
81
transaction.map.add_weave(file_id, w)
82
transaction.register_clean(w, precious=self._precious)
152
@deprecated_method(zero_eight)
153
85
def get_lines(self, file_id, rev_id, transaction):
154
86
"""Return text from a particular version of a weave.
156
Returned as a list of lines.
88
Returned as a list of lines."""
158
89
w = self.get_weave(file_id, transaction)
159
return w.get_lines(rev_id)
90
return w.get(w.lookup(rev_id))
161
def _make_new_versionedfile(self, file_id, transaction,
162
known_missing=False, _filename=None):
163
"""Make a new versioned file.
165
:param _filename: filename that would be returned from self.filename for
166
file_id. This is used to reduce duplicate filename calculations when
167
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
169
if not known_missing and self.has_id(file_id):
170
self.delete(file_id, transaction)
171
if _filename is None:
172
_filename = self.filename(file_id)
174
# we try without making the directory first because thats optimising
175
# for the common case.
176
weave = self._versionedfile_class(_filename, self._transport, self._file_mode, create=True,
177
**self._versionedfile_kwargs)
178
except errors.NoSuchFile:
179
if not self._prefixed:
180
# unexpected error - NoSuchFile is expected to be raised on a
181
# missing dir only and that only occurs when we are prefixed.
183
self._transport.mkdir(self.hash_prefix(file_id), mode=self._dir_mode)
184
weave = self._versionedfile_class(_filename, self._transport,
185
self._file_mode, create=True,
186
**self._versionedfile_kwargs)
189
92
def get_weave_or_empty(self, file_id, transaction):
190
"""Return a weave, or an empty one if it doesn't exist."""
191
# This is typically used from 'commit' and 'fetch/push/pull' where
192
# we scan across many versioned files once. As such the small overhead
193
# of calculating the filename before doing a cache lookup is more than
194
# compensated for by not calculating the filename when making new
196
_filename = self.filename(file_id)
93
"""Return a weave, or an empty one if it doesn't exist."""
198
return self.get_weave(file_id, transaction, _filename=_filename)
199
except errors.NoSuchFile:
200
weave = self._make_new_versionedfile(file_id, transaction,
201
known_missing=True, _filename=_filename)
95
return self.get_weave(file_id, transaction)
97
weave = Weave(weave_name=file_id)
202
98
transaction.map.add_weave(file_id, weave)
203
# has to be dirty - its able to mutate on its own.
204
transaction.register_dirty(weave)
99
transaction.register_clean(weave, precious=self._precious)
207
@deprecated_method(zero_eight)
208
102
def put_weave(self, file_id, weave, transaction):
209
"""This is a deprecated API: It writes an entire collection of ids out.
211
This became inappropriate when we made a versioned file api which
212
tracks the state of the collection of versions for a single id.
214
Its maintained for backwards compatability but will only work on
215
weave stores - pre 0.8 repositories.
217
self._put_weave(file_id, weave, transaction)
219
def _put_weave(self, file_id, weave, transaction):
220
"""Preserved here for upgrades-to-weaves to use."""
221
myweave = self._make_new_versionedfile(file_id, transaction)
224
@deprecated_method(zero_eight)
103
"""Write back a modified weave"""
104
transaction.register_dirty(weave)
105
# TODO FOR WRITE TRANSACTIONS: this should be done in a callback
106
# from the transaction, when it decides to save.
108
write_weave_v5(weave, sio)
110
self._put(file_id, sio)
225
112
def add_text(self, file_id, rev_id, new_lines, parents, transaction):
226
"""This method was a shorthand for
228
vfile = self.get_weave_or_empty(file_id, transaction)
229
vfile.add_lines(rev_id, parents, new_lines)
231
vfile = self.get_weave_or_empty(file_id, transaction)
232
vfile.add_lines(rev_id, parents, new_lines)
113
w = self.get_weave_or_empty(file_id, transaction)
114
parent_idxs = map(w.lookup, parents)
115
w.add(rev_id, parent_idxs, new_lines)
116
self.put_weave(file_id, w, transaction)
234
@deprecated_method(zero_eight)
235
118
def add_identical_text(self, file_id, old_rev_id, new_rev_id, parents,
237
"""This method was a shorthand for
239
vfile = self.get_weave_or_empty(file_id, transaction)
240
vfile.clone_text(new_rev_id, old_rev_id, parents)
242
vfile = self.get_weave_or_empty(file_id, transaction)
243
vfile.clone_text(new_rev_id, old_rev_id, parents)
245
def copy(self, source, result_id, transaction):
246
"""Copy the source versioned file to result_id in this store."""
247
self._clear_cache_id(result_id, transaction)
248
source.copy_to(self.filename(result_id), self._transport)
250
def copy_all_ids(self, store_from, pb=None, from_transaction=None,
251
to_transaction=None):
252
"""Copy all the file ids from store_from into self."""
253
if from_transaction is None:
254
warn("Please pass from_transaction into "
255
"versioned_store.copy_all_ids.", stacklevel=2)
256
if to_transaction is None:
257
warn("Please pass to_transaction into "
258
"versioned_store.copy_all_ids.", stacklevel=2)
259
if not store_from.listable():
260
raise errors.UnlistableStore(store_from)
262
for count, file_id in enumerate(store_from):
264
pb.update('listing files', count, count)
268
mutter('copy_all ids: %r', ids)
269
self.copy_multi(store_from, ids, pb=pb,
270
from_transaction=from_transaction,
271
to_transaction=to_transaction)
273
def copy_multi(self, from_store, file_ids, pb=None, from_transaction=None,
274
to_transaction=None):
275
"""Copy all the versions for multiple file_ids from from_store.
277
:param from_transaction: required current transaction in from_store.
279
from bzrlib.transactions import PassThroughTransaction
120
w = self.get_weave_or_empty(file_id, transaction)
121
parent_idxs = map(w.lookup, parents)
122
w.add_identical(old_rev_id, new_rev_id, parent_idxs)
123
self.put_weave(file_id, w, transaction)
125
def copy_multi(self, from_store, file_ids):
280
126
assert isinstance(from_store, WeaveStore)
281
if from_transaction is None:
282
warn("WeaveStore.copy_multi without a from_transaction parameter "
283
"is deprecated. Please provide a from_transaction.",
286
# we are reading one object - caching is irrelevant.
287
from_transaction = PassThroughTransaction()
288
if to_transaction is None:
289
warn("WeaveStore.copy_multi without a to_transaction parameter "
290
"is deprecated. Please provide a to_transaction.",
293
# we are copying single objects, and there may be open tranasactions
294
# so again with the passthrough
295
to_transaction = PassThroughTransaction()
296
pb = bzrlib.ui.ui_factory.nested_progress_bar()
297
for count, f in enumerate(file_ids):
298
128
mutter("copy weave {%s} into %s", f, self)
299
pb.update('copy', count, len(file_ids))
300
# if we have it in cache, its faster.
301
# joining is fast with knits, and bearable for weaves -
302
# indeed the new case can be optimised if needed.
303
target = self._make_new_versionedfile(f, to_transaction)
304
target.join(from_store.get_weave(f, from_transaction))
307
def total_size(self):
308
count, bytes = super(VersionedFileStore, self).total_size()
309
return (count / len(self._versionedfile_class.get_suffixes())), bytes
311
WeaveStore = VersionedFileStore
129
self._put(f, from_store._get(f))
added
removed
bzrlib/store/weave.py
