23
25
from cStringIO import StringIO
25
from warnings import warn
31
28
from bzrlib.weavefile import read_weave, write_weave_v5
32
from bzrlib.weave import WeaveFile, Weave
33
from bzrlib.store import TransportStore
29
from bzrlib.weave import Weave
30
from bzrlib.store import TransportStore, hash_prefix
34
31
from bzrlib.atomicfile import AtomicFile
35
from bzrlib.symbol_versioning import (deprecated_method,
32
from bzrlib.errors import NoSuchFile, FileExists
37
33
from bzrlib.trace import mutter
41
class VersionedFileStore(TransportStore):
42
"""Collection of many versioned files in a transport."""
44
# TODO: Rather than passing versionedfile_kwargs, perhaps pass in a
45
# transport factory callable?
46
def __init__(self, transport, prefixed=False, precious=False,
47
dir_mode=None, file_mode=None,
48
versionedfile_class=WeaveFile,
49
versionedfile_kwargs={},
51
super(VersionedFileStore, self).__init__(transport,
52
dir_mode=dir_mode, file_mode=file_mode,
53
prefixed=prefixed, compressed=False, escaped=escaped)
36
class WeaveStore(TransportStore):
37
"""Collection of several weave files in a directory.
39
This has some shortcuts for reading and writing them.
41
FILE_SUFFIX = '.weave'
43
def __init__(self, transport, prefixed=False, precious=False):
44
self._transport = transport
45
self._prefixed = prefixed
54
46
self._precious = precious
55
self._versionedfile_class = versionedfile_class
56
self._versionedfile_kwargs = versionedfile_kwargs
58
def _clear_cache_id(self, file_id, transaction):
59
"""WARNING may lead to inconsistent object references for file_id.
61
Remove file_id from the transaction map.
63
NOT in the transaction api because theres no reliable way to clear
64
callers. So its here for very specialised use rather than having an
67
weave = transaction.map.find_weave(file_id)
69
mutter("old data in transaction in %s for %s", self, file_id)
70
# FIXME abstraction violation - transaction now has stale data.
71
transaction.map.remove_object(weave)
73
48
def filename(self, file_id):
74
49
"""Return the path relative to the transport root."""
75
file_id = osutils.safe_file_id(file_id)
76
return self._relpath(file_id)
51
return hash_prefix(file_id) + urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
53
return urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
78
55
def __iter__(self):
79
suffixes = self._versionedfile_class.get_suffixes()
56
l = len(WeaveStore.FILE_SUFFIX)
81
57
for relpath in self._iter_files_recursive():
82
for suffix in suffixes:
83
if relpath.endswith(suffix):
84
# TODO: use standard remove_suffix function
85
escaped_id = os.path.basename(relpath[:-len(suffix)])
86
file_id = self._unescape(escaped_id)
87
if file_id not in ids:
90
break # only one suffix can match
92
def has_id(self, file_id):
93
file_id = osutils.safe_file_id(file_id)
94
suffixes = self._versionedfile_class.get_suffixes()
95
filename = self.filename(file_id)
96
for suffix in suffixes:
97
if not self._transport.has(filename + suffix):
101
def get_empty(self, file_id, transaction):
102
"""Get an empty weave, which implies deleting the existing one first."""
103
file_id = osutils.safe_file_id(file_id)
104
if self.has_id(file_id):
105
self.delete(file_id, transaction)
106
return self.get_weave_or_empty(file_id, transaction)
108
def delete(self, file_id, transaction):
109
"""Remove file_id from the store."""
110
file_id = osutils.safe_file_id(file_id)
111
suffixes = self._versionedfile_class.get_suffixes()
112
filename = self.filename(file_id)
113
for suffix in suffixes:
114
self._transport.delete(filename + suffix)
115
self._clear_cache_id(file_id, transaction)
58
if relpath.endswith(WeaveStore.FILE_SUFFIX):
59
yield os.path.basename(relpath[:-l])
61
def has_id(self, fileid):
62
return self._transport.has(self.filename(fileid))
117
64
def _get(self, file_id):
118
65
return self._transport.get(self.filename(file_id))
120
67
def _put(self, file_id, f):
121
fn = self.filename(file_id)
123
return self._transport.put_file(fn, f, mode=self._file_mode)
124
except errors.NoSuchFile:
125
if not self._prefixed:
127
self._transport.mkdir(os.path.dirname(fn), mode=self._dir_mode)
128
return self._transport.put_file(fn, f, mode=self._file_mode)
130
def get_weave(self, file_id, transaction, _filename=None):
131
"""Return the VersionedFile for file_id.
133
:param _filename: filename that would be returned from self.filename for
134
file_id. This is used to reduce duplicate filename calculations when
135
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
137
file_id = osutils.safe_file_id(file_id)
70
self._transport.mkdir(hash_prefix(file_id))
73
return self._transport.put(self.filename(file_id), f)
75
def get_weave(self, file_id, transaction):
138
76
weave = transaction.map.find_weave(file_id)
139
if weave is not None:
140
#mutter("cache hit in %s for %s", self, file_id)
78
mutter("cache hit in %s for %s", self, file_id)
142
if _filename is None:
143
_filename = self.filename(file_id)
144
if transaction.writeable():
145
w = self._versionedfile_class(_filename, self._transport, self._file_mode,
146
**self._versionedfile_kwargs)
147
transaction.map.add_weave(file_id, w)
148
transaction.register_dirty(w)
150
w = self._versionedfile_class(_filename,
155
**self._versionedfile_kwargs)
156
transaction.map.add_weave(file_id, w)
157
transaction.register_clean(w, precious=self._precious)
80
w = read_weave(self._get(file_id))
81
transaction.map.add_weave(file_id, w)
82
transaction.register_clean(w, precious=self._precious)
160
@deprecated_method(zero_eight)
161
85
def get_lines(self, file_id, rev_id, transaction):
162
86
"""Return text from a particular version of a weave.
164
Returned as a list of lines.
166
file_id = osutils.safe_file_id(file_id)
88
Returned as a list of lines."""
167
89
w = self.get_weave(file_id, transaction)
168
return w.get_lines(rev_id)
90
return w.get(w.lookup(rev_id))
170
def _make_new_versionedfile(self, file_id, transaction,
171
known_missing=False, _filename=None):
172
"""Make a new versioned file.
174
:param _filename: filename that would be returned from self.filename for
175
file_id. This is used to reduce duplicate filename calculations when
176
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
178
if not known_missing and self.has_id(file_id):
179
self.delete(file_id, transaction)
180
if _filename is None:
181
_filename = self.filename(file_id)
183
# we try without making the directory first because thats optimising
184
# for the common case.
185
weave = self._versionedfile_class(_filename, self._transport, self._file_mode, create=True,
186
**self._versionedfile_kwargs)
187
except errors.NoSuchFile:
188
if not self._prefixed:
189
# unexpected error - NoSuchFile is expected to be raised on a
190
# missing dir only and that only occurs when we are prefixed.
192
self._transport.mkdir(self.hash_prefix(file_id), mode=self._dir_mode)
193
weave = self._versionedfile_class(_filename, self._transport,
194
self._file_mode, create=True,
195
**self._versionedfile_kwargs)
198
92
def get_weave_or_empty(self, file_id, transaction):
199
"""Return a weave, or an empty one if it doesn't exist."""
200
# This is typically used from 'commit' and 'fetch/push/pull' where
201
# we scan across many versioned files once. As such the small overhead
202
# of calculating the filename before doing a cache lookup is more than
203
# compensated for by not calculating the filename when making new
205
file_id = osutils.safe_file_id(file_id)
206
_filename = self.filename(file_id)
93
"""Return a weave, or an empty one if it doesn't exist."""
208
return self.get_weave(file_id, transaction, _filename=_filename)
209
except errors.NoSuchFile:
210
weave = self._make_new_versionedfile(file_id, transaction,
211
known_missing=True, _filename=_filename)
95
return self.get_weave(file_id, transaction)
97
weave = Weave(weave_name=file_id)
212
98
transaction.map.add_weave(file_id, weave)
213
# has to be dirty - its able to mutate on its own.
214
transaction.register_dirty(weave)
99
transaction.register_clean(weave, precious=self._precious)
217
@deprecated_method(zero_eight)
218
102
def put_weave(self, file_id, weave, transaction):
219
"""This is a deprecated API: It writes an entire collection of ids out.
221
This became inappropriate when we made a versioned file api which
222
tracks the state of the collection of versions for a single id.
224
Its maintained for backwards compatability but will only work on
225
weave stores - pre 0.8 repositories.
227
file_id = osutils.safe_file_id(file_id)
228
self._put_weave(file_id, weave, transaction)
230
def _put_weave(self, file_id, weave, transaction):
231
"""Preserved here for upgrades-to-weaves to use."""
232
myweave = self._make_new_versionedfile(file_id, transaction)
235
@deprecated_method(zero_eight)
103
"""Write back a modified weave"""
104
transaction.register_dirty(weave)
105
# TODO FOR WRITE TRANSACTIONS: this should be done in a callback
106
# from the transaction, when it decides to save.
108
write_weave_v5(weave, sio)
110
self._put(file_id, sio)
236
112
def add_text(self, file_id, rev_id, new_lines, parents, transaction):
237
"""This method was a shorthand for
239
vfile = self.get_weave_or_empty(file_id, transaction)
240
vfile.add_lines(rev_id, parents, new_lines)
242
file_id = osutils.safe_file_id(file_id)
243
vfile = self.get_weave_or_empty(file_id, transaction)
244
vfile.add_lines(rev_id, parents, new_lines)
113
w = self.get_weave_or_empty(file_id, transaction)
114
parent_idxs = map(w.lookup, parents)
115
w.add(rev_id, parent_idxs, new_lines)
116
self.put_weave(file_id, w, transaction)
246
@deprecated_method(zero_eight)
247
118
def add_identical_text(self, file_id, old_rev_id, new_rev_id, parents,
249
"""This method was a shorthand for
251
vfile = self.get_weave_or_empty(file_id, transaction)
252
vfile.clone_text(new_rev_id, old_rev_id, parents)
254
file_id = osutils.safe_file_id(file_id)
255
old_rev_id = osutils.safe_revision_id(old_rev_id)
256
new_rev_id = osutils.safe_revision_id(new_rev_id)
257
vfile = self.get_weave_or_empty(file_id, transaction)
258
vfile.clone_text(new_rev_id, old_rev_id, parents)
260
def copy(self, source, result_id, transaction):
261
"""Copy the source versioned file to result_id in this store."""
262
self._clear_cache_id(result_id, transaction)
263
source.copy_to(self.filename(result_id), self._transport)
265
def copy_all_ids(self, store_from, pb=None, from_transaction=None,
266
to_transaction=None):
267
"""Copy all the file ids from store_from into self."""
268
if from_transaction is None:
269
warn("Please pass from_transaction into "
270
"versioned_store.copy_all_ids.", stacklevel=2)
271
if to_transaction is None:
272
warn("Please pass to_transaction into "
273
"versioned_store.copy_all_ids.", stacklevel=2)
274
if not store_from.listable():
275
raise errors.UnlistableStore(store_from)
277
for count, file_id in enumerate(store_from):
279
pb.update('listing files', count, count)
283
mutter('copy_all ids: %r', ids)
284
self.copy_multi(store_from, ids, pb=pb,
285
from_transaction=from_transaction,
286
to_transaction=to_transaction)
288
def copy_multi(self, from_store, file_ids, pb=None, from_transaction=None,
289
to_transaction=None):
290
"""Copy all the versions for multiple file_ids from from_store.
292
:param from_transaction: required current transaction in from_store.
294
from bzrlib.transactions import PassThroughTransaction
120
w = self.get_weave_or_empty(file_id, transaction)
121
parent_idxs = map(w.lookup, parents)
122
w.add_identical(old_rev_id, new_rev_id, parent_idxs)
123
self.put_weave(file_id, w, transaction)
125
def copy_multi(self, from_store, file_ids):
295
126
assert isinstance(from_store, WeaveStore)
296
if from_transaction is None:
297
warn("WeaveStore.copy_multi without a from_transaction parameter "
298
"is deprecated. Please provide a from_transaction.",
301
# we are reading one object - caching is irrelevant.
302
from_transaction = PassThroughTransaction()
303
if to_transaction is None:
304
warn("WeaveStore.copy_multi without a to_transaction parameter "
305
"is deprecated. Please provide a to_transaction.",
308
# we are copying single objects, and there may be open tranasactions
309
# so again with the passthrough
310
to_transaction = PassThroughTransaction()
311
pb = bzrlib.ui.ui_factory.nested_progress_bar()
313
file_ids = [osutils.safe_file_id(f) for f in file_ids]
314
for count, f in enumerate(file_ids):
315
mutter("copy weave {%s} into %s", f, self)
316
pb.update('copy', count, len(file_ids))
317
# if we have it in cache, its faster.
318
# joining is fast with knits, and bearable for weaves -
319
# indeed the new case can be optimised if needed.
320
target = self._make_new_versionedfile(f, to_transaction)
321
target.join(from_store.get_weave(f, from_transaction))
325
def total_size(self):
326
count, bytes = super(VersionedFileStore, self).total_size()
327
return (count / len(self._versionedfile_class.get_suffixes())), bytes
329
WeaveStore = VersionedFileStore
128
mutter("copy weave {%s} into %s", f, self)
129
self._put(f, from_store._get(f))
added
removed
bzrlib/store/weave.py
