23
25
from cStringIO import StringIO
24
from warnings import warn
30
28
from bzrlib.weavefile import read_weave, write_weave_v5
31
from bzrlib.weave import WeaveFile, Weave
32
from bzrlib.store import TransportStore
29
from bzrlib.weave import Weave
30
from bzrlib.store import TransportStore, hash_prefix
33
31
from bzrlib.atomicfile import AtomicFile
34
from bzrlib.symbol_versioning import (deprecated_method,
32
from bzrlib.errors import NoSuchFile, FileExists
36
33
from bzrlib.trace import mutter
40
class VersionedFileStore(TransportStore):
41
"""Collection of many versioned files in a transport."""
43
# TODO: Rather than passing versionedfile_kwargs, perhaps pass in a
44
# transport factory callable?
36
class WeaveStore(TransportStore):
37
"""Collection of several weave files in a directory.
39
This has some shortcuts for reading and writing them.
41
FILE_SUFFIX = '.weave'
45
43
def __init__(self, transport, prefixed=False, precious=False,
46
dir_mode=None, file_mode=None,
47
versionedfile_class=WeaveFile,
48
versionedfile_kwargs={},
50
super(VersionedFileStore, self).__init__(transport,
44
dir_mode=None, file_mode=None):
45
super(WeaveStore, self).__init__(transport,
51
46
dir_mode=dir_mode, file_mode=file_mode,
52
prefixed=prefixed, compressed=False, escaped=escaped)
47
prefixed=prefixed, compressed=False)
53
48
self._precious = precious
54
self._versionedfile_class = versionedfile_class
55
self._versionedfile_kwargs = versionedfile_kwargs
56
# Used for passing get_scope to versioned file constructors;
59
50
def filename(self, file_id):
60
51
"""Return the path relative to the transport root."""
61
return self._relpath(file_id)
53
return hash_prefix(file_id) + urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
55
return urllib.quote(file_id) + WeaveStore.FILE_SUFFIX
63
57
def __iter__(self):
64
suffixes = self._versionedfile_class.get_suffixes()
58
l = len(WeaveStore.FILE_SUFFIX)
66
59
for relpath in self._iter_files_recursive():
67
for suffix in suffixes:
68
if relpath.endswith(suffix):
69
# TODO: use standard remove_suffix function
70
escaped_id = os.path.basename(relpath[:-len(suffix)])
71
file_id = self._mapper.unmap(escaped_id)[0]
72
if file_id not in ids:
75
break # only one suffix can match
77
def has_id(self, file_id):
78
suffixes = self._versionedfile_class.get_suffixes()
79
filename = self.filename(file_id)
80
for suffix in suffixes:
81
if not self._transport.has(filename + suffix):
85
def get_empty(self, file_id, transaction):
86
"""Get an empty weave, which implies deleting the existing one first."""
87
if self.has_id(file_id):
88
self.delete(file_id, transaction)
89
return self.get_weave_or_empty(file_id, transaction)
91
def delete(self, file_id, transaction):
92
"""Remove file_id from the store."""
93
suffixes = self._versionedfile_class.get_suffixes()
94
filename = self.filename(file_id)
95
for suffix in suffixes:
96
self._transport.delete(filename + suffix)
60
if relpath.endswith(WeaveStore.FILE_SUFFIX):
61
yield os.path.basename(relpath[:-l])
63
def has_id(self, fileid):
64
return self._transport.has(self.filename(fileid))
98
66
def _get(self, file_id):
99
67
return self._transport.get(self.filename(file_id))
101
69
def _put(self, file_id, f):
102
fn = self.filename(file_id)
104
return self._transport.put_file(fn, f, mode=self._file_mode)
105
except errors.NoSuchFile:
106
if not self._prefixed:
108
self._transport.mkdir(os.path.dirname(fn), mode=self._dir_mode)
109
return self._transport.put_file(fn, f, mode=self._file_mode)
111
def get_weave(self, file_id, transaction, _filename=None):
112
"""Return the VersionedFile for file_id.
114
:param _filename: filename that would be returned from self.filename for
115
file_id. This is used to reduce duplicate filename calculations when
116
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
118
if _filename is None:
119
_filename = self.filename(file_id)
120
if transaction.writeable():
121
w = self._versionedfile_class(_filename, self._transport, self._file_mode,
122
get_scope=self.get_scope, **self._versionedfile_kwargs)
124
w = self._versionedfile_class(_filename,
129
get_scope=self.get_scope,
130
**self._versionedfile_kwargs)
133
def _make_new_versionedfile(self, file_id, transaction,
134
known_missing=False, _filename=None):
135
"""Make a new versioned file.
137
:param _filename: filename that would be returned from self.filename for
138
file_id. This is used to reduce duplicate filename calculations when
139
using 'get_weave_or_empty'. FOR INTERNAL USE ONLY.
141
if not known_missing and self.has_id(file_id):
142
self.delete(file_id, transaction)
143
if _filename is None:
144
_filename = self.filename(file_id)
146
# we try without making the directory first because thats optimising
147
# for the common case.
148
weave = self._versionedfile_class(_filename, self._transport, self._file_mode, create=True,
149
get_scope=self.get_scope, **self._versionedfile_kwargs)
150
except errors.NoSuchFile:
151
if not self._prefixed:
152
# unexpected error - NoSuchFile is expected to be raised on a
153
# missing dir only and that only occurs when we are prefixed.
155
dirname = osutils.dirname(_filename)
156
self._transport.mkdir(dirname, mode=self._dir_mode)
157
weave = self._versionedfile_class(_filename, self._transport,
158
self._file_mode, create=True,
159
get_scope=self.get_scope,
160
**self._versionedfile_kwargs)
70
# less round trips to mkdir on failure than mkdir always
72
return self._transport.put(self.filename(file_id), f, mode=self._file_mode)
74
if not self._prefixed:
76
self._transport.mkdir(hash_prefix(file_id), mode=self._dir_mode)
77
return self._transport.put(self.filename(file_id), f, mode=self._file_mode)
79
def get_weave(self, file_id, transaction):
80
weave = transaction.map.find_weave(file_id)
82
mutter("cache hit in %s for %s", self, file_id)
84
w = read_weave(self._get(file_id))
85
transaction.map.add_weave(file_id, w)
86
transaction.register_clean(w, precious=self._precious)
87
# TODO: jam 20051219 This should check if there is a prelude
88
# which is already cached, and if so, should remove it
89
# But transaction doesn't seem to have a 'remove'
90
# One workaround would be to re-add the object with
94
def get_weave_prelude(self, file_id, transaction):
96
weave = transaction.map.find_weave(weave_id)
98
mutter("cache hit in %s for %s", self, weave_id)
100
# We want transactions to also cache preludes if that
101
# is all that we are loading. So we need a unique
102
# identifier, so that someone who wants the whole text
103
# won't get just the prelude
104
weave_id = 'PRELUDE-' + file_id
105
weave = transaction.map.find_weave(weave_id)
107
mutter("cache hit in %s for %s", self, weave_id)
109
w = read_weave(self._get(file_id), prelude=True)
110
transaction.map.add_weave(weave_id, w)
111
transaction.register_clean(w, precious=self._precious)
114
def get_lines(self, file_id, rev_id, transaction):
115
"""Return text from a particular version of a weave.
117
Returned as a list of lines."""
118
w = self.get_weave(file_id, transaction)
119
return w.get(w.lookup(rev_id))
121
def get_weave_prelude_or_empty(self, file_id, transaction):
122
"""cheap version that reads the prelude but not the lines
125
return self.get_weave_prelude(file_id, transaction)
127
# We can cache here, because we know that there
128
# is no complete object, since we got NoSuchFile
129
weave = Weave(weave_name=file_id)
130
transaction.map.add_weave(file_id, weave)
131
transaction.register_clean(weave, precious=self._precious)
163
134
def get_weave_or_empty(self, file_id, transaction):
164
"""Return a weave, or an empty one if it doesn't exist."""
165
# This is typically used from 'commit' and 'fetch/push/pull' where
166
# we scan across many versioned files once. As such the small overhead
167
# of calculating the filename before doing a cache lookup is more than
168
# compensated for by not calculating the filename when making new
170
_filename = self.filename(file_id)
135
"""Return a weave, or an empty one if it doesn't exist."""
172
return self.get_weave(file_id, transaction, _filename=_filename)
173
except errors.NoSuchFile:
174
weave = self._make_new_versionedfile(file_id, transaction,
175
known_missing=True, _filename=_filename)
137
return self.get_weave(file_id, transaction)
139
weave = Weave(weave_name=file_id)
140
transaction.map.add_weave(file_id, weave)
141
transaction.register_clean(weave, precious=self._precious)
178
def _put_weave(self, file_id, weave, transaction):
179
"""Preserved here for upgrades-to-weaves to use."""
180
myweave = self._make_new_versionedfile(file_id, transaction)
181
myweave.insert_record_stream(weave.get_record_stream(
182
[(version,) for version in weave.versions()],
183
'topological', False))
144
def put_weave(self, file_id, weave, transaction):
145
"""Write back a modified weave"""
146
transaction.register_dirty(weave)
147
# TODO FOR WRITE TRANSACTIONS: this should be done in a callback
148
# from the transaction, when it decides to save.
150
write_weave_v5(weave, sio)
152
self._put(file_id, sio)
185
def copy_all_ids(self, store_from, pb=None, from_transaction=None,
186
to_transaction=None):
187
"""Copy all the file ids from store_from into self."""
188
if from_transaction is None:
189
warn("Please pass from_transaction into "
190
"versioned_store.copy_all_ids.", stacklevel=2)
191
if to_transaction is None:
192
warn("Please pass to_transaction into "
193
"versioned_store.copy_all_ids.", stacklevel=2)
194
if not store_from.listable():
195
raise errors.UnlistableStore(store_from)
197
for count, file_id in enumerate(store_from):
154
def add_text(self, file_id, rev_id, new_lines, parents, transaction):
155
w = self.get_weave_or_empty(file_id, transaction)
156
parent_idxs = map(w.lookup, parents)
157
w.add(rev_id, parent_idxs, new_lines)
158
self.put_weave(file_id, w, transaction)
160
def add_identical_text(self, file_id, old_rev_id, new_rev_id, parents,
162
w = self.get_weave_or_empty(file_id, transaction)
163
parent_idxs = map(w.lookup, parents)
164
w.add_identical(old_rev_id, new_rev_id, parent_idxs)
165
self.put_weave(file_id, w, transaction)
167
def copy_multi(self, from_store, file_ids, pb=None):
168
assert isinstance(from_store, WeaveStore)
169
for count, f in enumerate(file_ids):
170
mutter("copy weave {%s} into %s", f, self)
199
pb.update('listing files', count, count)
172
pb.update('copy', count, len(file_ids))
173
self._put(f, from_store._get(f))
203
mutter('copy_all ids: %r', ids)
204
self.copy_multi(store_from, ids, pb=pb,
205
from_transaction=from_transaction,
206
to_transaction=to_transaction)
208
def copy_multi(self, from_store, file_ids, pb=None, from_transaction=None,
209
to_transaction=None):
210
"""Copy all the versions for multiple file_ids from from_store.
212
:param from_transaction: required current transaction in from_store.
214
from bzrlib.transactions import PassThroughTransaction
215
if from_transaction is None:
216
warn("WeaveStore.copy_multi without a from_transaction parameter "
217
"is deprecated. Please provide a from_transaction.",
220
# we are reading one object - caching is irrelevant.
221
from_transaction = PassThroughTransaction()
222
if to_transaction is None:
223
warn("WeaveStore.copy_multi without a to_transaction parameter "
224
"is deprecated. Please provide a to_transaction.",
227
# we are copying single objects, and there may be open tranasactions
228
# so again with the passthrough
229
to_transaction = PassThroughTransaction()
230
pb = bzrlib.ui.ui_factory.nested_progress_bar()
232
for count, f in enumerate(file_ids):
233
mutter("copy weave {%s} into %s", f, self)
234
pb.update('copy', count, len(file_ids))
235
# if we have it in cache, its faster.
236
# joining is fast with knits, and bearable for weaves -
237
# indeed the new case can be optimised if needed.
238
target = self._make_new_versionedfile(f, to_transaction)
239
source = from_store.get_weave(f, from_transaction)
240
target.insert_record_stream(source.get_record_stream(
241
[(version,) for version in source.versions()],
242
'topological', False))
246
def total_size(self):
247
count, bytes = super(VersionedFileStore, self).total_size()
248
return (count / len(self._versionedfile_class.get_suffixes())), bytes
250
WeaveStore = VersionedFileStore
added
removed
bzrlib/store/weave.py
