46
"""This class represents the abstract storage layout for saving information.
36
class ImmutableStore(object):
37
"""Store that holds files indexed by unique names.
48
39
Files can be added, but not modified once they are in. Typically
49
40
the hash is used as the name, or something else known to be unique,
43
>>> st = ImmutableScratchStore()
45
>>> st.add(StringIO('hello'), 'aa')
51
You are not allowed to add an id that is already present.
53
Entries can be retrieved as files, which may then be read.
55
>>> st.add(StringIO('goodbye'), '123123')
56
>>> st['123123'].read()
59
TODO: Atomic add by writing to a temporary file and renaming.
61
In bzr 0.0.5 and earlier, files within the store were marked
62
readonly on disk. This is no longer done but existing stores need
54
raise NotImplementedError('Children should define their length')
56
def get(self, file_id, suffix=None):
57
"""Returns a file reading from a particular entry.
59
If suffix is present, retrieve the named suffix for file_id.
61
raise NotImplementedError
63
def __getitem__(self, fileid):
64
"""DEPRECATED. Please use .get(file_id) instead."""
65
raise NotImplementedError
67
#def __contains__(self, fileid):
68
# """Deprecated, please use has_id"""
69
# raise NotImplementedError
72
raise NotImplementedError
74
def add(self, f, fileid):
75
"""Add a file object f to the store accessible from the given fileid"""
76
raise NotImplementedError('Children of Store must define their method of adding entries.')
78
def has_id(self, file_id, suffix=None):
79
"""Return True or false for the presence of file_id in the store.
81
suffix, if present, is a per file suffix, i.e. for digital signature
83
raise NotImplementedError
86
"""Return True if this store is able to be listed."""
87
return hasattr(self, "__iter__")
89
def copy_multi(self, other, ids, pb=None, permit_failure=False):
66
def __init__(self, basedir):
67
self._basedir = basedir
70
if '\\' in id or '/' in id:
71
raise ValueError("invalid store id %r" % id)
72
return os.path.join(self._basedir, id)
75
return "%s(%r)" % (self.__class__.__name__, self._basedir)
77
def add(self, f, fileid, compressed=True):
78
"""Add contents of a file into the store.
80
f -- An open file, or file-like object."""
81
# FIXME: Only works on files that will fit in memory
83
from bzrlib.atomicfile import AtomicFile
85
mutter("add store entry %r" % (fileid))
86
if isinstance(f, types.StringTypes):
91
p = self._path(fileid)
92
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
93
from bzrlib.errors import bailout
94
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
100
af = AtomicFile(fn, 'wb')
103
gf = gzip.GzipFile(mode='wb', fileobj=af)
113
def copy_multi(self, other, ids):
90
114
"""Copy texts for ids from other into self.
92
116
If an id is present in self, it is skipped. A count of copied
93
117
ids is returned, which may be less than len(ids).
95
:param other: Another Store object
96
:param ids: A list of entry ids to be copied
97
:param pb: A ProgressBar object, if none is given, the default will be created.
98
:param permit_failure: Allow missing entries to be ignored
99
:return: (n_copied, [failed]) The number of entries copied successfully,
100
followed by a list of entries which could not be copied (because they
104
pb = bzrlib.ui.ui_factory.progress_bar()
119
from bzrlib.progress import ProgressBar
105
121
pb.update('preparing to copy')
122
to_copy = [id for id in ids if id not in self]
123
if isinstance(other, ImmutableStore):
124
return self.copy_multi_immutable(other, to_copy, pb)
108
ids = list(ids) # get the list for showing a length.
111
if self.has_id(fileid):
128
pb.update('copy', count, len(to_copy))
129
self.add(other[id], id)
130
assert count == len(to_copy)
135
def copy_multi_immutable(self, other, to_copy, pb):
136
from shutil import copyfile
140
other_p = other._path(id)
114
self._copy_one(fileid, None, other, pb)
115
for suffix in self._suffixes:
117
self._copy_one(fileid, suffix, other, pb)
120
pb.update('copy', count, len(ids))
144
if e.errno == errno.ENOENT:
145
copyfile(other_p+".gz", p+".gz")
126
assert count == len(ids)
150
pb.update('copy', count, len(to_copy))
151
assert count == len(to_copy)
130
def _copy_one(self, fileid, suffix, other, pb):
131
"""Most generic copy-one object routine.
133
Subclasses can override this to provide an optimised
134
copy between their own instances. Such overriden routines
135
should call this if they have no optimised facility for a
138
f = other.get(fileid, suffix)
139
self.add(f, fileid, suffix)
142
class TransportStore(Store):
143
"""A TransportStore is a Store superclass for Stores that use Transports."""
145
def add(self, f, fileid, suffix=None):
146
"""Add contents of a file into the store.
148
f -- A file-like object, or string
150
mutter("add store entry %r" % (fileid))
152
if suffix is not None:
153
fn = self._relpath(fileid, [suffix])
155
fn = self._relpath(fileid)
156
if self._transport.has(fn):
157
raise BzrError("store %r already contains id %r" % (self._transport.base, fileid))
161
self._transport.mkdir(hash_prefix(fileid)[:-1])
162
except errors.FileExists:
167
def _check_fileid(self, fileid):
168
if not isinstance(fileid, basestring):
169
raise TypeError('Fileids should be a string type: %s %r' % (type(fileid), fileid))
170
if '\\' in fileid or '/' in fileid:
171
raise ValueError("invalid store id %r" % fileid)
173
def has_id(self, fileid, suffix=None):
174
"""See Store.has_id."""
175
if suffix is not None:
176
fn = self._relpath(fileid, [suffix])
178
fn = self._relpath(fileid)
179
return self._transport.has(fn)
181
def _get(self, filename):
182
"""Return an vanilla file stream for clients to read from.
184
This is the body of a template method on 'get', and should be
185
implemented by subclasses.
187
raise NotImplementedError
189
def get(self, fileid, suffix=None):
190
"""See Store.get()."""
191
if suffix is None or suffix == 'gz':
192
fn = self._relpath(fileid)
194
fn = self._relpath(fileid, [suffix])
197
except errors.NoSuchFile:
198
raise KeyError(fileid)
200
def __init__(self, a_transport, prefixed=False):
201
assert isinstance(a_transport, transport.Transport)
202
super(TransportStore, self).__init__()
203
self._transport = a_transport
204
self._prefixed = prefixed
205
# conflating the .gz extension and user suffixes was a mistake.
206
# RBC 20051017 - TODO SOON, separate them again.
207
self._suffixes = set()
156
def __contains__(self, fileid):
158
p = self._path(fileid)
159
return (os.access(p, os.R_OK)
160
or os.access(p + '.gz', os.R_OK))
162
# TODO: Guard against the same thing being stored twice, compressed and uncompresse
209
164
def __iter__(self):
210
for relpath in self._transport.iter_files_recursive():
211
# worst case is one of each suffix.
212
name = os.path.basename(relpath)
213
if name.endswith('.gz'):
216
for count in range(len(self._suffixes)):
217
for suffix in self._suffixes:
218
if name.endswith('.' + suffix):
165
for f in os.listdir(self._basedir):
167
# TODO: case-insensitive?
223
172
def __len__(self):
224
return len(list(self.__iter__()))
226
def _relpath(self, fileid, suffixes=[]):
227
self._check_fileid(fileid)
228
for suffix in suffixes:
229
if not suffix in self._suffixes:
230
raise ValueError("Unregistered suffix %r" % suffix)
231
self._check_fileid(suffix)
233
path = [hash_prefix(fileid) + fileid]
236
path.extend(suffixes)
237
return transport.urlescape('.'.join(path))
240
if self._transport is None:
241
return "%s(None)" % (self.__class__.__name__)
243
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
248
"""Return True if this store is able to be listed."""
249
return self._transport.listable()
251
def register_suffix(self, suffix):
252
"""Register a suffix as being expected in this store."""
253
self._check_fileid(suffix)
254
self._suffixes.add(suffix)
173
return len(os.listdir(self._basedir))
175
def __getitem__(self, fileid):
176
"""Returns a file reading from a particular entry."""
177
p = self._path(fileid)
179
return gzip.GzipFile(p + '.gz', 'rb')
181
if e.errno == errno.ENOENT:
256
186
def total_size(self):
257
187
"""Return (count, bytes)
263
for relpath in self._transport.iter_files_recursive():
265
total += self._transport.stat(relpath).st_size
197
total += os.stat(p)[ST_SIZE]
199
total += os.stat(p + '.gz')[ST_SIZE]
267
201
return count, total
270
def ImmutableMemoryStore():
271
return bzrlib.store.text.TextStore(transport.memory.MemoryTransport())
274
class CachedStore(Store):
275
"""A store that caches data locally, to avoid repeated downloads.
276
The precacache method should be used to avoid server round-trips for
206
class ImmutableScratchStore(ImmutableStore):
207
"""Self-destructing test subclass of ImmutableStore.
209
The Store only exists for the lifetime of the Python object.
210
Obviously you should not put anything precious in it.
280
def __init__(self, store, cache_dir):
281
super(CachedStore, self).__init__()
282
self.source_store = store
283
# This clones the source store type with a locally bound
284
# transport. FIXME: it assumes a constructor is == cloning.
285
# clonable store - it might be nicer to actually have a clone()
286
# or something. RBC 20051003
287
self.cache_store = store.__class__(LocalTransport(cache_dir))
290
mutter("Cache add %s" % id)
291
if id not in self.cache_store:
292
self.cache_store.add(self.source_store.get(id), id)
293
return self.cache_store.get(id)
295
def has_id(self, fileid, suffix=None):
296
"""See Store.has_id."""
297
if self.cache_store.has_id(fileid, suffix):
299
if self.source_store.has_id(fileid, suffix):
300
# We could copy at this time
305
def copy_all(store_from, store_to):
306
"""Copy all ids from one store to another."""
307
# TODO: Optional progress indicator
308
if not store_from.listable():
309
raise UnlistableStore(store_from)
310
ids = [f for f in store_from]
311
store_to.copy_multi(store_from, ids)
313
def hash_prefix(file_id):
314
return "%02x/" % (adler32(file_id) & 0xff)
213
ImmutableStore.__init__(self, tempfile.mkdtemp())
216
for f in os.listdir(self._basedir):
217
fpath = os.path.join(self._basedir, f)
218
# needed on windows, and maybe some other filesystems
219
os.chmod(fpath, 0600)
221
os.rmdir(self._basedir)
222
mutter("%r destroyed" % self)
added
removed
bzrlib/store.py
