46
"""This class represents the abstract storage layout for saving information.
38
"""Store that holds files indexed by unique names.
48
40
Files can be added, but not modified once they are in. Typically
49
41
the hash is used as the name, or something else known to be unique,
44
>>> st = ImmutableScratchStore()
46
>>> st.add(StringIO('hello'), 'aa')
52
You are not allowed to add an id that is already present.
54
Entries can be retrieved as files, which may then be read.
56
>>> st.add(StringIO('goodbye'), '123123')
57
>>> st['123123'].read()
60
TODO: Atomic add by writing to a temporary file and renaming.
62
TODO: Perhaps automatically transform to/from XML in a method?
63
Would just need to tell the constructor what class to
66
TODO: Even within a simple disk store like this, we could
67
gzip the files. But since many are less than one disk
68
block, that might not help a lot.
54
raise NotImplementedError('Children should define their length')
56
def get(self, file_id, suffix=None):
57
"""Returns a file reading from a particular entry.
59
If suffix is present, retrieve the named suffix for file_id.
61
raise NotImplementedError
63
def __getitem__(self, fileid):
64
"""DEPRECATED. Please use .get(file_id) instead."""
65
raise NotImplementedError
67
#def __contains__(self, fileid):
68
# """Deprecated, please use has_id"""
69
# raise NotImplementedError
72
raise NotImplementedError
74
def add(self, f, fileid):
75
"""Add a file object f to the store accessible from the given fileid"""
76
raise NotImplementedError('Children of Store must define their method of adding entries.')
78
def has_id(self, file_id, suffix=None):
79
"""Return True or false for the presence of file_id in the store.
81
suffix, if present, is a per file suffix, i.e. for digital signature
83
raise NotImplementedError
86
"""Return True if this store is able to be listed."""
87
return hasattr(self, "__iter__")
89
def copy_multi(self, other, ids, pb=None, permit_failure=False):
90
"""Copy texts for ids from other into self.
92
If an id is present in self, it is skipped. A count of copied
93
ids is returned, which may be less than len(ids).
95
:param other: Another Store object
96
:param ids: A list of entry ids to be copied
97
:param pb: A ProgressBar object, if none is given, the default will be created.
98
:param permit_failure: Allow missing entries to be ignored
99
:return: (n_copied, [failed]) The number of entries copied successfully,
100
followed by a list of entries which could not be copied (because they
104
pb = bzrlib.ui.ui_factory.progress_bar()
105
pb.update('preparing to copy')
108
ids = list(ids) # get the list for showing a length.
111
if self.has_id(fileid):
114
self._copy_one(fileid, None, other, pb)
115
for suffix in self._suffixes:
117
self._copy_one(fileid, suffix, other, pb)
120
pb.update('copy', count, len(ids))
126
assert count == len(ids)
130
def _copy_one(self, fileid, suffix, other, pb):
131
"""Most generic copy-one object routine.
133
Subclasses can override this to provide an optimised
134
copy between their own instances. Such overriden routines
135
should call this if they have no optimised facility for a
138
f = other.get(fileid, suffix)
139
self.add(f, fileid, suffix)
142
class TransportStore(Store):
143
"""A TransportStore is a Store superclass for Stores that use Transports."""
145
def add(self, f, fileid, suffix=None):
72
def __init__(self, basedir):
73
"""ImmutableStore constructor."""
74
self._basedir = basedir
77
return os.path.join(self._basedir, id)
80
return "%s(%r)" % (self.__class__.__name__, self._basedir)
82
def add(self, f, fileid, compressed=True):
146
83
"""Add contents of a file into the store.
148
f -- A file-like object, or string
85
f -- An open file, or file-like object."""
86
# FIXME: Only works on smallish files
87
# TODO: Can be optimized by copying at the same time as
150
89
mutter("add store entry %r" % (fileid))
152
if suffix is not None:
153
fn = self._relpath(fileid, [suffix])
155
fn = self._relpath(fileid)
156
if self._transport.has(fn):
157
raise BzrError("store %r already contains id %r" % (self._transport.base, fileid))
161
self._transport.mkdir(hash_prefix(fileid)[:-1])
162
except errors.FileExists:
167
def _check_fileid(self, fileid):
168
if not isinstance(fileid, basestring):
169
raise TypeError('Fileids should be a string type: %s %r' % (type(fileid), fileid))
170
if '\\' in fileid or '/' in fileid:
171
raise ValueError("invalid store id %r" % fileid)
173
def has_id(self, fileid, suffix=None):
174
"""See Store.has_id."""
175
if suffix is not None:
176
fn = self._relpath(fileid, [suffix])
178
fn = self._relpath(fileid)
179
return self._transport.has(fn)
181
def _get(self, filename):
182
"""Return an vanilla file stream for clients to read from.
184
This is the body of a template method on 'get', and should be
185
implemented by subclasses.
187
raise NotImplementedError
189
def get(self, fileid, suffix=None):
190
"""See Store.get()."""
191
if suffix is None or suffix == 'gz':
192
fn = self._relpath(fileid)
194
fn = self._relpath(fileid, [suffix])
90
if isinstance(f, types.StringTypes):
95
p = self._path(fileid)
96
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
97
bailout("store %r already contains id %r" % (self._basedir, fileid))
100
f = gzip.GzipFile(p + '.gz', 'wb')
101
os.chmod(p + '.gz', 0444)
110
def __contains__(self, fileid):
112
p = self._path(fileid)
113
return (os.access(p, os.R_OK)
114
or os.access(p + '.gz', os.R_OK))
116
# TODO: Guard against the same thing being stored twice, compressed and uncompresse
119
for f in os.listdir(self._basedir):
121
# TODO: case-insensitive?
127
return len(os.listdir(self._basedir))
129
def __getitem__(self, fileid):
130
"""Returns a file reading from a particular entry."""
131
p = self._path(fileid)
197
except errors.NoSuchFile:
198
raise KeyError(fileid)
200
def __init__(self, a_transport, prefixed=False):
201
assert isinstance(a_transport, transport.Transport)
202
super(TransportStore, self).__init__()
203
self._transport = a_transport
204
self._prefixed = prefixed
205
# conflating the .gz extension and user suffixes was a mistake.
206
# RBC 20051017 - TODO SOON, separate them again.
207
self._suffixes = set()
210
for relpath in self._transport.iter_files_recursive():
211
# worst case is one of each suffix.
212
name = os.path.basename(relpath)
213
if name.endswith('.gz'):
216
for count in range(len(self._suffixes)):
217
for suffix in self._suffixes:
218
if name.endswith('.' + suffix):
224
return len(list(self.__iter__()))
226
def _relpath(self, fileid, suffixes=[]):
227
self._check_fileid(fileid)
228
for suffix in suffixes:
229
if not suffix in self._suffixes:
230
raise ValueError("Unregistered suffix %r" % suffix)
231
self._check_fileid(suffix)
233
path = [hash_prefix(fileid) + fileid]
236
path.extend(suffixes)
237
return transport.urlescape('.'.join(path))
240
if self._transport is None:
241
return "%s(None)" % (self.__class__.__name__)
243
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
248
"""Return True if this store is able to be listed."""
249
return self._transport.listable()
251
def register_suffix(self, suffix):
252
"""Register a suffix as being expected in this store."""
253
self._check_fileid(suffix)
254
self._suffixes.add(suffix)
133
return gzip.GzipFile(p + '.gz', 'rb')
135
if e.errno == errno.ENOENT:
256
140
def total_size(self):
257
141
"""Return (count, bytes)
263
for relpath in self._transport.iter_files_recursive():
265
total += self._transport.stat(relpath).st_size
151
total += os.stat(p)[ST_SIZE]
153
total += os.stat(p + '.gz')[ST_SIZE]
267
155
return count, total
270
def ImmutableMemoryStore():
271
return bzrlib.store.text.TextStore(transport.memory.MemoryTransport())
274
class CachedStore(Store):
275
"""A store that caches data locally, to avoid repeated downloads.
276
The precacache method should be used to avoid server round-trips for
160
class ImmutableScratchStore(ImmutableStore):
161
"""Self-destructing test subclass of ImmutableStore.
163
The Store only exists for the lifetime of the Python object.
164
Obviously you should not put anything precious in it.
280
def __init__(self, store, cache_dir):
281
super(CachedStore, self).__init__()
282
self.source_store = store
283
# This clones the source store type with a locally bound
284
# transport. FIXME: it assumes a constructor is == cloning.
285
# clonable store - it might be nicer to actually have a clone()
286
# or something. RBC 20051003
287
self.cache_store = store.__class__(LocalTransport(cache_dir))
290
mutter("Cache add %s" % id)
291
if id not in self.cache_store:
292
self.cache_store.add(self.source_store.get(id), id)
293
return self.cache_store.get(id)
295
def has_id(self, fileid, suffix=None):
296
"""See Store.has_id."""
297
if self.cache_store.has_id(fileid, suffix):
299
if self.source_store.has_id(fileid, suffix):
300
# We could copy at this time
305
def copy_all(store_from, store_to):
306
"""Copy all ids from one store to another."""
307
# TODO: Optional progress indicator
308
if not store_from.listable():
309
raise UnlistableStore(store_from)
310
ids = [f for f in store_from]
311
store_to.copy_multi(store_from, ids)
313
def hash_prefix(file_id):
314
return "%02x/" % (adler32(file_id) & 0xff)
167
ImmutableStore.__init__(self, tempfile.mkdtemp())
170
for f in os.listdir(self._basedir):
171
fpath = os.path.join(self._basedir, f)
172
# needed on windows, and maybe some other filesystems
173
os.chmod(fpath, 0600)
175
os.rmdir(self._basedir)
176
mutter("%r destroyed" % self)
added
removed
bzrlib/store.py
