56
"""This class represents the abstract storage layout for saving information.
37
class ImmutableStore(object):
38
"""Store that holds files indexed by unique names.
58
40
Files can be added, but not modified once they are in. Typically
59
41
the hash is used as the name, or something else known to be unique,
44
>>> st = ImmutableScratchStore()
46
>>> st.add(StringIO('hello'), 'aa')
52
You are not allowed to add an id that is already present.
54
Entries can be retrieved as files, which may then be read.
56
>>> st.add(StringIO('goodbye'), '123123')
57
>>> st['123123'].read()
60
TODO: Atomic add by writing to a temporary file and renaming.
62
In bzr 0.0.5 and earlier, files within the store were marked
63
readonly on disk. This is no longer done but existing stores need
64
raise NotImplementedError('Children should define their length')
66
def get(self, fileid, suffix=None):
67
"""Returns a file reading from a particular entry.
69
If suffix is present, retrieve the named suffix for fileid.
71
raise NotImplementedError
73
def __getitem__(self, fileid):
74
"""DEPRECATED. Please use .get(fileid) instead."""
75
raise NotImplementedError
77
#def __contains__(self, fileid):
78
# """Deprecated, please use has_id"""
79
# raise NotImplementedError
82
raise NotImplementedError
84
def add(self, f, fileid):
85
"""Add a file object f to the store accessible from the given fileid"""
86
raise NotImplementedError('Children of Store must define their method of adding entries.')
88
def has_id(self, fileid, suffix=None):
89
"""Return True or false for the presence of fileid in the store.
91
suffix, if present, is a per file suffix, i.e. for digital signature
93
raise NotImplementedError
96
"""Return True if this store is able to be listed."""
97
return (getattr(self, "__iter__", None) is not None)
99
def copy_all_ids(self, store_from, pb=None):
100
"""Copy all the file ids from store_from into self."""
101
if not store_from.listable():
102
raise UnlistableStore(store_from)
104
for count, file_id in enumerate(store_from):
106
pb.update('listing files', count, count)
110
mutter('copy_all ids: %r', ids)
111
self.copy_multi(store_from, ids, pb=pb)
113
def copy_multi(self, other, ids, pb=None, permit_failure=False):
67
def __init__(self, basedir):
68
self._basedir = basedir
71
if '\\' in id or '/' in id:
72
raise ValueError("invalid store id %r" % id)
73
return os.path.join(self._basedir, id)
76
return "%s(%r)" % (self.__class__.__name__, self._basedir)
78
def add(self, f, fileid, compressed=True):
79
"""Add contents of a file into the store.
81
f -- An open file, or file-like object."""
82
# FIXME: Only works on files that will fit in memory
84
from bzrlib.atomicfile import AtomicFile
86
mutter("add store entry %r" % (fileid))
87
if isinstance(f, types.StringTypes):
92
p = self._path(fileid)
93
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
94
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
100
af = AtomicFile(fn, 'wb')
103
gf = gzip.GzipFile(mode='wb', fileobj=af)
113
def copy_multi(self, other, ids, permit_failure=False):
114
114
"""Copy texts for ids from other into self.
116
If an id is present in self, it is skipped. A count of copied
117
ids is returned, which may be less than len(ids).
116
If an id is present in self, it is skipped.
119
:param other: Another Store object
120
:param ids: A list of entry ids to be copied
121
:param pb: A ProgressBar object, if none is given, the default will be created.
122
:param permit_failure: Allow missing entries to be ignored
123
:return: (n_copied, [failed]) The number of entries copied successfully,
124
followed by a list of entries which could not be copied (because they
118
Returns (count_copied, failed), where failed is a collection of ids
119
that could not be copied.
128
pb.update('preparing to copy')
121
pb = bzrlib.ui.ui_factory.progress_bar()
123
pb.update('preparing to copy')
124
to_copy = [id for id in ids if id not in self]
125
if isinstance(other, ImmutableStore):
126
return self.copy_multi_immutable(other, to_copy, pb)
130
pb.update('copy', count, len(to_copy))
131
if not permit_failure:
132
self.add(other[id], id)
141
assert count == len(to_copy)
145
def copy_multi_immutable(self, other, to_copy, pb, permit_failure=False):
146
from shutil import copyfile
131
ids = list(ids) # get the list for showing a length.
134
if self.has_id(fileid):
151
other_p = other._path(id)
137
self._copy_one(fileid, None, other, pb)
138
for suffix in self._suffixes:
140
self._copy_one(fileid, suffix, other, pb)
144
pb.update('copy', count, len(ids))
155
if e.errno == errno.ENOENT:
156
if not permit_failure:
157
copyfile(other_p+".gz", p+".gz")
160
copyfile(other_p+".gz", p+".gz")
162
if e.errno == errno.ENOENT:
150
assert count == len(ids)
170
pb.update('copy', count, len(to_copy))
171
assert count == len(to_copy)
153
173
return count, failed
155
def _copy_one(self, fileid, suffix, other, pb):
156
"""Most generic copy-one object routine.
158
Subclasses can override this to provide an optimised
159
copy between their own instances. Such overriden routines
160
should call this if they have no optimised facility for a
163
mutter('Store._copy_one: %r', fileid)
164
f = other.get(fileid, suffix)
165
self.add(f, fileid, suffix)
168
class TransportStore(Store):
169
"""A TransportStore is a Store superclass for Stores that use Transports."""
171
def add(self, f, fileid, suffix=None):
172
"""Add contents of a file into the store.
174
f -- A file-like object
176
mutter("add store entry %r", fileid)
177
if isinstance(f, str):
178
symbol_versioning.warn(zero_eleven % 'Passing a string to Store.add',
179
DeprecationWarning, stacklevel=2)
182
names = self._id_to_names(fileid, suffix)
183
if self._transport.has_any(names):
184
raise BzrError("store %r already contains id %r"
185
% (self._transport.base, fileid))
187
# Most of the time, just adding the file will work
188
# if we find a time where it fails, (because the dir
189
# doesn't exist), then create the dir, and try again
190
self._add(names[0], f)
192
def _add(self, relpath, f):
193
"""Actually add the file to the given location.
194
This should be overridden by children.
196
raise NotImplementedError('children need to implement this function.')
198
def _check_fileid(self, fileid):
199
if not isinstance(fileid, basestring):
200
raise TypeError('Fileids should be a string type: %s %r' % (type(fileid), fileid))
201
if '\\' in fileid or '/' in fileid:
202
raise ValueError("invalid store id %r" % fileid)
204
def _id_to_names(self, fileid, suffix):
205
"""Return the names in the expected order"""
206
if suffix is not None:
207
fn = self._relpath(fileid, [suffix])
209
fn = self._relpath(fileid)
211
# FIXME RBC 20051128 this belongs in TextStore.
218
def has_id(self, fileid, suffix=None):
219
"""See Store.has_id."""
220
return self._transport.has_any(self._id_to_names(fileid, suffix))
222
def _get_name(self, fileid, suffix=None):
223
"""A special check, which returns the name of an existing file.
225
This is similar in spirit to 'has_id', but it is designed
226
to return information about which file the store has.
228
for name in self._id_to_names(fileid, suffix=suffix):
229
if self._transport.has(name):
233
def _get(self, filename):
234
"""Return an vanilla file stream for clients to read from.
236
This is the body of a template method on 'get', and should be
237
implemented by subclasses.
239
raise NotImplementedError
241
def get(self, fileid, suffix=None):
242
"""See Store.get()."""
243
names = self._id_to_names(fileid, suffix)
246
return self._get(name)
247
except errors.NoSuchFile:
249
raise KeyError(fileid)
251
def __init__(self, a_transport, prefixed=False, compressed=False,
252
dir_mode=None, file_mode=None,
254
assert isinstance(a_transport, Transport)
255
super(TransportStore, self).__init__()
256
self._transport = a_transport
257
self._prefixed = prefixed
258
# FIXME RBC 20051128 this belongs in TextStore.
259
self._compressed = compressed
260
self._suffixes = set()
261
self._escaped = escaped
263
# It is okay for these to be None, it just means they
264
# will just use the filesystem defaults
265
self._dir_mode = dir_mode
266
self._file_mode = file_mode
268
def _unescape(self, file_id):
269
"""If filename escaping is enabled for this store, unescape and return the filename."""
271
return urllib.unquote(file_id)
275
def _iter_files_recursive(self):
276
"""Iterate through the files in the transport."""
277
for quoted_relpath in self._transport.iter_files_recursive():
278
# transport iterator always returns quoted paths, regardless of
280
yield urllib.unquote(quoted_relpath)
176
def __contains__(self, fileid):
178
p = self._path(fileid)
179
return (os.access(p, os.R_OK)
180
or os.access(p + '.gz', os.R_OK))
182
# TODO: Guard against the same thing being stored twice, compressed and uncompresse
282
184
def __iter__(self):
283
for relpath in self._iter_files_recursive():
284
# worst case is one of each suffix.
285
name = os.path.basename(relpath)
286
if name.endswith('.gz'):
289
for count in range(len(self._suffixes)):
290
for suffix in self._suffixes:
291
if name.endswith('.' + suffix):
294
yield self._unescape(name)
185
for f in os.listdir(self._basedir):
187
# TODO: case-insensitive?
296
192
def __len__(self):
297
return len(list(self.__iter__()))
299
def _relpath(self, fileid, suffixes=None):
300
self._check_fileid(fileid)
302
for suffix in suffixes:
303
if not suffix in self._suffixes:
304
raise ValueError("Unregistered suffix %r" % suffix)
305
self._check_fileid(suffix)
308
fileid = self._escape_file_id(fileid)
310
# hash_prefix adds the '/' separator
311
prefix = self.hash_prefix(fileid, escaped=True)
314
path = prefix + fileid
315
full_path = u'.'.join([path] + suffixes)
316
return urlutils.escape(full_path)
318
def _escape_file_id(self, file_id):
319
"""Turn a file id into a filesystem safe string.
321
This is similar to a plain urllib.quote, except
322
it uses specific safe characters, so that it doesn't
323
have to translate a lot of valid file ids.
325
if not self._escaped:
327
if isinstance(file_id, unicode):
328
file_id = file_id.encode('utf-8')
329
# @ does not get escaped. This is because it is a valid
330
# filesystem character we use all the time, and it looks
331
# a lot better than seeing %40 all the time.
332
safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
333
r = [((c in safe) and c or ('%%%02x' % ord(c)))
337
def hash_prefix(self, fileid, escaped=False):
338
# fileid should be unescaped
339
if not escaped and self._escaped:
340
fileid = self._escape_file_id(fileid)
341
return "%02x/" % (adler32(fileid) & 0xff)
344
if self._transport is None:
345
return "%s(None)" % (self.__class__.__name__)
347
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
352
"""Return True if this store is able to be listed."""
353
return self._transport.listable()
355
def register_suffix(self, suffix):
356
"""Register a suffix as being expected in this store."""
357
self._check_fileid(suffix)
359
raise ValueError('You cannot register the "gz" suffix.')
360
self._suffixes.add(suffix)
193
return len(os.listdir(self._basedir))
196
def __getitem__(self, fileid):
197
"""Returns a file reading from a particular entry."""
198
p = self._path(fileid)
200
return gzip.GzipFile(p + '.gz', 'rb')
202
if e.errno != errno.ENOENT:
208
if e.errno != errno.ENOENT:
211
raise IndexError(fileid)
362
214
def total_size(self):
363
215
"""Return (count, bytes)
added
removed
bzrlib/store.py
