55
"""This class represents the abstract storage layout for saving information.
36
class ImmutableStore(object):
37
"""Store that holds files indexed by unique names.
57
39
Files can be added, but not modified once they are in. Typically
58
40
the hash is used as the name, or something else known to be unique,
43
>>> st = ImmutableScratchStore()
45
>>> st.add(StringIO('hello'), 'aa')
51
You are not allowed to add an id that is already present.
53
Entries can be retrieved as files, which may then be read.
55
>>> st.add(StringIO('goodbye'), '123123')
56
>>> st['123123'].read()
59
TODO: Atomic add by writing to a temporary file and renaming.
61
In bzr 0.0.5 and earlier, files within the store were marked
62
readonly on disk. This is no longer done but existing stores need
63
raise NotImplementedError('Children should define their length')
65
def get(self, fileid, suffix=None):
66
"""Returns a file reading from a particular entry.
68
If suffix is present, retrieve the named suffix for fileid.
70
raise NotImplementedError
72
def __getitem__(self, fileid):
73
"""DEPRECATED. Please use .get(fileid) instead."""
74
raise NotImplementedError
76
#def __contains__(self, fileid):
77
# """Deprecated, please use has_id"""
78
# raise NotImplementedError
81
raise NotImplementedError
83
def add(self, f, fileid):
84
"""Add a file object f to the store accessible from the given fileid"""
85
raise NotImplementedError('Children of Store must define their method of adding entries.')
87
def has_id(self, fileid, suffix=None):
88
"""Return True or false for the presence of fileid in the store.
90
suffix, if present, is a per file suffix, i.e. for digital signature
92
raise NotImplementedError
95
"""Return True if this store is able to be listed."""
96
return (getattr(self, "__iter__", None) is not None)
98
def copy_all_ids(self, store_from, pb=None):
99
"""Copy all the file ids from store_from into self."""
100
if not store_from.listable():
101
raise UnlistableStore(store_from)
103
for count, file_id in enumerate(store_from):
105
pb.update('listing files', count, count)
109
mutter('copy_all ids: %r', ids)
110
self.copy_multi(store_from, ids, pb=pb)
112
def copy_multi(self, other, ids, pb=None, permit_failure=False):
66
def __init__(self, basedir):
67
self._basedir = basedir
70
if '\\' in id or '/' in id:
71
raise ValueError("invalid store id %r" % id)
72
return os.path.join(self._basedir, id)
75
return "%s(%r)" % (self.__class__.__name__, self._basedir)
77
def add(self, f, fileid, compressed=True):
78
"""Add contents of a file into the store.
80
f -- An open file, or file-like object."""
81
# FIXME: Only works on files that will fit in memory
83
from bzrlib.atomicfile import AtomicFile
85
mutter("add store entry %r" % (fileid))
86
if isinstance(f, types.StringTypes):
91
p = self._path(fileid)
92
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
93
from bzrlib.errors import bailout
94
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
100
af = AtomicFile(fn, 'wb')
103
gf = gzip.GzipFile(mode='wb', fileobj=af)
113
def copy_multi(self, other, ids):
113
114
"""Copy texts for ids from other into self.
115
116
If an id is present in self, it is skipped. A count of copied
116
117
ids is returned, which may be less than len(ids).
118
:param other: Another Store object
119
:param ids: A list of entry ids to be copied
120
:param pb: A ProgressBar object, if none is given, the default will be created.
121
:param permit_failure: Allow missing entries to be ignored
122
:return: (n_copied, [failed]) The number of entries copied successfully,
123
followed by a list of entries which could not be copied (because they
127
pb.update('preparing to copy')
119
from bzrlib.progress import ProgressBar
121
pb.update('preparing to copy')
122
to_copy = [id for id in ids if id not in self]
123
if isinstance(other, ImmutableStore):
124
return self.copy_multi_immutable(other, to_copy, pb)
130
ids = [osutils.safe_file_id(i) for i in ids] # get the list for showing a length.
133
if self.has_id(fileid):
128
pb.update('copy', count, len(to_copy))
129
self.add(other[id], id)
130
assert count == len(to_copy)
135
def copy_multi_immutable(self, other, to_copy, pb):
136
from shutil import copyfile
140
other_p = other._path(id)
136
self._copy_one(fileid, None, other, pb)
137
for suffix in self._suffixes:
139
self._copy_one(fileid, suffix, other, pb)
143
pb.update('copy', count, len(ids))
144
if e.errno == errno.ENOENT:
145
copyfile(other_p+".gz", p+".gz")
149
assert count == len(ids)
154
def _copy_one(self, fileid, suffix, other, pb):
155
"""Most generic copy-one object routine.
157
Subclasses can override this to provide an optimised
158
copy between their own instances. Such overriden routines
159
should call this if they have no optimised facility for a
162
mutter('Store._copy_one: %r', fileid)
163
f = other.get(fileid, suffix)
164
self.add(f, fileid, suffix)
167
class TransportStore(Store):
168
"""A TransportStore is a Store superclass for Stores that use Transports."""
170
def add(self, f, fileid, suffix=None):
171
"""Add contents of a file into the store.
173
f -- A file-like object
175
fileid = osutils.safe_file_id(fileid)
176
mutter("add store entry %r", fileid)
177
names = self._id_to_names(fileid, suffix)
178
if self._transport.has_any(names):
179
raise BzrError("store %r already contains id %r"
180
% (self._transport.base, fileid))
182
# Most of the time, just adding the file will work
183
# if we find a time where it fails, (because the dir
184
# doesn't exist), then create the dir, and try again
185
self._add(names[0], f)
187
def _add(self, relpath, f):
188
"""Actually add the file to the given location.
189
This should be overridden by children.
191
raise NotImplementedError('children need to implement this function.')
193
def _check_fileid(self, fileid):
194
if not isinstance(fileid, basestring):
195
raise TypeError('Fileids should be a string type: %s %r' % (type(fileid), fileid))
196
if '\\' in fileid or '/' in fileid:
197
raise ValueError("invalid store id %r" % fileid)
199
def _id_to_names(self, fileid, suffix):
200
"""Return the names in the expected order"""
201
if suffix is not None:
202
fn = self._relpath(fileid, [suffix])
204
fn = self._relpath(fileid)
206
# FIXME RBC 20051128 this belongs in TextStore.
213
def has_id(self, fileid, suffix=None):
214
"""See Store.has_id."""
215
fileid = osutils.safe_file_id(fileid)
216
return self._transport.has_any(self._id_to_names(fileid, suffix))
218
def _get_name(self, fileid, suffix=None):
219
"""A special check, which returns the name of an existing file.
221
This is similar in spirit to 'has_id', but it is designed
222
to return information about which file the store has.
224
for name in self._id_to_names(fileid, suffix=suffix):
225
if self._transport.has(name):
229
def _get(self, filename):
230
"""Return an vanilla file stream for clients to read from.
232
This is the body of a template method on 'get', and should be
233
implemented by subclasses.
235
raise NotImplementedError
237
def get(self, fileid, suffix=None):
238
"""See Store.get()."""
239
fileid = osutils.safe_file_id(fileid)
240
names = self._id_to_names(fileid, suffix)
243
return self._get(name)
244
except errors.NoSuchFile:
246
raise KeyError(fileid)
248
def __init__(self, a_transport, prefixed=False, compressed=False,
249
dir_mode=None, file_mode=None,
251
assert isinstance(a_transport, Transport)
252
super(TransportStore, self).__init__()
253
self._transport = a_transport
254
self._prefixed = prefixed
255
# FIXME RBC 20051128 this belongs in TextStore.
256
self._compressed = compressed
257
self._suffixes = set()
258
self._escaped = escaped
260
# It is okay for these to be None, it just means they
261
# will just use the filesystem defaults
262
self._dir_mode = dir_mode
263
self._file_mode = file_mode
265
def _unescape(self, file_id):
266
"""If filename escaping is enabled for this store, unescape and return the filename."""
268
return urllib.unquote(file_id)
272
def _iter_files_recursive(self):
273
"""Iterate through the files in the transport."""
274
for quoted_relpath in self._transport.iter_files_recursive():
275
# transport iterator always returns quoted paths, regardless of
277
yield urllib.unquote(quoted_relpath)
150
pb.update('copy', count, len(to_copy))
151
assert count == len(to_copy)
156
def __contains__(self, fileid):
158
p = self._path(fileid)
159
return (os.access(p, os.R_OK)
160
or os.access(p + '.gz', os.R_OK))
162
# TODO: Guard against the same thing being stored twice, compressed and uncompresse
279
164
def __iter__(self):
280
for relpath in self._iter_files_recursive():
281
# worst case is one of each suffix.
282
name = os.path.basename(relpath)
283
if name.endswith('.gz'):
286
for count in range(len(self._suffixes)):
287
for suffix in self._suffixes:
288
if name.endswith('.' + suffix):
291
yield self._unescape(name)
165
for f in os.listdir(self._basedir):
167
# TODO: case-insensitive?
293
172
def __len__(self):
294
return len(list(self.__iter__()))
296
def _relpath(self, fileid, suffixes=None):
297
self._check_fileid(fileid)
299
for suffix in suffixes:
300
if not suffix in self._suffixes:
301
raise ValueError("Unregistered suffix %r" % suffix)
302
self._check_fileid(suffix)
305
fileid = self._escape_file_id(fileid)
307
# hash_prefix adds the '/' separator
308
prefix = self.hash_prefix(fileid, escaped=True)
311
path = prefix + fileid
312
full_path = u'.'.join([path] + suffixes)
313
return urlutils.escape(full_path)
315
def _escape_file_id(self, file_id):
316
"""Turn a file id into a filesystem safe string.
318
This is similar to a plain urllib.quote, except
319
it uses specific safe characters, so that it doesn't
320
have to translate a lot of valid file ids.
322
if not self._escaped:
324
if isinstance(file_id, unicode):
325
file_id = file_id.encode('utf-8')
326
# @ does not get escaped. This is because it is a valid
327
# filesystem character we use all the time, and it looks
328
# a lot better than seeing %40 all the time.
329
safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
330
r = [((c in safe) and c or ('%%%02x' % ord(c)))
334
def hash_prefix(self, fileid, escaped=False):
335
# fileid should be unescaped
336
if not escaped and self._escaped:
337
fileid = self._escape_file_id(fileid)
338
return "%02x/" % (adler32(fileid) & 0xff)
341
if self._transport is None:
342
return "%s(None)" % (self.__class__.__name__)
344
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
349
"""Return True if this store is able to be listed."""
350
return self._transport.listable()
352
def register_suffix(self, suffix):
353
"""Register a suffix as being expected in this store."""
354
self._check_fileid(suffix)
356
raise ValueError('You cannot register the "gz" suffix.')
357
self._suffixes.add(suffix)
173
return len(os.listdir(self._basedir))
175
def __getitem__(self, fileid):
176
"""Returns a file reading from a particular entry."""
177
p = self._path(fileid)
179
return gzip.GzipFile(p + '.gz', 'rb')
181
if e.errno == errno.ENOENT:
359
186
def total_size(self):
360
187
"""Return (count, bytes)
added
removed
bzrlib/store.py
