47
"""This class represents the abstract storage layout for saving information.
37
class ImmutableStore(object):
38
"""Store that holds files indexed by unique names.
49
40
Files can be added, but not modified once they are in. Typically
50
41
the hash is used as the name, or something else known to be unique,
44
>>> st = ImmutableScratchStore()
46
>>> st.add(StringIO('hello'), 'aa')
52
You are not allowed to add an id that is already present.
54
Entries can be retrieved as files, which may then be read.
56
>>> st.add(StringIO('goodbye'), '123123')
57
>>> st['123123'].read()
60
TODO: Atomic add by writing to a temporary file and renaming.
62
In bzr 0.0.5 and earlier, files within the store were marked
63
readonly on disk. This is no longer done but existing stores need
55
raise NotImplementedError('Children should define their length')
57
def get(self, fileid, suffix=None):
58
"""Returns a file reading from a particular entry.
60
If suffix is present, retrieve the named suffix for fileid.
62
raise NotImplementedError
64
def __getitem__(self, fileid):
65
"""DEPRECATED. Please use .get(fileid) instead."""
66
raise NotImplementedError
68
#def __contains__(self, fileid):
69
# """Deprecated, please use has_id"""
70
# raise NotImplementedError
73
raise NotImplementedError
75
def add(self, f, fileid):
76
"""Add a file object f to the store accessible from the given fileid"""
77
raise NotImplementedError('Children of Store must define their method of adding entries.')
79
def has_id(self, fileid, suffix=None):
80
"""Return True or false for the presence of fileid in the store.
82
suffix, if present, is a per file suffix, i.e. for digital signature
84
raise NotImplementedError
87
"""Return True if this store is able to be listed."""
88
return hasattr(self, "__iter__")
90
def copy_multi(self, other, ids, pb=None, permit_failure=False):
67
def __init__(self, basedir):
68
self._basedir = basedir
71
if '\\' in id or '/' in id:
72
raise ValueError("invalid store id %r" % id)
73
return os.path.join(self._basedir, id)
76
return "%s(%r)" % (self.__class__.__name__, self._basedir)
78
def add(self, f, fileid, compressed=True):
79
"""Add contents of a file into the store.
81
f -- An open file, or file-like object."""
82
# FIXME: Only works on files that will fit in memory
84
from bzrlib.atomicfile import AtomicFile
86
mutter("add store entry %r" % (fileid))
87
if isinstance(f, types.StringTypes):
92
p = self._path(fileid)
93
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
94
from bzrlib.errors import bailout
95
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
101
af = AtomicFile(fn, 'wb')
104
gf = gzip.GzipFile(mode='wb', fileobj=af)
114
def copy_multi(self, other, ids):
91
115
"""Copy texts for ids from other into self.
93
117
If an id is present in self, it is skipped. A count of copied
94
118
ids is returned, which may be less than len(ids).
96
:param other: Another Store object
97
:param ids: A list of entry ids to be copied
98
:param pb: A ProgressBar object, if none is given, the default will be created.
99
:param permit_failure: Allow missing entries to be ignored
100
:return: (n_copied, [failed]) The number of entries copied successfully,
101
followed by a list of entries which could not be copied (because they
105
pb = bzrlib.ui.ui_factory.progress_bar()
120
pb = bzrlib.ui.ui_factory.progress_bar()
106
122
pb.update('preparing to copy')
123
to_copy = [id for id in ids if id not in self]
124
if isinstance(other, ImmutableStore):
125
return self.copy_multi_immutable(other, to_copy, pb)
109
ids = list(ids) # get the list for showing a length.
112
if self.has_id(fileid):
129
pb.update('copy', count, len(to_copy))
130
self.add(other[id], id)
131
assert count == len(to_copy)
136
def copy_multi_immutable(self, other, to_copy, pb):
137
from shutil import copyfile
141
other_p = other._path(id)
115
self._copy_one(fileid, None, other, pb)
116
for suffix in self._suffixes:
118
self._copy_one(fileid, suffix, other, pb)
121
pb.update('copy', count, len(ids))
145
if e.errno == errno.ENOENT:
146
copyfile(other_p+".gz", p+".gz")
127
assert count == len(ids)
151
pb.update('copy', count, len(to_copy))
152
assert count == len(to_copy)
131
def _copy_one(self, fileid, suffix, other, pb):
132
"""Most generic copy-one object routine.
134
Subclasses can override this to provide an optimised
135
copy between their own instances. Such overriden routines
136
should call this if they have no optimised facility for a
139
mutter('Store._copy_one: %r', fileid)
140
f = other.get(fileid, suffix)
141
self.add(f, fileid, suffix)
144
class TransportStore(Store):
145
"""A TransportStore is a Store superclass for Stores that use Transports."""
147
def add(self, f, fileid, suffix=None):
148
"""Add contents of a file into the store.
150
f -- A file-like object, or string
152
mutter("add store entry %r", fileid)
154
names = self._id_to_names(fileid, suffix)
155
if self._transport.has_any(names):
156
raise BzrError("store %r already contains id %r"
157
% (self._transport.base, fileid))
159
# Most of the time, just adding the file will work
160
# if we find a time where it fails, (because the dir
161
# doesn't exist), then create the dir, and try again
162
self._add(names[0], f)
165
def _add(self, relpath, f):
166
"""Actually add the file to the given location.
167
This should be overridden by children.
169
raise NotImplementedError('children need to implement this function.')
171
def _check_fileid(self, fileid):
172
if not isinstance(fileid, basestring):
173
raise TypeError('Fileids should be a string type: %s %r' % (type(fileid), fileid))
174
if '\\' in fileid or '/' in fileid:
175
raise ValueError("invalid store id %r" % fileid)
177
def _id_to_names(self, fileid, suffix):
178
"""Return the names in the expected order"""
179
if suffix is not None:
180
fn = self._relpath(fileid, [suffix])
182
fn = self._relpath(fileid)
184
# FIXME RBC 20051128 this belongs in TextStore.
191
def has_id(self, fileid, suffix=None):
192
"""See Store.has_id."""
193
return self._transport.has_any(self._id_to_names(fileid, suffix))
195
def _get_name(self, fileid, suffix=None):
196
"""A special check, which returns the name of an existing file.
198
This is similar in spirit to 'has_id', but it is designed
199
to return information about which file the store has.
201
for name in self._id_to_names(fileid, suffix=suffix):
202
if self._transport.has(name):
206
def _get(self, filename):
207
"""Return an vanilla file stream for clients to read from.
209
This is the body of a template method on 'get', and should be
210
implemented by subclasses.
212
raise NotImplementedError
214
def get(self, fileid, suffix=None):
215
"""See Store.get()."""
216
names = self._id_to_names(fileid, suffix)
219
return self._get(name)
220
except errors.NoSuchFile:
222
raise KeyError(fileid)
224
def __init__(self, a_transport, prefixed=False, compressed=False,
225
dir_mode=None, file_mode=None):
226
assert isinstance(a_transport, transport.Transport)
227
super(TransportStore, self).__init__()
228
self._transport = a_transport
229
self._prefixed = prefixed
230
# FIXME RBC 20051128 this belongs in TextStore.
231
self._compressed = compressed
232
self._suffixes = set()
234
# It is okay for these to be None, it just means they
235
# will just use the filesystem defaults
236
self._dir_mode = dir_mode
237
self._file_mode = file_mode
239
def _iter_files_recursive(self):
240
"""Iterate through the files in the transport."""
241
for quoted_relpath in self._transport.iter_files_recursive():
242
yield urllib.unquote(quoted_relpath)
157
def __contains__(self, fileid):
159
p = self._path(fileid)
160
return (os.access(p, os.R_OK)
161
or os.access(p + '.gz', os.R_OK))
163
# TODO: Guard against the same thing being stored twice, compressed and uncompresse
244
165
def __iter__(self):
245
for relpath in self._iter_files_recursive():
246
# worst case is one of each suffix.
247
name = os.path.basename(relpath)
248
if name.endswith('.gz'):
251
for count in range(len(self._suffixes)):
252
for suffix in self._suffixes:
253
if name.endswith('.' + suffix):
166
for f in os.listdir(self._basedir):
168
# TODO: case-insensitive?
258
173
def __len__(self):
259
return len(list(self.__iter__()))
261
def _relpath(self, fileid, suffixes=None):
262
self._check_fileid(fileid)
264
for suffix in suffixes:
265
if not suffix in self._suffixes:
266
raise ValueError("Unregistered suffix %r" % suffix)
267
self._check_fileid(suffix)
271
path = [hash_prefix(fileid) + fileid]
274
path.extend(suffixes)
275
return transport.urlescape(u'.'.join(path))
278
if self._transport is None:
279
return "%s(None)" % (self.__class__.__name__)
281
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
286
"""Return True if this store is able to be listed."""
287
return self._transport.listable()
289
def register_suffix(self, suffix):
290
"""Register a suffix as being expected in this store."""
291
self._check_fileid(suffix)
293
raise ValueError('You cannot register the "gz" suffix.')
294
self._suffixes.add(suffix)
174
return len(os.listdir(self._basedir))
177
def __getitem__(self, fileid):
178
"""Returns a file reading from a particular entry."""
179
p = self._path(fileid)
181
return gzip.GzipFile(p + '.gz', 'rb')
183
if e.errno != errno.ENOENT:
189
if e.errno != errno.ENOENT:
192
raise IndexError(fileid)
296
195
def total_size(self):
297
196
"""Return (count, bytes)
added
removed
bzrlib/store.py
