14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
Stores are the main data-storage mechanism for Bazaar-NG.
17
"""Stores are the main data-storage mechanism for Bazaar-NG.
20
19
A store is a simple write-once container indexed by a universally
20
unique ID, which is typically the SHA-1 of the content."""
22
__copyright__ = "Copyright (C) 2005 Canonical Ltd."
23
__author__ = "Martin Pool <mbp@canonical.com>"
24
25
import os, tempfile, types, osutils, gzip, errno
25
26
from stat import ST_SIZE
26
27
from StringIO import StringIO
27
from bzrlib.errors import BzrError
28
from bzrlib.trace import mutter
30
from bzrlib.remotebranch import get_url
28
from trace import mutter
32
30
######################################################################
58
56
>>> st.add(StringIO('goodbye'), '123123')
59
57
>>> st['123123'].read()
64
"""Return (count, bytes)
66
This is the (compressed) size stored on disk, not the size of
72
total += self._item_size(fid)
76
class ImmutableStore(Store):
77
"""Store that stores files on disk.
79
60
TODO: Atomic add by writing to a temporary file and renaming.
80
TODO: Guard against the same thing being stored twice, compressed and
81
uncompressed during copy_multi_immutable - the window is for a
82
matching store with some crack code that lets it offer a
83
non gz FOO and then a fz FOO.
85
In bzr 0.0.5 and earlier, files within the store were marked
86
readonly on disk. This is no longer done but existing stores need
62
TODO: Perhaps automatically transform to/from XML in a method?
63
Would just need to tell the constructor what class to
66
TODO: Even within a simple disk store like this, we could
67
gzip the files. But since many are less than one disk
68
block, that might not help a lot.
90
72
def __init__(self, basedir):
91
super(ImmutableStore, self).__init__()
73
"""ImmutableStore constructor."""
92
74
self._basedir = basedir
94
76
def _path(self, id):
95
if '\\' in id or '/' in id:
96
raise ValueError("invalid store id %r" % id)
97
78
return os.path.join(self._basedir, id)
99
80
def __repr__(self):
103
84
"""Add contents of a file into the store.
105
86
f -- An open file, or file-like object."""
106
# FIXME: Only works on files that will fit in memory
108
from bzrlib.atomicfile import AtomicFile
87
# FIXME: Only works on smallish files
88
# TODO: Can be optimized by copying at the same time as
110
90
mutter("add store entry %r" % (fileid))
111
91
if isinstance(f, types.StringTypes):
114
94
content = f.read()
116
96
p = self._path(fileid)
117
97
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
118
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
98
bailout("store %r already contains id %r" % (self._basedir, fileid))
101
f = gzip.GzipFile(p + '.gz', 'wb')
102
os.chmod(p + '.gz', 0444)
124
af = AtomicFile(fn, 'wb')
127
gf = gzip.GzipFile(mode='wb', fileobj=af)
137
def copy_multi(self, other, ids, permit_failure=False):
110
def copy_multi(self, other, ids):
138
111
"""Copy texts for ids from other into self.
140
If an id is present in self, it is skipped.
142
Returns (count_copied, failed), where failed is a collection of ids
143
that could not be copied.
113
If an id is present in self, it is skipped. A count of copied
114
ids is returned, which may be less than len(ids).
145
pb = bzrlib.ui.ui_factory.progress_bar()
147
pb.update('preparing to copy')
148
to_copy = [id for id in ids if id not in self]
149
if isinstance(other, ImmutableStore):
150
return self.copy_multi_immutable(other, to_copy, pb)
155
pb.update('copy', count, len(to_copy))
156
if not permit_failure:
157
self.add(other[id], id)
166
if not permit_failure:
167
assert count == len(to_copy)
171
def copy_multi_immutable(self, other, to_copy, pb, permit_failure=False):
172
from shutil import copyfile
177
other_p = other._path(id)
181
if e.errno == errno.ENOENT:
182
if not permit_failure:
183
copyfile(other_p+".gz", p+".gz")
186
copyfile(other_p+".gz", p+".gz")
188
if e.errno == errno.ENOENT:
196
pb.update('copy', count, len(to_copy))
197
assert count == len(to_copy)
120
self.add(other[id], id)
201
124
def __contains__(self, fileid):
204
127
return (os.access(p, os.R_OK)
205
128
or os.access(p + '.gz', os.R_OK))
207
def _item_size(self, fid):
210
return os.stat(p)[ST_SIZE]
212
return os.stat(p + '.gz')[ST_SIZE]
130
# TODO: Guard against the same thing being stored twice, compressed and uncompresse
214
132
def __iter__(self):
215
133
for f in os.listdir(self._basedir):
229
147
return gzip.GzipFile(p + '.gz', 'rb')
230
148
except IOError, e:
231
if e.errno != errno.ENOENT:
237
if e.errno != errno.ENOENT:
240
raise IndexError(fileid)
149
if e.errno == errno.ENOENT:
154
def total_size(self):
155
"""Return (count, bytes)
157
This is the (compressed) size stored on disk, not the size of
165
total += os.stat(p)[ST_SIZE]
167
total += os.stat(p + '.gz')[ST_SIZE]
243
174
class ImmutableScratchStore(ImmutableStore):
244
175
"""Self-destructing test subclass of ImmutableStore.
246
177
The Store only exists for the lifetime of the Python object.
247
Obviously you should not put anything precious in it.
178
Obviously you should not put anything precious in it.
249
180
def __init__(self):
250
super(ImmutableScratchStore, self).__init__(tempfile.mkdtemp())
181
ImmutableStore.__init__(self, tempfile.mkdtemp())
252
183
def __del__(self):
253
184
for f in os.listdir(self._basedir):
258
189
os.rmdir(self._basedir)
259
190
mutter("%r destroyed" % self)
262
class ImmutableMemoryStore(Store):
263
"""A memory only store."""
266
super(ImmutableMemoryStore, self).__init__()
269
def add(self, stream, fileid, compressed=True):
270
if self._contents.has_key(fileid):
271
raise StoreError("fileid %s already in the store" % fileid)
272
self._contents[fileid] = stream.read()
274
def __getitem__(self, fileid):
275
"""Returns a file reading from a particular entry."""
276
if not self._contents.has_key(fileid):
278
return StringIO(self._contents[fileid])
280
def _item_size(self, fileid):
281
return len(self._contents[fileid])
284
return iter(self._contents.keys())
287
class RemoteStore(object):
289
def __init__(self, baseurl):
290
self._baseurl = baseurl
292
def _path(self, name):
294
raise ValueError('invalid store id', name)
295
return self._baseurl + '/' + name
297
def __getitem__(self, fileid):
298
p = self._path(fileid)
300
return get_url(p, compressed=True)
302
raise KeyError(fileid)
added
removed
bzrlib/store.py
