14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Stores are the main data-storage mechanism for Bazaar-NG.
17
# TODO: Could remember a bias towards whether a particular store is typically
21
Stores are the main data-storage mechanism for Bazaar-NG.
19
23
A store is a simple write-once container indexed by a universally
20
unique ID, which is typically the SHA-1 of the content."""
22
__copyright__ = "Copyright (C) 2005 Canonical Ltd."
23
__author__ = "Martin Pool <mbp@canonical.com>"
25
import os, tempfile, types, osutils, gzip, errno
26
32
from stat import ST_SIZE
27
33
from StringIO import StringIO
28
from trace import mutter
35
from bzrlib.errors import BzrError, UnlistableStore
36
from bzrlib.trace import mutter
38
import bzrlib.osutils as osutils
40
#from bzrlib.remotebranch import get_url
30
44
######################################################################
56
70
>>> st.add(StringIO('goodbye'), '123123')
57
71
>>> st['123123'].read()
76
"""Return (count, bytes)
78
This is the (compressed) size stored on disk, not the size of
84
total += self._item_size(fid)
88
class ImmutableStore(Store):
89
"""Store that stores files on disk.
60
91
TODO: Atomic add by writing to a temporary file and renaming.
62
TODO: Perhaps automatically transform to/from XML in a method?
63
Would just need to tell the constructor what class to
66
TODO: Even within a simple disk store like this, we could
67
gzip the files. But since many are less than one disk
68
block, that might not help a lot.
92
TODO: Guard against the same thing being stored twice, compressed and
93
uncompressed during copy_multi_immutable - the window is for a
94
matching store with some crack code that lets it offer a
95
non gz FOO and then a fz FOO.
97
In bzr 0.0.5 and earlier, files within the store were marked
98
readonly on disk. This is no longer done but existing stores need
72
102
def __init__(self, basedir):
73
"""ImmutableStore constructor."""
103
super(ImmutableStore, self).__init__()
74
104
self._basedir = basedir
77
return os.path.join(self._basedir, id)
106
def _path(self, entry_id):
107
if not isinstance(entry_id, basestring):
108
raise TypeError(type(entry_id))
109
if '\\' in entry_id or '/' in entry_id:
110
raise ValueError("invalid store id %r" % entry_id)
111
return os.path.join(self._basedir, entry_id)
79
113
def __repr__(self):
80
114
return "%s(%r)" % (self.__class__.__name__, self._basedir)
83
117
"""Add contents of a file into the store.
85
119
f -- An open file, or file-like object."""
86
# FIXME: Only works on smallish files
87
# TODO: Can be optimized by copying at the same time as
120
# FIXME: Only works on files that will fit in memory
122
from bzrlib.atomicfile import AtomicFile
89
124
mutter("add store entry %r" % (fileid))
90
125
if isinstance(f, types.StringTypes):
93
128
content = f.read()
95
130
p = self._path(fileid)
96
131
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
97
bailout("store %r already contains id %r" % (self._basedir, fileid))
132
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
100
f = gzip.GzipFile(p + '.gz', 'wb')
101
os.chmod(p + '.gz', 0444)
138
af = AtomicFile(fn, 'wb')
141
gf = gzip.GzipFile(mode='wb', fileobj=af)
151
def copy_multi(self, other, ids, permit_failure=False):
152
"""Copy texts for ids from other into self.
154
If an id is present in self, it is skipped.
156
Returns (count_copied, failed), where failed is a collection of ids
157
that could not be copied.
159
pb = bzrlib.ui.ui_factory.progress_bar()
161
pb.update('preparing to copy')
162
to_copy = [id for id in ids if id not in self]
163
if isinstance(other, ImmutableStore):
164
return self.copy_multi_immutable(other, to_copy, pb,
165
permit_failure=permit_failure)
170
pb.update('copy', count, len(to_copy))
171
if not permit_failure:
172
self.add(other[id], id)
181
if not permit_failure:
182
assert count == len(to_copy)
186
def copy_multi_immutable(self, other, to_copy, pb, permit_failure=False):
191
other_p = other._path(id)
193
osutils.link_or_copy(other_p, p)
194
except (IOError, OSError), e:
195
if e.errno == errno.ENOENT:
196
if not permit_failure:
197
osutils.link_or_copy(other_p+".gz", p+".gz")
200
osutils.link_or_copy(other_p+".gz", p+".gz")
202
if e.errno == errno.ENOENT:
210
pb.update('copy', count, len(to_copy))
211
assert count == len(to_copy)
110
215
def __contains__(self, fileid):
113
218
return (os.access(p, os.R_OK)
114
219
or os.access(p + '.gz', os.R_OK))
116
# TODO: Guard against the same thing being stored twice, compressed and uncompresse
221
def _item_size(self, fid):
224
return os.stat(p)[ST_SIZE]
226
return os.stat(p + '.gz')[ST_SIZE]
228
# TODO: Guard against the same thing being stored twice,
229
# compressed and uncompressed
118
231
def __iter__(self):
119
232
for f in os.listdir(self._basedir):
133
246
return gzip.GzipFile(p + '.gz', 'rb')
134
247
except IOError, e:
135
if e.errno == errno.ENOENT:
140
def total_size(self):
141
"""Return (count, bytes)
143
This is the (compressed) size stored on disk, not the size of
151
total += os.stat(p)[ST_SIZE]
153
total += os.stat(p + '.gz')[ST_SIZE]
248
if e.errno != errno.ENOENT:
254
if e.errno != errno.ENOENT:
257
raise KeyError(fileid)
160
260
class ImmutableScratchStore(ImmutableStore):
161
261
"""Self-destructing test subclass of ImmutableStore.
163
263
The Store only exists for the lifetime of the Python object.
164
Obviously you should not put anything precious in it.
264
Obviously you should not put anything precious in it.
166
266
def __init__(self):
167
ImmutableStore.__init__(self, tempfile.mkdtemp())
267
super(ImmutableScratchStore, self).__init__(tempfile.mkdtemp())
169
269
def __del__(self):
170
270
for f in os.listdir(self._basedir):
175
275
os.rmdir(self._basedir)
176
276
mutter("%r destroyed" % self)
279
class ImmutableMemoryStore(Store):
280
"""A memory only store."""
283
super(ImmutableMemoryStore, self).__init__()
286
def add(self, stream, fileid, compressed=True):
287
if self._contents.has_key(fileid):
288
raise StoreError("fileid %s already in the store" % fileid)
289
self._contents[fileid] = stream.read()
291
def __getitem__(self, fileid):
292
"""Returns a file reading from a particular entry."""
293
if not self._contents.has_key(fileid):
295
return StringIO(self._contents[fileid])
297
def _item_size(self, fileid):
298
return len(self._contents[fileid])
301
return iter(self._contents.keys())
304
class RemoteStore(object):
306
def __init__(self, baseurl):
307
self._baseurl = baseurl
309
def _path(self, name):
311
raise ValueError('invalid store id', name)
312
return self._baseurl + '/' + name
314
def __getitem__(self, fileid):
316
from bzrlib.remotebranch import get_url
317
p = self._path(fileid)
319
return get_url(p, compressed=True)
320
except urllib2.URLError:
323
return get_url(p, compressed=False)
324
except urllib2.URLError:
325
raise KeyError(fileid)
327
def __contains__(self, fileid):
336
"""A store that caches data locally, to avoid repeated downloads.
337
The precacache method should be used to avoid server round-trips for
341
def __init__(self, store, cache_dir):
342
self.source_store = store
343
self.cache_store = ImmutableStore(cache_dir)
345
def __getitem__(self, id):
346
mutter("Cache add %s" % id)
347
if id not in self.cache_store:
348
self.cache_store.add(self.source_store[id], id)
349
return self.cache_store[id]
351
def prefetch(self, ids):
352
"""Copy a series of ids into the cache, before they are used.
353
For remote stores that support pipelining or async downloads, this can
354
increase speed considerably.
355
Failures while prefetching are ignored.
357
mutter("Prefetch of ids %s" % ",".join(ids))
358
self.cache_store.copy_multi(self.source_store, ids,
362
def copy_all(store_from, store_to):
363
"""Copy all ids from one store to another."""
364
if not hasattr(store_from, "__iter__"):
365
raise UnlistableStore(store_from)
366
ids = [f for f in store_from]
367
store_to.copy_multi(store_from, ids)
added
removed
bzrlib/store.py
