« back to all changes in this revision
Viewing changes to bzrlib/store.py
- browse files at revision 907.1.1
- compare with another revision
- compare with revision 5401
- download diff
- download tarball
- view history from revision 907.1.1
- Committer: John Arbash Meinel
- Date: 2005-07-11 16:59:50 UTC
- mto: (1185.11.1)
- mto: This revision was merged to the branch mainline in revision 1396.
- Revision ID: john@arbash-meinel.com-20050711165950-c98a663c84318ebc
Reworking the Branch and Store code to support an abstracted filesystem layer.
- files added:
- bzrlib/local_transport.py
- files modified:
- bzrlib/branch.py
added
removed
bzrlib/store.py
21
21
unique ID.
22
22
"""
23
23
24
import os, tempfile, types, osutils, gzip, errno
24
import os, tempfile, osutils, gzip, errno
25
25
from stat import ST_SIZE
26
26
from StringIO import StringIO
27
27
from trace import mutter
32
32
class StoreError(Exception):
33
33
pass
34
34
35
36
class ImmutableStore(object):
35
class Storage(object):
36
"""This class represents the abstract storage layout for saving information.
37
"""
38
def __init__(self, transport):
39
self._transport = transport
40
self._max_buffered_requests = 10
41
42
def __repr__(self):
43
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
44
45
__str__ == __repr__
46
47
def __len__(self):
48
raise NotImplementedError('Children should define their length')
49
50
def __getitem__(self, fileid):
51
"""Returns a file reading from a particular entry."""
52
raise NotImplementedError
53
54
def __contains__(self, fileid):
55
""""""
56
raise NotImplementedError
57
58
def __iter__(self):
59
raise NotImplementedError
60
61
def add(self, f, fileid):
62
"""Add a file object f to the store accessible from the given fileid"""
63
raise NotImplementedError('Children of Storage must define their method of adding entries.')
64
65
def copy_multi(self, other, ids):
66
"""Copy texts for ids from other into self.
67
68
If an id is present in self, it is skipped. A count of copied
69
ids is returned, which may be less than len(ids).
70
"""
71
from bzrlib.progress import ProgressBar
72
pb = ProgressBar()
73
pb.update('preparing to copy')
74
to_copy = [fileid for fileid in ids if fileid not in self]
75
return self._do_copy(other, to_copy, pb)
76
77
def _do_copy(self, other, to_copy, pb):
78
"""This is the standard copying mechanism, just get them one at
79
a time from remote, and store them locally.
80
"""
81
count = 0
82
buffered_requests = []
83
for fileid in to_copy:
84
buffered_requests.append((other[fileid], fileid))
85
if len(buffered_requests) > self._max_buffered_requests:
86
self.add(*buffered_requests.pop(0))
87
count += 1
88
pb.update('copy', count, len(to_copy))
89
90
for req in buffered_requests:
91
self.add(*req)
92
count += 1
93
pb.update('copy', count, len(to_copy))
94
95
assert count == len(to_copy)
96
pb.clear()
97
return count
98
99
100
101
class CompressedTextStore(Storage):
37
102
"""Store that holds files indexed by unique names.
38
103
39
104
Files can be added, but not modified once they are in. Typically
40
105
the hash is used as the name, or something else known to be unique,
41
106
such as a UUID.
42
107
43
>>> st = ImmutableScratchStore()
108
Files are stored gzip compressed, with no delta compression.
109
110
>>> st = ScratchFlatTextStore()
44
111
45
112
>>> st.add(StringIO('hello'), 'aa')
46
113
>>> 'aa' in st
64
131
"""
65
132
66
133
def __init__(self, basedir):
67
self._basedir = basedir
134
super(CompressedTextStore, self).__init__(basedir)
68
135
69
def _path(self, id):
70
if '\\' in id or '/' in id:
71
raise ValueError("invalid store id %r" % id)
72
return os.path.join(self._basedir, id)
136
def _path(self, fileid):
137
if '\\' in fileid or '/' in fileid:
138
raise ValueError("invalid store id %r" % fileid)
139
return self._transport.get_filename(fileid)
73
140
74
141
def __repr__(self):
75
return "%s(%r)" % (self.__class__.__name__, self._basedir)
142
return "%s(%r)" % (self.__class__.__name__, self._location)
76
143
77
144
def add(self, f, fileid, compressed=True):
78
145
"""Add contents of a file into the store.
80
147
f -- An open file, or file-like object."""
81
148
# FIXME: Only works on files that will fit in memory
82
149
83
from bzrlib.atomicfile import AtomicFile
150
from cStringIO import StringIO
84
151
85
152
mutter("add store entry %r" % (fileid))
86
if isinstance(f, types.StringTypes):
153
if isinstance(f, basestring):
87
154
content = f
88
155
else:
89
156
content = f.read()
90
157
91
p = self._path(fileid)
92
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
93
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
158
if self._transport.has(fileid) or self._transport.has(fileid + '.gz'):
159
raise BzrError("store %r already contains id %r" % (self._location, fileid))
94
160
95
fn = p
161
fn = fileid
96
162
if compressed:
97
163
fn = fn + '.gz'
98
164
99
af = AtomicFile(fn, 'wb')
100
try:
101
if compressed:
102
gf = gzip.GzipFile(mode='wb', fileobj=af)
103
gf.write(content)
104
gf.close()
105
else:
106
af.write(content)
107
af.commit()
108
finally:
109
af.close()
110
111
112
def copy_multi(self, other, ids):
113
"""Copy texts for ids from other into self.
114
115
If an id is present in self, it is skipped. A count of copied
116
ids is returned, which may be less than len(ids).
117
"""
118
from bzrlib.progress import ProgressBar
119
pb = ProgressBar()
120
pb.update('preparing to copy')
121
to_copy = [id for id in ids if id not in self]
122
if isinstance(other, ImmutableStore):
123
return self.copy_multi_immutable(other, to_copy, pb)
124
count = 0
125
for id in to_copy:
126
count += 1
127
pb.update('copy', count, len(to_copy))
128
self.add(other[id], id)
129
assert count == len(to_copy)
130
pb.clear()
131
return count
132
133
134
def copy_multi_immutable(self, other, to_copy, pb):
165
sio = StringIO()
166
if compressed:
167
gf = gzip.GzipFile(mode='wb', fileobj=sio)
168
gf.write(content)
169
gf.close()
170
else:
171
sio.write(content)
172
sio.seek(0)
173
self._transport.put(fn, sio)
174
175
def _do_copy(self, other, to_copy, pb):
176
if isinstance(other, CompressedTextStore):
177
return self._copy_multi_text(other, to_copy, pb)
178
return super(CompressedTextStore, self)._do_copy(other, to_copy, pb)
179
180
181
def _copy_multi_text(self, other, to_copy, pb):
135
182
from shutil import copyfile
136
183
count = 0
137
184
for id in to_copy:
161
208
# TODO: Guard against the same thing being stored twice, compressed and uncompresse
162
209
163
210
def __iter__(self):
164
for f in os.listdir(self._basedir):
211
for f in os.listdir(self._location):
165
212
if f[-3:] == '.gz':
166
213
# TODO: case-insensitive?
167
214
yield f[:-3]
169
216
yield f
170
217
171
218
def __len__(self):
172
return len(os.listdir(self._basedir))
219
return len(os.listdir(self._location))
173
220
174
221
def __getitem__(self, fileid):
175
222
"""Returns a file reading from a particular entry."""
202
249
203
250
204
251
205
class ImmutableScratchStore(ImmutableStore):
252
class ScratchFlatTextStore(CompressedTextStore):
206
253
"""Self-destructing test subclass of ImmutableStore.
207
254
208
255
The Store only exists for the lifetime of the Python object.
209
Obviously you should not put anything precious in it.
256
Obviously you should not put anything precious in it.
210
257
"""
211
258
def __init__(self):
212
ImmutableStore.__init__(self, tempfile.mkdtemp())
259
super(ScratchFlatTextStore, self).__init__(tempfile.mkdtemp())
213
260
214
261
def __del__(self):
215
for f in os.listdir(self._basedir):
216
fpath = os.path.join(self._basedir, f)
262
for f in os.listdir(self._location):
263
fpath = os.path.join(self._location, f)
217
264
# needed on windows, and maybe some other filesystems
218
265
os.chmod(fpath, 0600)
219
266
os.remove(fpath)
220
os.rmdir(self._basedir)
267
os.rmdir(self._location)
221
268
mutter("%r destroyed" % self)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1