15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
import stat, os, sha, time
18
from binascii import b2a_qp, a2b_qp
20
19
from trace import mutter
21
from errors import BzrError
20
from errors import BzrError, BzrCheckError
24
23
"""File stat cache to speed up tree comparisons.
27
26
information of a file in the working directory, without actually
28
27
reading and hashing the whole file.
34
Users of this module should not need to know about how this is
35
implemented, and in particular should not depend on the particular
36
data which is stored or its format.
30
38
This is done by maintaining a cache indexed by a file fingerprint of
31
39
(path, size, mtime, ctime, ino, dev) pointing to the SHA-1. If the
32
40
fingerprint has changed, we assume the file content has not changed
50
58
to gradually adjust your clock or don't use bzr over the step.
52
60
At the moment this is stored in a simple textfile; it might be nice
61
to use a tdb instead to allow faster lookup by file-id.
55
63
The cache is represented as a map from file_id to a tuple of (file_id,
56
64
sha1, path, size, mtime, ctime, ino, dev).
66
The SHA-1 is stored in memory as a hexdigest.
68
This version of the file on disk has one line per record, and fields
69
separated by \0 records.
72
# order of fields returned by fingerprint()
79
# order of fields in the statcache file and in the in-memory map
91
CACHE_HEADER = "### bzr statcache v4"
71
94
def fingerprint(abspath):
82
105
fs.st_ctime, fs.st_ino, fs.st_dev)
85
def _write_cache(basedir, entry_iter, dangerfiles):
109
def _write_cache(basedir, entries):
86
110
from atomicfile import AtomicFile
88
112
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
89
outf = AtomicFile(cachefn, 'wb', 'utf-8')
113
outf = AtomicFile(cachefn, 'wb')
91
for entry in entry_iter:
92
if entry[0] in dangerfiles:
94
outf.write(entry[0] + ' ' + entry[1] + ' ')
95
outf.write(b2a_qp(entry[2], True))
96
outf.write(' %d %d %d %d %d\n' % entry[3:])
115
outf.write(CACHE_HEADER + '\n')
117
for entry in entries:
119
raise ValueError("invalid statcache entry tuple %r" % entry)
120
outf.write(entry[0].encode('utf-8')) # file id
122
outf.write(entry[1]) # hex sha1
124
outf.write(entry[2].encode('utf-8')) # name
126
outf.write('\0%d' % nf)
100
131
if not outf.closed:
135
def _try_write_cache(basedir, entries):
137
return _write_cache(basedir, entries)
139
mutter("cannot update statcache in %s: %s" % (basedir, e))
141
mutter("cannot update statcache in %s: %s" % (basedir, e))
104
145
def load_cache(basedir):
149
from bzrlib.trace import warning
151
assert isinstance(basedir, basestring)
153
sha_re = re.compile(r'[a-f0-9]{40}')
110
156
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
111
cachefile = codecs.open(cachefn, 'r', 'utf-8')
157
cachefile = open(cachefn, 'rb')
161
line1 = cachefile.readline().rstrip('\r\n')
162
if line1 != CACHE_HEADER:
163
mutter('cache header marker not found at top of %s; discarding cache'
115
167
for l in cachefile:
170
file_id = f[0].decode('utf-8')
118
171
if file_id in cache:
119
raise BzrError("duplicated file_id in cache: {%s}" % file_id)
120
cache[file_id] = (f[0], f[1], a2b_qp(f[2])) + tuple([long(x) for x in f[3:]])
172
warning("duplicated file_id in cache: {%s}" % file_id)
175
if len(text_sha) != 40 or not sha_re.match(text_sha):
176
raise BzrCheckError("invalid file SHA-1 in cache: %r" % text_sha)
178
path = f[2].decode('utf-8')
179
if path in seen_paths:
180
warning("duplicated path in cache: %r" % path)
181
seen_paths[path] = True
183
entry = (file_id, text_sha, path) + tuple([long(x) for x in f[3:]])
185
raise ValueError("invalid statcache entry tuple %r" % entry)
187
cache[file_id] = entry
126
192
def _files_from_inventory(inv):
127
193
for path, ie in inv.iter_entries():
128
194
if ie.kind != 'file':
140
206
flush -- discard any previous cache and recalculate from scratch.
209
# load the existing cache; use information there to find a list of
210
# files ordered by inode, which is alleged to be the fastest order
144
# TODO: It's supposed to be faster to stat the files in order by inum.
145
# We don't directly know the inum of the files of course but we do
146
# know where they were last sighted, so we can sort by that.
213
to_update = _files_from_inventory(inv)
148
215
assert isinstance(flush, bool)
152
219
cache = load_cache(basedir)
153
return _update_cache_from_list(basedir, cache, _files_from_inventory(inv))
157
def _update_cache_from_list(basedir, cache, to_update):
158
"""Update and return the cache for given files.
160
cache -- Previously cached values to be validated.
162
to_update -- Sequence of (file_id, path) pairs to check.
167
stat_cnt = missing_cnt = hardcheck = change_cnt = 0
169
# files that have been recently touched and can't be
170
# committed to a persistent cache yet.
223
for file_id, path in to_update:
225
by_inode.append((cache[file_id][SC_INO], file_id, path))
227
without_inode.append((file_id, path))
230
to_update = [a[1:] for a in by_inode] + without_inode
232
stat_cnt = missing_cnt = new_cnt = hardcheck = change_cnt = 0
234
# dangerfiles have been recently touched and can't be committed to
235
# a persistent cache yet, but they are returned to the caller.
173
238
now = int(time.time())
175
240
## mutter('update statcache under %r' % basedir)
198
265
dig = sha.new(file(abspath, 'rb').read()).hexdigest()
200
if cacheentry == None or dig != cacheentry[1]:
201
# if there was no previous entry for this file, or if the
202
# SHA has changed, then update the cache
203
cacheentry = (file_id, dig, path) + fp
204
cache[file_id] = cacheentry
267
# We update the cache even if the digest has not changed from
268
# last time we looked, so that the fingerprint fields will
270
cacheentry = (file_id, dig, path) + fp
271
cache[file_id] = cacheentry
207
274
mutter('statcache: statted %d files, read %d files, %d changed, %d dangerous, '
275
'%d deleted, %d new, '
209
% (stat_cnt, hardcheck, change_cnt, len(dangerfiles), len(cache)))
277
% (stat_cnt, hardcheck, change_cnt, len(dangerfiles),
278
missing_cnt, new_cnt, len(cache)))
212
281
mutter('updating on-disk statcache')
213
_write_cache(basedir, cache.itervalues(), dangerfiles)
284
safe_cache = cache.copy()
285
for file_id in dangerfiles:
286
del safe_cache[file_id]
290
_try_write_cache(basedir, safe_cache.itervalues())
added
removed
bzrlib/statcache.py
