1
# (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
import stat, os, sha, time
19
from trace import mutter
20
from errors import BzrError, BzrCheckError
23
"""File stat cache to speed up tree comparisons.
25
This module basically gives a quick way to find the SHA-1 and related
26
information of a file in the working directory, without actually
27
reading and hashing the whole file.
34
Users of this module should not need to know about how this is
35
implemented, and in particular should not depend on the particular
36
data which is stored or its format.
38
This is done by maintaining a cache indexed by a file fingerprint of
39
(path, size, mtime, ctime, ino, dev) pointing to the SHA-1. If the
40
fingerprint has changed, we assume the file content has not changed
41
either and the SHA-1 is therefore the same.
43
If any of the fingerprint fields have changed then the file content
44
*may* have changed, or it may not have. We need to reread the file
45
contents to make sure, but this is not visible to the user or
46
higher-level code (except as a delay of course).
48
The mtime and ctime are stored with nanosecond fields, but not all
49
filesystems give this level of precision. There is therefore a
50
possible race: the file might be modified twice within a second
51
without changing the size or mtime, and a SHA-1 cached from the first
52
version would be wrong. We handle this by not recording a cached hash
53
for any files which were modified in the current second and that
54
therefore have the chance to change again before the second is up.
56
The only known hole in this design is if the system clock jumps
57
backwards crossing invocations of bzr. Please don't do that; use ntp
58
to gradually adjust your clock or don't use bzr over the step.
60
At the moment this is stored in a simple textfile; it might be nice
61
to use a tdb instead to allow faster lookup by file-id.
63
The cache is represented as a map from file_id to a tuple of (file_id,
64
sha1, path, size, mtime, ctime, ino, dev).
66
The SHA-1 is stored in memory as a hexdigest.
68
This version of the file on disk has one line per record, and fields
69
separated by \0 records.
72
# order of fields returned by fingerprint()
79
# order of fields in the statcache file and in the in-memory map
91
CACHE_HEADER = "### bzr statcache v4"
94
def fingerprint(abspath):
96
fs = os.lstat(abspath)
98
# might be missing, etc
101
if stat.S_ISDIR(fs.st_mode):
104
return (fs.st_size, fs.st_mtime,
105
fs.st_ctime, fs.st_ino, fs.st_dev)
109
def _write_cache(basedir, entries):
110
from atomicfile import AtomicFile
112
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
113
outf = AtomicFile(cachefn, 'wb')
115
outf.write(CACHE_HEADER + '\n')
117
for entry in entries:
119
raise ValueError("invalid statcache entry tuple %r" % entry)
120
outf.write(entry[0].encode('utf-8')) # file id
122
outf.write(entry[1]) # hex sha1
124
outf.write(entry[2].encode('utf-8')) # name
126
outf.write('\0%d' % nf)
135
def _try_write_cache(basedir, entries):
137
return _write_cache(basedir, entries)
139
mutter("cannot update statcache in %s: %s" % (basedir, e))
141
mutter("cannot update statcache in %s: %s" % (basedir, e))
145
def load_cache(basedir):
149
from bzrlib.trace import warning
151
sha_re = re.compile(r'[a-f0-9]{40}')
154
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
155
cachefile = open(cachefn, 'rb')
159
line1 = cachefile.readline().rstrip('\r\n')
160
if line1 != CACHE_HEADER:
161
mutter('cache header marker not found at top of %s; discarding cache'
168
file_id = f[0].decode('utf-8')
170
warning("duplicated file_id in cache: {%s}" % file_id)
173
if len(text_sha) != 40 or not sha_re.match(text_sha):
174
raise BzrCheckError("invalid file SHA-1 in cache: %r" % text_sha)
176
path = f[2].decode('utf-8')
177
if path in seen_paths:
178
warning("duplicated path in cache: %r" % path)
179
seen_paths[path] = True
181
entry = (file_id, text_sha, path) + tuple([long(x) for x in f[3:]])
183
raise ValueError("invalid statcache entry tuple %r" % entry)
185
cache[file_id] = entry
190
def _files_from_inventory(inv):
191
for path, ie in inv.iter_entries():
192
if ie.kind != 'file':
194
yield ie.file_id, path
198
def update_cache(basedir, inv, flush=False):
199
"""Update and return the cache for the branch.
201
The returned cache may contain entries that have not been written
202
to disk for files recently touched.
204
flush -- discard any previous cache and recalculate from scratch.
207
# load the existing cache; use information there to find a list of
208
# files ordered by inode, which is alleged to be the fastest order
211
to_update = _files_from_inventory(inv)
213
assert isinstance(flush, bool)
217
cache = load_cache(basedir)
221
for file_id, path in to_update:
223
by_inode.append((cache[file_id][SC_INO], file_id, path))
225
without_inode.append((file_id, path))
228
to_update = [a[1:] for a in by_inode] + without_inode
230
stat_cnt = missing_cnt = new_cnt = hardcheck = change_cnt = 0
232
# dangerfiles have been recently touched and can't be committed to
233
# a persistent cache yet, but they are returned to the caller.
236
now = int(time.time())
238
## mutter('update statcache under %r' % basedir)
239
for file_id, path in to_update:
240
abspath = os.path.join(basedir, path)
241
fp = fingerprint(abspath)
244
cacheentry = cache.get(file_id)
246
if fp == None: # not here
255
if (fp[FP_MTIME] >= now) or (fp[FP_CTIME] >= now):
256
dangerfiles.append(file_id)
258
if cacheentry and (cacheentry[3:] == fp):
259
continue # all stat fields unchanged
263
dig = sha.new(file(abspath, 'rb').read()).hexdigest()
265
# We update the cache even if the digest has not changed from
266
# last time we looked, so that the fingerprint fields will
268
cacheentry = (file_id, dig, path) + fp
269
cache[file_id] = cacheentry
272
mutter('statcache: statted %d files, read %d files, %d changed, %d dangerous, '
273
'%d deleted, %d new, '
275
% (stat_cnt, hardcheck, change_cnt, len(dangerfiles),
276
missing_cnt, new_cnt, len(cache)))
279
mutter('updating on-disk statcache')
282
safe_cache = cache.copy()
283
for file_id in dangerfiles:
284
del safe_cache[file_id]
288
_try_write_cache(basedir, safe_cache.itervalues())
added
removed
bzrlib/statcache.py
