1
# Copyright (C) 2005, 2006 by Canonical Ltd
1
# Copyright (C) 2005, 2006 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
30
30
CACHE_HEADER = "### bzr hashcache v5\n"
32
32
import os, stat, time
35
from bzrlib.osutils import sha_file, pathjoin, safe_unicode
34
from bzrlib.osutils import sha_file, sha_string, pathjoin, safe_unicode
36
35
from bzrlib.trace import mutter, warning
37
36
from bzrlib.atomicfile import AtomicFile
38
37
from bzrlib.errors import BzrError
42
41
FP_CTIME_COLUMN = 2
45
def _fingerprint(abspath):
47
fs = os.lstat(abspath)
49
# might be missing, etc
52
if stat.S_ISDIR(fs.st_mode):
55
# we discard any high precision because it's not reliable; perhaps we
56
# could do better on some systems?
57
return (fs.st_size, long(fs.st_mtime),
58
long(fs.st_ctime), fs.st_ino, fs.st_dev, fs.st_mode)
61
46
class HashCache(object):
97
82
def __init__(self, root, cache_file_name, mode=None):
98
83
"""Create a hash cache in base dir, and set the file mode to mode."""
99
84
self.root = safe_unicode(root)
85
self.root_utf8 = self.root.encode('utf8') # where is the filesystem encoding ?
100
86
self.hit_count = 0
101
87
self.miss_count = 0
102
88
self.stat_count = 0
132
118
for inum, path, cache_entry in prep:
133
119
abspath = pathjoin(self.root, path)
134
fp = _fingerprint(abspath)
120
fp = self._fingerprint(abspath)
135
121
self.stat_count += 1
137
123
cache_fp = cache_entry[1]
142
128
self.needs_write = True
143
129
del self._cache[path]
146
def get_sha1(self, path):
131
def get_sha1(self, path, stat_value=None):
147
132
"""Return the sha1 of a file.
149
abspath = pathjoin(self.root, path)
134
if path.__class__ is str:
135
abspath = pathjoin(self.root_utf8, path)
137
abspath = pathjoin(self.root, path)
150
138
self.stat_count += 1
151
file_fp = _fingerprint(abspath)
139
file_fp = self._fingerprint(abspath, stat_value)
154
142
# not a regular file or not existing
164
152
cache_sha1, cache_fp = None, None
166
154
if cache_fp == file_fp:
155
## mutter("hashcache hit for %s %r -> %s", path, file_fp, cache_sha1)
156
## mutter("now = %s", time.time())
167
157
self.hit_count += 1
168
158
return cache_sha1
170
160
self.miss_count += 1
173
162
mode = file_fp[FP_MODE_COLUMN]
174
163
if stat.S_ISREG(mode):
175
digest = sha_file(file(abspath, 'rb', buffering=65000))
164
digest = self._really_sha1_file(abspath)
176
165
elif stat.S_ISLNK(mode):
177
digest = sha.new(os.readlink(abspath)).hexdigest()
166
digest = sha_string(os.readlink(abspath))
179
168
raise BzrError("file %r: unknown file stat mode: %o"%(abspath,mode))
181
now = int(time.time())
182
if file_fp[FP_MTIME_COLUMN] >= now or file_fp[FP_CTIME_COLUMN] >= now:
170
# window of 3 seconds to allow for 2s resolution on windows,
171
# unsynchronized file servers, etc.
172
cutoff = self._cutoff_time()
173
if file_fp[FP_MTIME_COLUMN] >= cutoff \
174
or file_fp[FP_CTIME_COLUMN] >= cutoff:
183
175
# changed too recently; can't be cached. we can
184
176
# return the result and it could possibly be cached
191
183
# need to let sufficient time elapse before we may cache this entry
192
184
# again. If we didn't do this, then, for example, a very quick 1
193
185
# byte replacement in the file might go undetected.
194
self.danger_count += 1
186
## mutter('%r modified too recently; not caching', path)
187
self.danger_count += 1
196
189
self.removed_count += 1
197
190
self.needs_write = True
198
191
del self._cache[path]
193
## mutter('%r added to cache: now=%f, mtime=%d, ctime=%d',
194
## path, time.time(), file_fp[FP_MTIME_COLUMN],
195
## file_fp[FP_CTIME_COLUMN])
200
196
self.update_count += 1
201
197
self.needs_write = True
202
198
self._cache[path] = (digest, file_fp)
201
def _really_sha1_file(self, abspath):
202
"""Calculate the SHA1 of a file by reading the full text"""
203
return sha_file(file(abspath, 'rb', buffering=65000))
206
206
"""Write contents of cache to file."""
207
207
outf = AtomicFile(self.cache_file_name(), 'wb', new_mode=self._mode)
209
print >>outf, CACHE_HEADER,
209
outf.write(CACHE_HEADER)
211
211
for path, c in self._cache.iteritems():
212
assert '//' not in path, path
213
outf.write(path.encode('utf-8'))
215
print >>outf, c[0], # hex sha1
217
print >>outf, "%d" % fld,
212
line_info = [path.encode('utf-8'), '// ', c[0], ' ']
213
line_info.append(' '.join([str(fld) for fld in c[1]]))
214
line_info.append('\n')
215
outf.write(''.join(line_info))
220
217
self.needs_write = False
221
mutter("write hash cache: %s hits=%d misses=%d stat=%d recent=%d updates=%d",
222
self.cache_file_name(), self.hit_count, self.miss_count,
224
self.danger_count, self.update_count)
218
## mutter("write hash cache: %s hits=%d misses=%d stat=%d recent=%d updates=%d",
219
## self.cache_file_name(), self.hit_count, self.miss_count,
221
## self.danger_count, self.update_count)
230
226
"""Reinstate cache from file.
275
270
self._cache[path] = (sha1, fp)
277
272
self.needs_write = False
274
def _cutoff_time(self):
275
"""Return cutoff time.
277
Files modified more recently than this time are at risk of being
278
undetectably modified and so can't be cached.
280
return int(time.time()) - 3
282
def _fingerprint(self, abspath, stat_value=None):
283
if stat_value is None:
285
stat_value = os.lstat(abspath)
287
# might be missing, etc
289
if stat.S_ISDIR(stat_value.st_mode):
291
# we discard any high precision because it's not reliable; perhaps we
292
# could do better on some systems?
293
return (stat_value.st_size, long(stat_value.st_mtime),
294
long(stat_value.st_ctime), stat_value.st_ino,
295
stat_value.st_dev, stat_value.st_mode)