1
# Copyright (C) 2005, 2006 Canonical Ltd
1
# Copyright (C) 2005, 2006 by Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
42
42
FP_CTIME_COLUMN = 2
45
def _fingerprint(abspath):
47
fs = os.lstat(abspath)
49
# might be missing, etc
52
if stat.S_ISDIR(fs.st_mode):
55
# we discard any high precision because it's not reliable; perhaps we
56
# could do better on some systems?
57
return (fs.st_size, long(fs.st_mtime),
58
long(fs.st_ctime), fs.st_ino, fs.st_dev, fs.st_mode)
47
61
class HashCache(object):
83
97
def __init__(self, root, cache_file_name, mode=None):
84
98
"""Create a hash cache in base dir, and set the file mode to mode."""
85
99
self.root = safe_unicode(root)
86
self.root_utf8 = self.root.encode('utf8') # where is the filesystem encoding ?
87
100
self.hit_count = 0
88
101
self.miss_count = 0
89
102
self.stat_count = 0
119
132
for inum, path, cache_entry in prep:
120
133
abspath = pathjoin(self.root, path)
121
fp = self._fingerprint(abspath)
134
fp = _fingerprint(abspath)
122
135
self.stat_count += 1
124
137
cache_fp = cache_entry[1]
129
142
self.needs_write = True
130
143
del self._cache[path]
132
def get_sha1(self, path, stat_value=None):
146
def get_sha1(self, path):
133
147
"""Return the sha1 of a file.
135
if path.__class__ is str:
136
abspath = pathjoin(self.root_utf8, path)
138
abspath = pathjoin(self.root, path)
149
abspath = pathjoin(self.root, path)
139
150
self.stat_count += 1
140
file_fp = self._fingerprint(abspath, stat_value)
151
file_fp = _fingerprint(abspath)
143
154
# not a regular file or not existing
153
164
cache_sha1, cache_fp = None, None
155
166
if cache_fp == file_fp:
156
## mutter("hashcache hit for %s %r -> %s", path, file_fp, cache_sha1)
157
## mutter("now = %s", time.time())
158
167
self.hit_count += 1
159
168
return cache_sha1
161
170
self.miss_count += 1
163
173
mode = file_fp[FP_MODE_COLUMN]
164
174
if stat.S_ISREG(mode):
165
digest = self._really_sha1_file(abspath)
175
digest = sha_file(file(abspath, 'rb', buffering=65000))
166
176
elif stat.S_ISLNK(mode):
167
177
digest = sha.new(os.readlink(abspath)).hexdigest()
169
179
raise BzrError("file %r: unknown file stat mode: %o"%(abspath,mode))
171
# window of 3 seconds to allow for 2s resolution on windows,
172
# unsynchronized file servers, etc.
173
cutoff = self._cutoff_time()
174
if file_fp[FP_MTIME_COLUMN] >= cutoff \
175
or file_fp[FP_CTIME_COLUMN] >= cutoff:
181
now = int(time.time())
182
if file_fp[FP_MTIME_COLUMN] >= now or file_fp[FP_CTIME_COLUMN] >= now:
176
183
# changed too recently; can't be cached. we can
177
184
# return the result and it could possibly be cached
184
191
# need to let sufficient time elapse before we may cache this entry
185
192
# again. If we didn't do this, then, for example, a very quick 1
186
193
# byte replacement in the file might go undetected.
187
## mutter('%r modified too recently; not caching', path)
188
self.danger_count += 1
194
self.danger_count += 1
190
196
self.removed_count += 1
191
197
self.needs_write = True
192
198
del self._cache[path]
194
## mutter('%r added to cache: now=%f, mtime=%d, ctime=%d',
195
## path, time.time(), file_fp[FP_MTIME_COLUMN],
196
## file_fp[FP_CTIME_COLUMN])
197
200
self.update_count += 1
198
201
self.needs_write = True
199
202
self._cache[path] = (digest, file_fp)
202
def _really_sha1_file(self, abspath):
203
"""Calculate the SHA1 of a file by reading the full text"""
204
return sha_file(file(abspath, 'rb', buffering=65000))
207
206
"""Write contents of cache to file."""
208
207
outf = AtomicFile(self.cache_file_name(), 'wb', new_mode=self._mode)
210
outf.write(CACHE_HEADER)
209
print >>outf, CACHE_HEADER,
212
211
for path, c in self._cache.iteritems():
213
212
assert '//' not in path, path
214
line_info = [path.encode('utf-8'), '// ', c[0], ' ']
215
line_info.append(' '.join([str(fld) for fld in c[1]]))
216
line_info.append('\n')
217
outf.write(''.join(line_info))
213
outf.write(path.encode('utf-8'))
215
print >>outf, c[0], # hex sha1
217
print >>outf, "%d" % fld,
219
220
self.needs_write = False
220
## mutter("write hash cache: %s hits=%d misses=%d stat=%d recent=%d updates=%d",
221
## self.cache_file_name(), self.hit_count, self.miss_count,
223
## self.danger_count, self.update_count)
221
mutter("write hash cache: %s hits=%d misses=%d stat=%d recent=%d updates=%d",
222
self.cache_file_name(), self.hit_count, self.miss_count,
224
self.danger_count, self.update_count)
228
230
"""Reinstate cache from file.
272
275
self._cache[path] = (sha1, fp)
274
277
self.needs_write = False
276
def _cutoff_time(self):
277
"""Return cutoff time.
279
Files modified more recently than this time are at risk of being
280
undetectably modified and so can't be cached.
282
return int(time.time()) - 3
284
def _fingerprint(self, abspath, stat_value=None):
285
if stat_value is None:
287
stat_value = os.lstat(abspath)
289
# might be missing, etc
291
if stat.S_ISDIR(stat_value.st_mode):
293
# we discard any high precision because it's not reliable; perhaps we
294
# could do better on some systems?
295
return (stat_value.st_size, long(stat_value.st_mtime),
296
long(stat_value.st_ctime), stat_value.st_ino,
297
stat_value.st_dev, stat_value.st_mode)
added
removed
bzrlib/hashcache.py
