1
# Copyright (C) 2005, 2006 by Canonical Ltd
1
# Copyright (C) 2005, 2006 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
42
42
FP_CTIME_COLUMN = 2
45
def _fingerprint(abspath):
47
fs = os.lstat(abspath)
49
# might be missing, etc
52
if stat.S_ISDIR(fs.st_mode):
55
# we discard any high precision because it's not reliable; perhaps we
56
# could do better on some systems?
57
return (fs.st_size, long(fs.st_mtime),
58
long(fs.st_ctime), fs.st_ino, fs.st_dev, fs.st_mode)
61
47
class HashCache(object):
97
83
def __init__(self, root, cache_file_name, mode=None):
98
84
"""Create a hash cache in base dir, and set the file mode to mode."""
99
85
self.root = safe_unicode(root)
86
self.root_utf8 = self.root.encode('utf8') # where is the filesystem encoding ?
100
87
self.hit_count = 0
101
88
self.miss_count = 0
102
89
self.stat_count = 0
132
119
for inum, path, cache_entry in prep:
133
120
abspath = pathjoin(self.root, path)
134
fp = _fingerprint(abspath)
121
fp = self._fingerprint(abspath)
135
122
self.stat_count += 1
137
124
cache_fp = cache_entry[1]
142
129
self.needs_write = True
143
130
del self._cache[path]
146
def get_sha1(self, path):
132
def get_sha1(self, path, stat_value=None):
147
133
"""Return the sha1 of a file.
149
abspath = pathjoin(self.root, path)
135
if path.__class__ is str:
136
abspath = pathjoin(self.root_utf8, path)
138
abspath = pathjoin(self.root, path)
150
139
self.stat_count += 1
151
file_fp = _fingerprint(abspath)
140
file_fp = self._fingerprint(abspath, stat_value)
154
143
# not a regular file or not existing
164
153
cache_sha1, cache_fp = None, None
166
155
if cache_fp == file_fp:
156
## mutter("hashcache hit for %s %r -> %s", path, file_fp, cache_sha1)
157
## mutter("now = %s", time.time())
167
158
self.hit_count += 1
168
159
return cache_sha1
170
161
self.miss_count += 1
173
163
mode = file_fp[FP_MODE_COLUMN]
174
164
if stat.S_ISREG(mode):
175
digest = sha_file(file(abspath, 'rb', buffering=65000))
165
digest = self._really_sha1_file(abspath)
176
166
elif stat.S_ISLNK(mode):
177
167
digest = sha.new(os.readlink(abspath)).hexdigest()
179
169
raise BzrError("file %r: unknown file stat mode: %o"%(abspath,mode))
181
now = int(time.time())
182
if file_fp[FP_MTIME_COLUMN] >= now or file_fp[FP_CTIME_COLUMN] >= now:
171
# window of 3 seconds to allow for 2s resolution on windows,
172
# unsynchronized file servers, etc.
173
cutoff = self._cutoff_time()
174
if file_fp[FP_MTIME_COLUMN] >= cutoff \
175
or file_fp[FP_CTIME_COLUMN] >= cutoff:
183
176
# changed too recently; can't be cached. we can
184
177
# return the result and it could possibly be cached
191
184
# need to let sufficient time elapse before we may cache this entry
192
185
# again. If we didn't do this, then, for example, a very quick 1
193
186
# byte replacement in the file might go undetected.
194
self.danger_count += 1
187
## mutter('%r modified too recently; not caching', path)
188
self.danger_count += 1
196
190
self.removed_count += 1
197
191
self.needs_write = True
198
192
del self._cache[path]
194
## mutter('%r added to cache: now=%f, mtime=%d, ctime=%d',
195
## path, time.time(), file_fp[FP_MTIME_COLUMN],
196
## file_fp[FP_CTIME_COLUMN])
200
197
self.update_count += 1
201
198
self.needs_write = True
202
199
self._cache[path] = (digest, file_fp)
202
def _really_sha1_file(self, abspath):
203
"""Calculate the SHA1 of a file by reading the full text"""
204
return sha_file(file(abspath, 'rb', buffering=65000))
206
207
"""Write contents of cache to file."""
207
208
outf = AtomicFile(self.cache_file_name(), 'wb', new_mode=self._mode)
209
print >>outf, CACHE_HEADER,
210
outf.write(CACHE_HEADER)
211
212
for path, c in self._cache.iteritems():
212
213
assert '//' not in path, path
213
outf.write(path.encode('utf-8'))
215
print >>outf, c[0], # hex sha1
217
print >>outf, "%d" % fld,
214
line_info = [path.encode('utf-8'), '// ', c[0], ' ']
215
line_info.append(' '.join([str(fld) for fld in c[1]]))
216
line_info.append('\n')
217
outf.write(''.join(line_info))
220
219
self.needs_write = False
221
mutter("write hash cache: %s hits=%d misses=%d stat=%d recent=%d updates=%d",
222
self.cache_file_name(), self.hit_count, self.miss_count,
224
self.danger_count, self.update_count)
220
## mutter("write hash cache: %s hits=%d misses=%d stat=%d recent=%d updates=%d",
221
## self.cache_file_name(), self.hit_count, self.miss_count,
223
## self.danger_count, self.update_count)
230
228
"""Reinstate cache from file.
275
272
self._cache[path] = (sha1, fp)
277
274
self.needs_write = False
276
def _cutoff_time(self):
277
"""Return cutoff time.
279
Files modified more recently than this time are at risk of being
280
undetectably modified and so can't be cached.
282
return int(time.time()) - 3
284
def _fingerprint(self, abspath, stat_value=None):
285
if stat_value is None:
287
stat_value = os.lstat(abspath)
289
# might be missing, etc
291
if stat.S_ISDIR(stat_value.st_mode):
293
# we discard any high precision because it's not reliable; perhaps we
294
# could do better on some systems?
295
return (stat_value.st_size, long(stat_value.st_mtime),
296
long(stat_value.st_ctime), stat_value.st_ino,
297
stat_value.st_dev, stat_value.st_mode)
added
removed
bzrlib/hashcache.py
