1
# (C) 2005 Canonical Ltd
1
# Copyright (C) 2005, 2006 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
32
32
import os, stat, time
35
from bzrlib.osutils import sha_file, pathjoin
35
from bzrlib.osutils import sha_file, pathjoin, safe_unicode
36
36
from bzrlib.trace import mutter, warning
37
37
from bzrlib.atomicfile import AtomicFile
38
38
from bzrlib.errors import BzrError
42
42
FP_CTIME_COLUMN = 2
45
def _fingerprint(abspath):
47
fs = os.lstat(abspath)
49
# might be missing, etc
52
if stat.S_ISDIR(fs.st_mode):
55
# we discard any high precision because it's not reliable; perhaps we
56
# could do better on some systems?
57
return (fs.st_size, long(fs.st_mtime),
58
long(fs.st_ctime), fs.st_ino, fs.st_dev, fs.st_mode)
61
47
class HashCache(object):
95
81
needs_write = False
97
def __init__(self, basedir):
98
self.basedir = basedir
83
def __init__(self, root, cache_file_name, mode=None):
84
"""Create a hash cache in base dir, and set the file mode to mode."""
85
self.root = safe_unicode(root)
86
self.root_utf8 = self.root.encode('utf8') # where is the filesystem encoding ?
100
88
self.miss_count = 0
101
89
self.stat_count = 0
103
91
self.removed_count = 0
104
92
self.update_count = 0
95
self._cache_file_name = safe_unicode(cache_file_name)
107
97
def cache_file_name(self):
108
# FIXME: duplicate path logic here, this should be
109
# something like 'branch.controlfile'.
110
return pathjoin(self.basedir, '.bzr', 'stat-cache')
98
return self._cache_file_name
113
101
"""Discard all cached information.
117
105
self.needs_write = True
122
109
"""Scan all files and remove entries where the cache entry is obsolete.
124
111
Obsolete entries are those where the file has been modified or deleted
125
112
since the entry was inserted.
114
# FIXME optimisation opportunity, on linux [and check other oses]:
115
# rather than iteritems order, stat in inode order.
127
116
prep = [(ce[1][3], path, ce) for (path, ce) in self._cache.iteritems()]
130
119
for inum, path, cache_entry in prep:
131
abspath = pathjoin(self.basedir, path)
132
fp = _fingerprint(abspath)
120
abspath = pathjoin(self.root, path)
121
fp = self._fingerprint(abspath)
133
122
self.stat_count += 1
135
124
cache_fp = cache_entry[1]
140
129
self.needs_write = True
141
130
del self._cache[path]
144
def get_sha1(self, path):
132
def get_sha1(self, path, stat_value=None):
145
133
"""Return the sha1 of a file.
147
abspath = pathjoin(self.basedir, path)
135
if path.__class__ is str:
136
abspath = pathjoin(self.root_utf8, path)
138
abspath = pathjoin(self.root, path)
148
139
self.stat_count += 1
149
file_fp = _fingerprint(abspath)
140
file_fp = self._fingerprint(abspath, stat_value)
152
143
# not a regular file or not existing
162
153
cache_sha1, cache_fp = None, None
164
155
if cache_fp == file_fp:
156
## mutter("hashcache hit for %s %r -> %s", path, file_fp, cache_sha1)
157
## mutter("now = %s", time.time())
165
158
self.hit_count += 1
166
159
return cache_sha1
168
161
self.miss_count += 1
171
163
mode = file_fp[FP_MODE_COLUMN]
172
164
if stat.S_ISREG(mode):
173
digest = sha_file(file(abspath, 'rb', buffering=65000))
165
digest = self._really_sha1_file(abspath)
174
166
elif stat.S_ISLNK(mode):
175
167
digest = sha.new(os.readlink(abspath)).hexdigest()
177
169
raise BzrError("file %r: unknown file stat mode: %o"%(abspath,mode))
179
now = int(time.time())
180
if file_fp[FP_MTIME_COLUMN] >= now or file_fp[FP_CTIME_COLUMN] >= now:
171
# window of 3 seconds to allow for 2s resolution on windows,
172
# unsynchronized file servers, etc.
173
cutoff = self._cutoff_time()
174
if file_fp[FP_MTIME_COLUMN] >= cutoff \
175
or file_fp[FP_CTIME_COLUMN] >= cutoff:
181
176
# changed too recently; can't be cached. we can
182
177
# return the result and it could possibly be cached
189
184
# need to let sufficient time elapse before we may cache this entry
190
185
# again. If we didn't do this, then, for example, a very quick 1
191
186
# byte replacement in the file might go undetected.
192
self.danger_count += 1
187
## mutter('%r modified too recently; not caching', path)
188
self.danger_count += 1
194
190
self.removed_count += 1
195
191
self.needs_write = True
196
192
del self._cache[path]
194
## mutter('%r added to cache: now=%f, mtime=%d, ctime=%d',
195
## path, time.time(), file_fp[FP_MTIME_COLUMN],
196
## file_fp[FP_CTIME_COLUMN])
198
197
self.update_count += 1
199
198
self.needs_write = True
200
199
self._cache[path] = (digest, file_fp)
202
def _really_sha1_file(self, abspath):
203
"""Calculate the SHA1 of a file by reading the full text"""
204
return sha_file(file(abspath, 'rb', buffering=65000))
204
207
"""Write contents of cache to file."""
205
outf = AtomicFile(self.cache_file_name(), 'wb')
208
outf = AtomicFile(self.cache_file_name(), 'wb', new_mode=self._mode)
207
print >>outf, CACHE_HEADER,
210
outf.write(CACHE_HEADER)
209
212
for path, c in self._cache.iteritems():
210
assert '//' not in path, path
211
outf.write(path.encode('utf-8'))
213
print >>outf, c[0], # hex sha1
215
print >>outf, "%d" % fld,
213
line_info = [path.encode('utf-8'), '// ', c[0], ' ']
214
line_info.append(' '.join([str(fld) for fld in c[1]]))
215
line_info.append('\n')
216
outf.write(''.join(line_info))
219
218
self.needs_write = False
219
## mutter("write hash cache: %s hits=%d misses=%d stat=%d recent=%d updates=%d",
220
## self.cache_file_name(), self.hit_count, self.miss_count,
222
## self.danger_count, self.update_count)
225
227
"""Reinstate cache from file.
270
271
self._cache[path] = (sha1, fp)
272
273
self.needs_write = False
275
def _cutoff_time(self):
276
"""Return cutoff time.
278
Files modified more recently than this time are at risk of being
279
undetectably modified and so can't be cached.
281
return int(time.time()) - 3
283
def _fingerprint(self, abspath, stat_value=None):
284
if stat_value is None:
286
stat_value = os.lstat(abspath)
288
# might be missing, etc
290
if stat.S_ISDIR(stat_value.st_mode):
292
# we discard any high precision because it's not reliable; perhaps we
293
# could do better on some systems?
294
return (stat_value.st_size, long(stat_value.st_mtime),
295
long(stat_value.st_ctime), stat_value.st_ino,
296
stat_value.st_dev, stat_value.st_mode)