« back to all changes in this revision
Viewing changes to bzrlib/cache.py
- browse files at revision 413
- compare with another revision
- download diff
- download tarball
- view history from revision 413
- Committer: Martin Pool
- Date: 2005-05-09 06:09:42 UTC
- Revision ID: mbp@sourcefrog.net-20050509060942-d9c9efd7feed0894
- more indicators at top of test output
- tidy up remotebranch stuff
- tidy up remotebranch stuff
- files added:
- bzrlib/tests.py
- files removed:
- bzrlib/changeset.py
- patches
- files renamed:
- bzrlib/statcache.py => bzrlib/cache.py
- files modified:
- .bzrignore
added
removed
bzrlib/cache.py
18
18
from binascii import b2a_qp, a2b_qp
19
19
20
20
from trace import mutter
21
from errors import BzrError, BzrCheckError
22
23
24
"""File stat cache to speed up tree comparisons.
25
26
This module basically gives a quick way to find the SHA-1 and related
27
information of a file in the working directory, without actually
28
reading and hashing the whole file.
29
30
31
32
Implementation
33
==============
34
35
Users of this module should not need to know about how this is
36
implemented, and in particular should not depend on the particular
37
data which is stored or its format.
38
39
This is done by maintaining a cache indexed by a file fingerprint of
40
(path, size, mtime, ctime, ino, dev) pointing to the SHA-1. If the
41
fingerprint has changed, we assume the file content has not changed
42
either and the SHA-1 is therefore the same.
43
44
If any of the fingerprint fields have changed then the file content
45
*may* have changed, or it may not have. We need to reread the file
46
contents to make sure, but this is not visible to the user or
47
higher-level code (except as a delay of course).
48
49
The mtime and ctime are stored with nanosecond fields, but not all
50
filesystems give this level of precision. There is therefore a
51
possible race: the file might be modified twice within a second
52
without changing the size or mtime, and a SHA-1 cached from the first
53
version would be wrong. We handle this by not recording a cached hash
54
for any files which were modified in the current second and that
55
therefore have the chance to change again before the second is up.
56
57
The only known hole in this design is if the system clock jumps
58
backwards crossing invocations of bzr. Please don't do that; use ntp
59
to gradually adjust your clock or don't use bzr over the step.
60
61
At the moment this is stored in a simple textfile; it might be nice
62
to use a tdb instead.
63
64
The cache is represented as a map from file_id to a tuple of (file_id,
65
sha1, path, size, mtime, ctime, ino, dev).
66
67
The SHA-1 is stored in memory as a hexdigest.
68
69
File names are written out as the quoted-printable encoding of their
70
UTF-8 representation.
71
"""
72
73
# order of fields returned by fingerprint()
74
FP_SIZE = 0
75
FP_MTIME = 1
76
FP_CTIME = 2
77
FP_INO = 3
78
FP_DEV = 4
79
80
# order of fields in the statcache file and in the in-memory map
81
SC_FILE_ID = 0
82
SC_SHA1 = 1
83
SC_PATH = 2
84
SC_SIZE = 3
85
SC_MTIME = 4
86
SC_CTIME = 5
87
SC_INO = 6
88
SC_DEV = 7
89
90
91
def fingerprint(abspath):
21
22
23
# file fingerprints are: (path, size, mtime, ctime, ino, dev).
24
#
25
# if this is the same for this file as in the previous revision, we
26
# assume the content is the same and the SHA-1 is the same.
27
28
# This is stored in a fingerprint file that also contains the file-id
29
# and the content SHA-1.
30
31
# Thus for any given file we can quickly get the SHA-1, either from
32
# the cache or if the cache is out of date.
33
34
# At the moment this is stored in a simple textfile; it might be nice
35
# to use a tdb instead.
36
37
38
# What we need:
39
40
# build a new cache from scratch
41
# load cache, incrementally update it
42
43
# TODO: Have a paranoid mode where we always compare the texts and
44
# always recalculate the digest, to trap modification without stat
45
# change and SHA collisions.
46
47
48
49
def fingerprint(path, abspath):
92
50
try:
93
51
fs = os.lstat(abspath)
94
52
except OSError:
102
60
fs.st_ctime, fs.st_ino, fs.st_dev)
103
61
104
62
105
def _write_cache(basedir, entry_iter, dangerfiles):
106
from atomicfile import AtomicFile
107
108
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
109
outf = AtomicFile(cachefn, 'wb')
110
try:
111
for entry in entry_iter:
112
if len(entry) != 8:
113
raise ValueError("invalid statcache entry tuple %r" % entry)
114
115
if entry[SC_FILE_ID] in dangerfiles:
116
continue # changed too recently
117
outf.write(entry[0]) # file id
118
outf.write(' ')
119
outf.write(entry[1]) # hex sha1
120
outf.write(' ')
121
outf.write(b2a_qp(entry[2].encode('utf-8'), True)) # name
122
for nf in entry[3:]:
123
outf.write(' %d' % nf)
124
outf.write('\n')
125
126
outf.commit()
127
finally:
128
if not outf.closed:
129
outf.abort()
130
131
132
def load_cache(basedir):
133
from sets import Set
63
def write_cache(branch, entry_iter):
64
outf = branch.controlfile('work-cache.tmp', 'wt')
65
for entry in entry_iter:
66
outf.write(entry[0] + ' ' + entry[1] + ' ')
67
outf.write(b2a_qp(entry[2], True))
68
outf.write(' %d %d %d %d %d\n' % entry[3:])
69
70
outf.close()
71
os.rename(branch.controlfilename('work-cache.tmp'),
72
branch.controlfilename('work-cache'))
73
74
75
76
def load_cache(branch):
134
77
cache = {}
135
seen_paths = Set()
136
78
137
79
try:
138
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
139
cachefile = open(cachefn, 'r')
80
cachefile = branch.controlfile('work-cache', 'rt')
140
81
except IOError:
141
82
return cache
142
83
143
84
for l in cachefile:
144
85
f = l.split(' ')
145
146
86
file_id = f[0]
147
87
if file_id in cache:
148
88
raise BzrError("duplicated file_id in cache: {%s}" % file_id)
149
150
path = a2b_qp(f[2]).decode('utf-8')
151
if path in seen_paths:
152
raise BzrCheckError("duplicated path in cache: %r" % path)
153
seen_paths.add(path)
154
155
entry = (file_id, f[1], path) + tuple([long(x) for x in f[3:]])
156
if len(entry) != 8:
157
raise ValueError("invalid statcache entry tuple %r" % entry)
158
159
cache[file_id] = entry
89
cache[file_id] = (f[0], f[1], a2b_qp(f[2])) + tuple([long(x) for x in f[3:]])
160
90
return cache
161
91
162
92
169
99
yield ie.file_id, path
170
100
171
101
172
173
def update_cache(basedir, inv, flush=False):
174
"""Update and return the cache for the branch.
175
176
The returned cache may contain entries that have not been written
177
to disk for files recently touched.
178
179
flush -- discard any previous cache and recalculate from scratch.
180
"""
181
102
def build_cache(branch):
103
inv = branch.read_working_inventory()
104
105
cache = {}
106
_update_cache_from_list(branch, cache, _files_from_inventory(inv))
182
107
108
109
110
def update_cache(branch, inv):
183
111
# TODO: It's supposed to be faster to stat the files in order by inum.
184
112
# We don't directly know the inum of the files of course but we do
185
113
# know where they were last sighted, so we can sort by that.
186
114
187
assert isinstance(flush, bool)
188
if flush:
189
cache = {}
190
else:
191
cache = load_cache(basedir)
192
return _update_cache_from_list(basedir, cache, _files_from_inventory(inv))
193
194
195
196
def _update_cache_from_list(basedir, cache, to_update):
197
"""Update and return the cache for given files.
198
199
cache -- Previously cached values to be validated.
200
201
to_update -- Sequence of (file_id, path) pairs to check.
115
cache = load_cache(branch)
116
return _update_cache_from_list(branch, cache, _files_from_inventory(inv))
117
118
119
120
def _update_cache_from_list(branch, cache, to_update):
121
"""Update the cache to have info on the named files.
122
123
to_update is a sequence of (file_id, path) pairs.
202
124
"""
203
204
from sets import Set
205
206
stat_cnt = missing_cnt = hardcheck = change_cnt = 0
207
208
# files that have been recently touched and can't be
209
# committed to a persistent cache yet.
210
211
dangerfiles = Set()
212
now = int(time.time())
213
214
## mutter('update statcache under %r' % basedir)
125
hardcheck = dirty = 0
215
126
for file_id, path in to_update:
216
abspath = os.path.join(basedir, path)
217
fp = fingerprint(abspath)
218
stat_cnt += 1
219
127
fap = branch.abspath(path)
128
fp = fingerprint(fap, path)
220
129
cacheentry = cache.get(file_id)
221
130
222
131
if fp == None: # not here
223
132
if cacheentry:
224
133
del cache[file_id]
225
change_cnt += 1
226
missing_cnt += 1
134
dirty += 1
227
135
continue
228
136
229
if (fp[FP_MTIME] >= now) or (fp[FP_CTIME] >= now):
230
dangerfiles.add(file_id)
231
232
137
if cacheentry and (cacheentry[3:] == fp):
233
138
continue # all stat fields unchanged
234
139
235
140
hardcheck += 1
236
141
237
dig = sha.new(file(abspath, 'rb').read()).hexdigest()
238
239
# We update the cache even if the digest has not changed from
240
# last time we looked, so that the fingerprint fields will
241
# match in future.
242
cacheentry = (file_id, dig, path) + fp
243
cache[file_id] = cacheentry
244
change_cnt += 1
245
246
mutter('statcache: statted %d files, read %d files, %d changed, %d dangerous, '
247
'%d in cache'
248
% (stat_cnt, hardcheck, change_cnt, len(dangerfiles), len(cache)))
142
dig = sha.new(file(fap, 'rb').read()).hexdigest()
143
144
if cacheentry == None or dig != cacheentry[1]:
145
# if there was no previous entry for this file, or if the
146
# SHA has changed, then update the cache
147
cacheentry = (file_id, dig, path) + fp
148
cache[file_id] = cacheentry
149
dirty += 1
150
151
mutter('work cache: read %d files, %d changed' % (hardcheck, dirty))
249
152
250
if change_cnt:
251
mutter('updating on-disk statcache')
252
_write_cache(basedir, cache.itervalues(), dangerfiles)
153
if dirty:
154
write_cache(branch, cache.itervalues())
253
155
254
156
return cache
Loggerhead is a web-based interface for Breezy
Version: 2.0.1