14
14
# You should have received a copy of the GNU General Public License
15
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
18
"""Bzrlib specific gzip tunings. We plan to feed these to the upstream gzip."""
22
22
# make GzipFile faster:
24
from gzip import U32, LOWU32, FEXTRA, FCOMMENT, FNAME, FHCRC
24
from gzip import FEXTRA, FCOMMENT, FNAME, FHCRC
29
29
# we want a \n preserved, break on \n only splitlines.
32
__all__ = ["GzipFile"]
32
__all__ = ["GzipFile", "bytes_to_gzip"]
36
"""Return i as an unsigned integer, assuming it fits in 32 bits.
38
If it's >= 2GB when viewed as a 32-bit unsigned int, return a long.
46
"""Return the low-order 32 bits of an int, as a non-negative int."""
47
return i & 0xFFFFFFFFL
50
def bytes_to_gzip(bytes, factory=zlib.compressobj,
51
level=zlib.Z_DEFAULT_COMPRESSION, method=zlib.DEFLATED,
52
width=-zlib.MAX_WBITS, mem=zlib.DEF_MEM_LEVEL,
54
"""Create a gzip file containing bytes and return its content."""
56
'\037\213' # self.fileobj.write('\037\213') # magic header
57
'\010' # self.fileobj.write('\010') # compression method
58
# fname = self.filename[:-3]
62
'\x00' # self.fileobj.write(chr(flags))
63
'\0\0\0\0' # write32u(self.fileobj, long(time.time()))
64
'\002' # self.fileobj.write('\002')
65
'\377' # self.fileobj.write('\377')
67
'' # self.fileobj.write(fname + '\000')
69
# using a compressobj avoids a small header and trailer that the compress()
70
# utility function adds.
71
compress = factory(level, method, width, mem, 0)
72
result.append(compress.compress(bytes))
73
result.append(compress.flush())
74
result.append(struct.pack("<L", LOWU32(crc32(bytes))))
75
# size may exceed 2GB, or even 4GB
76
result.append(struct.pack("<L", LOWU32(len(bytes))))
77
return ''.join(result)
35
80
class GzipFile(gzip.GzipFile):
69
114
"""A tuned version of gzip._write_gzip_header
71
116
We have some extra constrains that plain Gzip does not.
72
1) We want to write the whole blob at once. rather than multiple
117
1) We want to write the whole blob at once. rather than multiple
73
118
calls to fileobj.write().
74
119
2) We never have a filename
75
120
3) We don't care about the time
92
137
def _read(self, size=1024):
93
138
# various optimisations:
94
# reduces lsprof count from 2500 to
139
# reduces lsprof count from 2500 to
95
140
# 8337 calls in 1272, 365 internal
96
141
if self.fileobj is None:
97
142
raise EOFError, "Reached EOF"
121
166
self._add_read_data(self.decompress.flush())
122
assert len(self.decompress.unused_data) >= 8, "what does flush do?"
167
if len(self.decompress.unused_data) < 8:
168
raise AssertionError("what does flush do?")
123
169
self._gzip_tail = self.decompress.unused_data[0:8]
125
171
# tell the driving read() call we have stuffed all the data
145
191
self._gzip_tail = self.decompress.unused_data[0:8]
146
192
elif seek_length < 0:
147
193
# we haven't read enough to check the checksum.
148
assert -8 < seek_length, "too great a seek."
194
if not (-8 < seek_length):
195
raise AssertionError("too great a seek")
149
196
buf = self.fileobj.read(-seek_length)
150
197
self._gzip_tail = self.decompress.unused_data + buf
160
207
"""tuned to reduce function calls and eliminate file seeking:
162
209
reduces lsprof count from 800 to 288
164
211
avoid U32 call by using struct format L
167
# We've read to the end of the file, so we should have 8 bytes of
214
# We've read to the end of the file, so we should have 8 bytes of
168
215
# unused data in the decompressor. If we don't, there is a corrupt file.
169
216
# We use these 8 bytes to calculate the CRC and the recorded file size.
170
217
# We then check the that the computed CRC and size of the
171
218
# uncompressed data matches the stored values. Note that the size
172
219
# stored is the true file size mod 2**32.
173
assert len(self._gzip_tail) == 8, "gzip trailer is incorrect length."
220
if not (len(self._gzip_tail) == 8):
221
raise AssertionError("gzip trailer is incorrect length.")
174
222
crc32, isize = struct.unpack("<LL", self._gzip_tail)
175
223
# note that isize is unsigned - it can exceed 2GB
176
224
if crc32 != U32(self.crc):
181
229
def _read_gzip_header(self, bytes=None):
182
230
"""Supply bytes if the minimum header size is already read.
184
232
:param bytes: 10 bytes of header data.
186
234
"""starting cost: 300 in 3998
224
272
def readline(self, size=-1):
225
273
"""Tuned to remove buffer length calls in _unread and...
227
275
also removes multiple len(c) calls, inlines _unread,
228
276
total savings - lsprof 5800 to 5300
282
330
# 4168 calls in 417.
283
331
# Negative numbers result in reading all the lines
285
333
# python's gzip routine uses sizehint. This is a more efficient way
286
334
# than python uses to honor it. But it is even more efficient to
287
335
# just read the entire thing and use cStringIO to split into lines.
295
343
def _unread(self, buf, len_buf=None):
296
344
"""tuned to remove unneeded len calls.
298
346
because this is such an inner routine in readline, and readline is
299
347
in many inner loops, this has been inlined into readline().
301
349
The len_buf parameter combined with the reduction in len calls dropped
302
the lsprof ms count for this routine on my test data from 800 to 200 -
350
the lsprof ms count for this routine on my test data from 800 to 200 -
305
353
if len_buf is None:
323
371
self.offset += data_len
325
373
def writelines(self, lines):
326
# profiling indicated a significant overhead
374
# profiling indicated a significant overhead
327
375
# calling write for each line.
328
376
# this batch call is a lot faster :).
329
377
# (4 seconds to 1 seconds for the sample upgrades I was testing).
added
removed
bzrlib/tuned_gzip.py
