1
# Copyright (C) 2006-2011 Canonical Ltd
2
# Written by Robert Collins <robert.collins@canonical.com>
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU General Public License for more details.
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
"""Bzrlib specific gzip tunings. We plan to feed these to the upstream gzip."""
20
from cStringIO import StringIO
22
# make GzipFile faster:
24
from gzip import FEXTRA, FCOMMENT, FNAME, FHCRC
29
# we want a \n preserved, break on \n only splitlines.
30
from bzrlib import symbol_versioning
32
__all__ = ["GzipFile", "bytes_to_gzip"]
36
"""Return i as an unsigned integer, assuming it fits in 32 bits.
38
If it's >= 2GB when viewed as a 32-bit unsigned int, return a long.
46
"""Return the low-order 32 bits of an int, as a non-negative int."""
47
return i & 0xFFFFFFFFL
50
def bytes_to_gzip(bytes, factory=zlib.compressobj,
51
level=zlib.Z_DEFAULT_COMPRESSION, method=zlib.DEFLATED,
52
width=-zlib.MAX_WBITS, mem=zlib.DEF_MEM_LEVEL,
54
"""Create a gzip file containing bytes and return its content."""
55
return chunks_to_gzip([bytes])
58
def chunks_to_gzip(chunks, factory=zlib.compressobj,
59
level=zlib.Z_DEFAULT_COMPRESSION, method=zlib.DEFLATED,
60
width=-zlib.MAX_WBITS, mem=zlib.DEF_MEM_LEVEL,
62
"""Create a gzip file containing chunks and return its content.
64
:param chunks: An iterable of strings. Each string can have arbitrary
68
'\037\213' # self.fileobj.write('\037\213') # magic header
69
'\010' # self.fileobj.write('\010') # compression method
70
# fname = self.filename[:-3]
74
'\x00' # self.fileobj.write(chr(flags))
75
'\0\0\0\0' # write32u(self.fileobj, long(time.time()))
76
'\002' # self.fileobj.write('\002')
77
'\377' # self.fileobj.write('\377')
79
'' # self.fileobj.write(fname + '\000')
81
# using a compressobj avoids a small header and trailer that the compress()
82
# utility function adds.
83
compress = factory(level, method, width, mem, 0)
87
crc = crc32(chunk, crc)
88
total_len += len(chunk)
89
zbytes = compress.compress(chunk)
92
result.append(compress.flush())
93
# size may exceed 2GB, or even 4GB
94
result.append(struct.pack("<LL", LOWU32(crc), LOWU32(total_len)))
95
return ''.join(result)
98
class GzipFile(gzip.GzipFile):
99
"""Knit tuned version of GzipFile.
101
This is based on the following lsprof stats:
102
python 2.4 stock GzipFile write:
103
58971 0 5644.3090 2721.4730 gzip:193(write)
104
+58971 0 1159.5530 1159.5530 +<built-in method compress>
105
+176913 0 987.0320 987.0320 +<len>
106
+58971 0 423.1450 423.1450 +<zlib.crc32>
107
+58971 0 353.1060 353.1060 +<method 'write' of 'cStringIO.
109
tuned GzipFile write:
110
58971 0 4477.2590 2103.1120 bzrlib.knit:1250(write)
111
+58971 0 1297.7620 1297.7620 +<built-in method compress>
112
+58971 0 406.2160 406.2160 +<zlib.crc32>
113
+58971 0 341.9020 341.9020 +<method 'write' of 'cStringIO.
115
+58971 0 328.2670 328.2670 +<len>
118
Yes, its only 1.6 seconds, but they add up.
121
def __init__(self, *args, **kwargs):
122
symbol_versioning.warn(
123
symbol_versioning.deprecated_in((2, 3, 0))
124
% 'bzrlib.tuned_gzip.GzipFile',
125
DeprecationWarning, stacklevel=2)
126
gzip.GzipFile.__init__(self, *args, **kwargs)
128
def _add_read_data(self, data):
130
# temp var for len(data) and switch to +='s.
133
self.crc = zlib.crc32(data, self.crc)
134
self.extrabuf += data
135
self.extrasize += len_data
136
self.size += len_data
138
def _write_gzip_header(self):
139
"""A tuned version of gzip._write_gzip_header
141
We have some extra constrains that plain Gzip does not.
142
1) We want to write the whole blob at once. rather than multiple
143
calls to fileobj.write().
144
2) We never have a filename
145
3) We don't care about the time
148
'\037\213' # self.fileobj.write('\037\213') # magic header
149
'\010' # self.fileobj.write('\010') # compression method
150
# fname = self.filename[:-3]
154
'\x00' # self.fileobj.write(chr(flags))
155
'\0\0\0\0' # write32u(self.fileobj, long(time.time()))
156
'\002' # self.fileobj.write('\002')
157
'\377' # self.fileobj.write('\377')
159
'' # self.fileobj.write(fname + '\000')
162
def _read(self, size=1024):
163
# various optimisations:
164
# reduces lsprof count from 2500 to
165
# 8337 calls in 1272, 365 internal
166
if self.fileobj is None:
167
raise EOFError, "Reached EOF"
170
# If the _new_member flag is set, we have to
171
# jump to the next member, if there is one.
173
# First, check if we're at the end of the file;
174
# if so, it's time to stop; no more members to read.
175
next_header_bytes = self.fileobj.read(10)
176
if next_header_bytes == '':
177
raise EOFError, "Reached EOF"
180
self._read_gzip_header(next_header_bytes)
181
self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
182
self._new_member = False
184
# Read a chunk of data from the file
185
buf = self.fileobj.read(size)
187
# If the EOF has been reached, flush the decompression object
188
# and mark this object as finished.
191
self._add_read_data(self.decompress.flush())
192
if len(self.decompress.unused_data) < 8:
193
raise AssertionError("what does flush do?")
194
self._gzip_tail = self.decompress.unused_data[0:8]
196
# tell the driving read() call we have stuffed all the data
198
raise EOFError, 'Reached EOF'
200
self._add_read_data(self.decompress.decompress(buf))
202
if self.decompress.unused_data != "":
203
# Ending case: we've come to the end of a member in the file,
204
# so seek back to the start of the data for the next member which
205
# is the length of the decompress objects unused data - the first
206
# 8 bytes for the end crc and size records.
208
# so seek back to the start of the unused data, finish up
209
# this member, and read a new gzip header.
210
# (The number of bytes to seek back is the length of the unused
211
# data, minus 8 because those 8 bytes are part of this member.
212
seek_length = len (self.decompress.unused_data) - 8
214
# we read too much data
215
self.fileobj.seek(-seek_length, 1)
216
self._gzip_tail = self.decompress.unused_data[0:8]
217
elif seek_length < 0:
218
# we haven't read enough to check the checksum.
219
if not (-8 < seek_length):
220
raise AssertionError("too great a seek")
221
buf = self.fileobj.read(-seek_length)
222
self._gzip_tail = self.decompress.unused_data + buf
224
self._gzip_tail = self.decompress.unused_data
226
# Check the CRC and file size, and set the flag so we read
227
# a new member on the next call
229
self._new_member = True
232
"""tuned to reduce function calls and eliminate file seeking:
234
reduces lsprof count from 800 to 288
236
avoid U32 call by using struct format L
239
# We've read to the end of the file, so we should have 8 bytes of
240
# unused data in the decompressor. If we don't, there is a corrupt file.
241
# We use these 8 bytes to calculate the CRC and the recorded file size.
242
# We then check the that the computed CRC and size of the
243
# uncompressed data matches the stored values. Note that the size
244
# stored is the true file size mod 2**32.
245
if not (len(self._gzip_tail) == 8):
246
raise AssertionError("gzip trailer is incorrect length.")
247
crc32, isize = struct.unpack("<LL", self._gzip_tail)
248
# note that isize is unsigned - it can exceed 2GB
249
if crc32 != U32(self.crc):
250
raise IOError, "CRC check failed %d %d" % (crc32, U32(self.crc))
251
elif isize != LOWU32(self.size):
252
raise IOError, "Incorrect length of data produced"
254
def _read_gzip_header(self, bytes=None):
255
"""Supply bytes if the minimum header size is already read.
257
:param bytes: 10 bytes of header data.
259
"""starting cost: 300 in 3998
260
15998 reads from 3998 calls
264
bytes = self.fileobj.read(10)
266
if magic != '\037\213':
267
raise IOError, 'Not a gzipped file'
268
method = ord(bytes[2:3])
270
raise IOError, 'Unknown compression method'
271
flag = ord(bytes[3:4])
272
# modtime = self.fileobj.read(4) (bytes [4:8])
273
# extraflag = self.fileobj.read(1) (bytes[8:9])
274
# os = self.fileobj.read(1) (bytes[9:10])
275
# self.fileobj.read(6)
278
# Read & discard the extra field, if present
279
xlen = ord(self.fileobj.read(1))
280
xlen = xlen + 256*ord(self.fileobj.read(1))
281
self.fileobj.read(xlen)
283
# Read and discard a null-terminated string containing the filename
285
s = self.fileobj.read(1)
286
if not s or s=='\000':
289
# Read and discard a null-terminated string containing a comment
291
s = self.fileobj.read(1)
292
if not s or s=='\000':
295
self.fileobj.read(2) # Read & discard the 16-bit header CRC
297
def readline(self, size=-1):
298
"""Tuned to remove buffer length calls in _unread and...
300
also removes multiple len(c) calls, inlines _unread,
301
total savings - lsprof 5800 to 5300
304
8176 calls to read() in 1684
305
changing the min chunk size to 200 halved all the cache misses
306
leading to a drop to:
308
4168 call to read() in 1646
309
- i.e. just reduced the function call overhead. May be worth
312
if size < 0: size = sys.maxint
314
readsize = min(200, size) # Read from the file in small chunks
317
return "".join(bufs) # Return resulting line
320
c = self.read(readsize)
321
# number of bytes read
325
# We set i=size to break out of the loop under two
326
# conditions: 1) there's no newline, and the chunk is
327
# larger than size, or 2) there is a newline, but the
328
# resulting line would be longer than 'size'.
329
if i==-1 and len_c > size: i=size-1
330
elif size <= i: i = size -1
332
if i >= 0 or c == '':
333
# if i>= 0 we have a newline or have triggered the above
334
# if size is not None condition.
335
# if c == '' its EOF.
336
bufs.append(c[:i+1]) # Add portion of last chunk
337
# -- inlined self._unread --
338
## self._unread(c[i+1:], len_c - i) # Push back rest of chunk
339
self.extrabuf = c[i+1:] + self.extrabuf
340
self.extrasize = len_c - i + self.extrasize
341
self.offset -= len_c - i
342
# -- end inlined self._unread --
343
return ''.join(bufs) # Return resulting line
345
# Append chunk to list, decrease 'size',
348
readsize = min(size, readsize * 2)
350
def readlines(self, sizehint=0):
351
# optimise to avoid all the buffer manipulation
352
# lsprof changed from:
353
# 4168 calls in 5472 with 32000 calls to readline()
356
# Negative numbers result in reading all the lines
358
# python's gzip routine uses sizehint. This is a more efficient way
359
# than python uses to honor it. But it is even more efficient to
360
# just read the entire thing and use cStringIO to split into lines.
363
# content = self.read(sizehint)
364
# return bzrlib.osutils.split_lines(content)
365
content = StringIO(self.read(-1))
366
return content.readlines()
368
def _unread(self, buf, len_buf=None):
369
"""tuned to remove unneeded len calls.
371
because this is such an inner routine in readline, and readline is
372
in many inner loops, this has been inlined into readline().
374
The len_buf parameter combined with the reduction in len calls dropped
375
the lsprof ms count for this routine on my test data from 800 to 200 -
380
self.extrabuf = buf + self.extrabuf
381
self.extrasize = len_buf + self.extrasize
382
self.offset -= len_buf
384
def write(self, data):
385
if self.mode != gzip.WRITE:
387
raise IOError(errno.EBADF, "write() on read-only GzipFile object")
389
if self.fileobj is None:
390
raise ValueError, "write() on closed GzipFile object"
393
self.size = self.size + data_len
394
self.crc = zlib.crc32(data, self.crc)
395
self.fileobj.write( self.compress.compress(data) )
396
self.offset += data_len
398
def writelines(self, lines):
399
# profiling indicated a significant overhead
400
# calling write for each line.
401
# this batch call is a lot faster :).
402
# (4 seconds to 1 seconds for the sample upgrades I was testing).
403
self.write(''.join(lines))
405
if sys.version_info > (2, 7):
406
# As of Python 2.7 the crc32 must be positive when close is called
408
if self.fileobj is None:
410
if self.mode == gzip.WRITE:
411
self.crc &= 0xFFFFFFFFL
412
gzip.GzipFile.close(self)
added
removed
bzrlib/tuned_gzip.py
