14
14
# You should have received a copy of the GNU General Public License
15
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
18
"""Bzrlib specific gzip tunings. We plan to feed these to the upstream gzip."""
20
from __future__ import absolute_import
22
20
from cStringIO import StringIO
24
22
# make GzipFile faster:
26
from gzip import FEXTRA, FCOMMENT, FNAME, FHCRC
24
from gzip import U32, LOWU32, FEXTRA, FCOMMENT, FNAME, FHCRC
31
29
# we want a \n preserved, break on \n only splitlines.
32
from bzrlib import symbol_versioning
34
__all__ = ["GzipFile", "bytes_to_gzip"]
38
"""Return i as an unsigned integer, assuming it fits in 32 bits.
40
If it's >= 2GB when viewed as a 32-bit unsigned int, return a long.
48
"""Return the low-order 32 bits of an int, as a non-negative int."""
49
return i & 0xFFFFFFFFL
52
def bytes_to_gzip(bytes, factory=zlib.compressobj,
53
level=zlib.Z_DEFAULT_COMPRESSION, method=zlib.DEFLATED,
54
width=-zlib.MAX_WBITS, mem=zlib.DEF_MEM_LEVEL,
56
"""Create a gzip file containing bytes and return its content."""
57
return chunks_to_gzip([bytes])
60
def chunks_to_gzip(chunks, factory=zlib.compressobj,
61
level=zlib.Z_DEFAULT_COMPRESSION, method=zlib.DEFLATED,
62
width=-zlib.MAX_WBITS, mem=zlib.DEF_MEM_LEVEL,
64
"""Create a gzip file containing chunks and return its content.
66
:param chunks: An iterable of strings. Each string can have arbitrary
70
'\037\213' # self.fileobj.write('\037\213') # magic header
71
'\010' # self.fileobj.write('\010') # compression method
72
# fname = self.filename[:-3]
76
'\x00' # self.fileobj.write(chr(flags))
77
'\0\0\0\0' # write32u(self.fileobj, long(time.time()))
78
'\002' # self.fileobj.write('\002')
79
'\377' # self.fileobj.write('\377')
81
'' # self.fileobj.write(fname + '\000')
83
# using a compressobj avoids a small header and trailer that the compress()
84
# utility function adds.
85
compress = factory(level, method, width, mem, 0)
89
crc = crc32(chunk, crc)
90
total_len += len(chunk)
91
zbytes = compress.compress(chunk)
94
result.append(compress.flush())
95
# size may exceed 2GB, or even 4GB
96
result.append(struct.pack("<LL", LOWU32(crc), LOWU32(total_len)))
97
return ''.join(result)
32
__all__ = ["GzipFile"]
100
35
class GzipFile(gzip.GzipFile):
120
55
Yes, its only 1.6 seconds, but they add up.
123
def __init__(self, *args, **kwargs):
124
symbol_versioning.warn(
125
symbol_versioning.deprecated_in((2, 3, 0))
126
% 'bzrlib.tuned_gzip.GzipFile',
127
DeprecationWarning, stacklevel=2)
128
gzip.GzipFile.__init__(self, *args, **kwargs)
130
if sys.version_info >= (2, 7, 4):
131
def _add_read_data(self, data):
133
# temp var for len(data) and switch to +='s.
136
self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
137
offset = self.offset - self.extrastart
138
self.extrabuf = self.extrabuf[offset:] + data
139
self.extrasize = self.extrasize + len_data
140
self.extrastart = self.offset
141
self.size = self.size + len_data
143
def _add_read_data(self, data):
145
# temp var for len(data) and switch to +='s.
148
self.crc = zlib.crc32(data, self.crc)
149
self.extrabuf += data
150
self.extrasize += len_data
151
self.size += len_data
58
def _add_read_data(self, data):
60
# temp var for len(data) and switch to +='s.
63
self.crc = zlib.crc32(data, self.crc)
65
self.extrasize += len_data
153
68
def _write_gzip_header(self):
154
69
"""A tuned version of gzip._write_gzip_header
156
71
We have some extra constrains that plain Gzip does not.
157
1) We want to write the whole blob at once. rather than multiple
72
1) We want to write the whole blob at once. rather than multiple
158
73
calls to fileobj.write().
159
74
2) We never have a filename
160
75
3) We don't care about the time
174
89
'' # self.fileobj.write(fname + '\000')
177
if sys.version_info < (2, 7, 4):
178
def _read(self, size=1024):
179
# various optimisations:
180
# reduces lsprof count from 2500 to
181
# 8337 calls in 1272, 365 internal
182
if self.fileobj is None:
92
def _read(self, size=1024):
93
# various optimisations:
94
# reduces lsprof count from 2500 to
95
# 8337 calls in 1272, 365 internal
96
if self.fileobj is None:
97
raise EOFError, "Reached EOF"
100
# If the _new_member flag is set, we have to
101
# jump to the next member, if there is one.
103
# First, check if we're at the end of the file;
104
# if so, it's time to stop; no more members to read.
105
next_header_bytes = self.fileobj.read(10)
106
if next_header_bytes == '':
183
107
raise EOFError, "Reached EOF"
186
# If the _new_member flag is set, we have to
187
# jump to the next member, if there is one.
189
# First, check if we're at the end of the file;
190
# if so, it's time to stop; no more members to read.
191
next_header_bytes = self.fileobj.read(10)
192
if next_header_bytes == '':
193
raise EOFError, "Reached EOF"
196
self._read_gzip_header(next_header_bytes)
197
self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
198
self._new_member = False
200
# Read a chunk of data from the file
201
buf = self.fileobj.read(size)
203
# If the EOF has been reached, flush the decompression object
204
# and mark this object as finished.
207
self._add_read_data(self.decompress.flush())
208
if len(self.decompress.unused_data) < 8:
209
raise AssertionError("what does flush do?")
110
self._read_gzip_header(next_header_bytes)
111
self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
112
self._new_member = False
114
# Read a chunk of data from the file
115
buf = self.fileobj.read(size)
117
# If the EOF has been reached, flush the decompression object
118
# and mark this object as finished.
121
self._add_read_data(self.decompress.flush())
122
assert len(self.decompress.unused_data) >= 8, "what does flush do?"
123
self._gzip_tail = self.decompress.unused_data[0:8]
125
# tell the driving read() call we have stuffed all the data
127
raise EOFError, 'Reached EOF'
129
self._add_read_data(self.decompress.decompress(buf))
131
if self.decompress.unused_data != "":
132
# Ending case: we've come to the end of a member in the file,
133
# so seek back to the start of the data for the next member which
134
# is the length of the decompress objects unused data - the first
135
# 8 bytes for the end crc and size records.
137
# so seek back to the start of the unused data, finish up
138
# this member, and read a new gzip header.
139
# (The number of bytes to seek back is the length of the unused
140
# data, minus 8 because those 8 bytes are part of this member.
141
seek_length = len (self.decompress.unused_data) - 8
143
# we read too much data
144
self.fileobj.seek(-seek_length, 1)
210
145
self._gzip_tail = self.decompress.unused_data[0:8]
212
# tell the driving read() call we have stuffed all the data
214
raise EOFError, 'Reached EOF'
216
self._add_read_data(self.decompress.decompress(buf))
218
if self.decompress.unused_data != "":
219
# Ending case: we've come to the end of a member in the file,
220
# so seek back to the start of the data for the next member
221
# which is the length of the decompress objects unused data -
222
# the first 8 bytes for the end crc and size records.
224
# so seek back to the start of the unused data, finish up
225
# this member, and read a new gzip header.
226
# (The number of bytes to seek back is the length of the unused
227
# data, minus 8 because those 8 bytes are part of this member.
228
seek_length = len (self.decompress.unused_data) - 8
230
# we read too much data
231
self.fileobj.seek(-seek_length, 1)
232
self._gzip_tail = self.decompress.unused_data[0:8]
233
elif seek_length < 0:
234
# we haven't read enough to check the checksum.
235
if not (-8 < seek_length):
236
raise AssertionError("too great a seek")
237
buf = self.fileobj.read(-seek_length)
238
self._gzip_tail = self.decompress.unused_data + buf
240
self._gzip_tail = self.decompress.unused_data
242
# Check the CRC and file size, and set the flag so we read
243
# a new member on the next call
245
self._new_member = True
248
"""tuned to reduce function calls and eliminate file seeking:
250
reduces lsprof count from 800 to 288
252
avoid U32 call by using struct format L
255
# We've read to the end of the file, so we should have 8 bytes of
256
# unused data in the decompressor. If we don't, there is a corrupt
257
# file. We use these 8 bytes to calculate the CRC and the recorded
258
# file size. We then check the that the computed CRC and size of
259
# the uncompressed data matches the stored values. Note that the
260
# size stored is the true file size mod 2**32.
261
if not (len(self._gzip_tail) == 8):
262
raise AssertionError("gzip trailer is incorrect length.")
263
crc32, isize = struct.unpack("<LL", self._gzip_tail)
264
# note that isize is unsigned - it can exceed 2GB
265
if crc32 != U32(self.crc):
266
raise IOError, "CRC check failed %d %d" % (crc32, U32(self.crc))
267
elif isize != LOWU32(self.size):
268
raise IOError, "Incorrect length of data produced"
146
elif seek_length < 0:
147
# we haven't read enough to check the checksum.
148
assert -8 < seek_length, "too great a seek."
149
buf = self.fileobj.read(-seek_length)
150
self._gzip_tail = self.decompress.unused_data + buf
152
self._gzip_tail = self.decompress.unused_data
154
# Check the CRC and file size, and set the flag so we read
155
# a new member on the next call
157
self._new_member = True
160
"""tuned to reduce function calls and eliminate file seeking:
162
reduces lsprof count from 800 to 288
164
avoid U32 call by using struct format L
167
# We've read to the end of the file, so we should have 8 bytes of
168
# unused data in the decompressor. If we don't, there is a corrupt file.
169
# We use these 8 bytes to calculate the CRC and the recorded file size.
170
# We then check the that the computed CRC and size of the
171
# uncompressed data matches the stored values. Note that the size
172
# stored is the true file size mod 2**32.
173
assert len(self._gzip_tail) == 8, "gzip trailer is incorrect length."
174
crc32, isize = struct.unpack("<LL", self._gzip_tail)
175
# note that isize is unsigned - it can exceed 2GB
176
if crc32 != U32(self.crc):
177
raise IOError, "CRC check failed %d %d" % (crc32, U32(self.crc))
178
elif isize != LOWU32(self.size):
179
raise IOError, "Incorrect length of data produced"
270
181
def _read_gzip_header(self, bytes=None):
271
182
"""Supply bytes if the minimum header size is already read.
273
184
:param bytes: 10 bytes of header data.
275
186
"""starting cost: 300 in 3998
added
removed
bzrlib/tuned_gzip.py
