← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/chunk_writer.py

  • Committer: John Arbash Meinel
  • Date: 2008-08-21 19:35:58 UTC
  • mto: This revision was merged to the branch mainline in revision 3644.
  • Revision ID: john@arbash-meinel.com-20080821193558-0a4qni76jso98gxn
Now that we have real data, remove the copy() code.
It didn't really benefit us over just doing another repack.
There was a small benefit when the compressor is genuinely full,
but that path is rarely encountered because we stop repacking by effort
rather than by space. It also stablizes the tests for all
platforms, because the copy() code would alter the packing
slightly.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/chunk_writer.py
85
85
        self.unused_bytes = None
86
86
        self.reserved_size = reserved
87
87
        self.min_compress_size = self._default_min_compression_size
88
 
        self.num_zsync = 0
89
 
        self.compressor_has_copy = (getattr(self.compressor, 'copy', None)
90
 
                                    is not None)
91
88
 
92
89
    def finish(self):
93
90
        """Finish the chunk.
132
129
            out = compress(accepted_bytes)
133
130
            if out:
134
131
                append(out)
135
 
        alt_compressed = None
136
132
        if extra_bytes:
137
 
            if self.compressor_has_copy:
138
 
                alt_compressed = (list(bytes_out), compressor.copy())
139
133
            out = compress(extra_bytes)
140
134
            if out:
141
135
                append(out)
142
136
            out = compressor.flush(Z_SYNC_FLUSH)
143
137
            if out:
144
138
                append(out)
145
 
        return bytes_out, compressor, alt_compressed
 
139
        return bytes_out, compressor
146
140
 
147
141
    def write(self, bytes):
148
142
        """Write some bytes to the chunk.
180
174
            self.seen_bytes = next_seen_size
181
175
        else:
182
176
            if not reserved and self.num_repack >= self._max_repack:
183
 
            # if (not reserved
184
 
            #     and (self.num_repack > self._max_repack
185
 
            #          or (self._max_repack == self._max_repack
186
 
            #              and not self.compressor_has_copy))):
187
 
            #     # We have packed too many times already.
 
177
                # We already know we don't want to try to fit more
188
178
                return True
189
 
            if not reserved and self.num_repack == self._max_repack:
190
 
                assert self.compressor_has_copy
191
 
                # We are trying to sneak in a few more keys before we run out
192
 
                # of room, so copy the compressor. If we bust, we stop right
193
 
                # now
194
 
                copy = self.compressor.copy()
195
 
                out = self.compressor.compress(bytes)
196
 
                out += self.compressor.flush(Z_SYNC_FLUSH)
197
 
                total_len = sum(map(len, self.bytes_list)) + len(out)
198
 
                if total_len + 10 > capacity:
199
 
                    self.compressor = copy
200
 
                    # Don't try any more
201
 
                    self.num_repack += 1
202
 
                    return True
203
 
                # It is tempting to use the copied compressor here, because it
204
 
                # is more tightly packed. It gets us to the maximum packing
205
 
                # value. However, it adds about the same overhead as setting
206
 
                # _max_repack to a higher value
207
 
                # self.compressor = copy
208
 
                # out = self.compressor.compress(bytes)
209
 
                self.bytes_in.append(bytes)
210
 
                if out:
211
 
                    self.bytes_list.append(out)
212
 
                return False
213
179
            # This may or may not fit, try to add it with Z_SYNC_FLUSH
214
180
            out = self.compressor.compress(bytes)
215
 
            if out:
216
 
                self.bytes_list.append(out)
217
 
            out = self.compressor.flush(Z_SYNC_FLUSH)
218
 
            if out:
219
 
                self.bytes_list.append(out)
220
 
            self.num_zsync += 1
221
 
            # TODO: We may want to cache total_len, as the 'sum' call seems to
222
 
            #       be showing up a bit on lsprof output
 
181
            out += self.compressor.flush(Z_SYNC_FLUSH)
 
182
            if out:
 
183
                self.bytes_list.append(out)
223
184
            total_len = sum(map(len, self.bytes_list))
224
 
            # Give us some extra room for a final Z_FINISH call.
 
185
            # total_len + 10 is to give some room for Z_FINISH
225
186
            if total_len + 10 > capacity:
226
187
                # We are over budget, try to squeeze this in without any
227
188
                # Z_SYNC_FLUSH calls
228
189
                self.num_repack += 1
229
 
                if False and self.num_repack >= self._max_repack:
230
 
                    this_len = None
231
 
                    alt_compressed = None
232
 
                else:
233
 
                    (bytes_out, compressor,
234
 
                     alt_compressed) = self._recompress_all_bytes_in(bytes)
235
 
                    this_len = sum(map(len, bytes_out))
 
190
                bytes_out, compressor = self._recompress_all_bytes_in(bytes)
 
191
                this_len = sum(map(len, bytes_out))
236
192
                if this_len is None or this_len + 10 > capacity:
237
193
                    # No way we can add anymore, we need to re-pack because our
238
 
                    # compressor is now out of sync
239
 
                    if alt_compressed is None:
240
 
                        bytes_out, compressor, _ = self._recompress_all_bytes_in()
241
 
                    else:
242
 
                        bytes_out, compressor = alt_compressed
 
194
                    # compressor is now out of sync.
 
195
                    # This seems to be rarely triggered over
 
196
                    #   num_repack > _max_repack
 
197
                    bytes_out, compressor = self._recompress_all_bytes_in()
243
198
                    self.compressor = compressor
244
199
                    self.bytes_list = bytes_out
245
200
                    self.unused_bytes = bytes
246
201
                    return True
247
202
                else:
248
203
                    # This fits when we pack it tighter, so use the new packing
249
 
                    if alt_compressed is not None:
250
 
                        # We know it will fit, so put it into another
251
 
                        # compressor without Z_SYNC_FLUSH
252
 
                        bytes_out, compressor = alt_compressed
253
 
                        compressor.compress(bytes)
254
 
                        self.num_zsync = 0
255
 
                    else:
256
 
                        # There is one Z_SYNC_FLUSH call in
257
 
                        # _recompress_all_bytes_in
258
 
                        self.num_zsync = 1
 
204
                    # There is one Z_SYNC_FLUSH call in
 
205
                    # _recompress_all_bytes_in
259
206
                    self.compressor = compressor
260
207
                    self.bytes_in.append(bytes)
261
208
                    self.bytes_list = bytes_out