← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to groupcompress.py

  • Committer: John Arbash Meinel
  • Date: 2009-03-03 22:02:15 UTC
  • mto: (0.17.31 trunk)
  • mto: This revision was merged to the branch mainline in revision 4280.
  • Revision ID: john@arbash-meinel.com-20090303220215-1luhz4zfr9vrdmud
Use the max_delta flag.
Prefer to extract and compress bytes rather than chunks/lines.
This has a fairly positive impact on the 'bzr pack' times.
We still do a ''.join([bytes]), but we know that doesn't have
to do any memory copying.

Show diffs side-by-side

added added

removed removed

Lines of Context:
groupcompress.py
150
150
        :return: The sha1 of lines, and the number of bytes accumulated in
151
151
            the group output so far.
152
152
        """
153
 
        target_text = ''.join(chunks)
154
 
        sha1 = sha_string(target_text)
 
153
        # TODO: Change this to a bytes interface, since the output is now a
 
154
        #       bytes interface anyway.
 
155
        bytes = ''.join(chunks)
 
156
        sha1 = sha_string(bytes)
155
157
        if key[-1] is None:
156
158
            key = key[:-1] + ('sha1:' + sha1,)
157
159
        label = '\x00'.join(key)
158
 
        input_len = len(target_text)
 
160
        input_len = len(bytes)
159
161
        # By having action/label/sha1/len, we can parse the group if the index
160
162
        # was ever destroyed, we have the key in 'label', we know the final
161
163
        # bytes are valid from sha1, and we know where to find the end of this
172
174
            raise AssertionError('_source_offset != endpoint'
173
175
                ' somehow the DeltaIndex got out of sync with'
174
176
                ' the output lines')
175
 
        delta = self._delta_index.make_delta(target_text)
176
 
        if (delta is None
177
 
            or len(delta) > len(target_text) / 2):
 
177
        max_delta_size = len(bytes) / 2
 
178
        delta = self._delta_index.make_delta(bytes, max_delta_size)
 
179
        if (delta is None):
178
180
            # We can't delta (perhaps source_text is empty)
179
181
            # so mark this as an insert
180
182
            if _NO_LABELS:
183
185
                new_chunks.insert(0, 'fulltext\n')
184
186
                new_chunks.append('len:%s\n' % (input_len,))
185
187
            unadded_bytes = sum(map(len, new_chunks))
186
 
            self._delta_index.add_source(target_text, unadded_bytes)
187
 
            new_chunks.append(target_text)
 
188
            self._delta_index.add_source(bytes, unadded_bytes)
 
189
            new_chunks.append(bytes)
188
190
        else:
189
191
            if _NO_LABELS:
190
192
                new_chunks = ['d']
605
607
            if record.storage_kind == 'absent':
606
608
                raise errors.RevisionNotPresent(record.key, self)
607
609
            try:
608
 
                lines = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
 
610
                bytes = record.get_bytes_as('fulltext')
609
611
            except errors.UnavailableRepresentation:
610
612
                adapter_key = record.storage_kind, 'fulltext'
611
613
                adapter = get_adapter(adapter_key)
612
614
                bytes = adapter.get_bytes(record)
613
 
                lines = osutils.split_lines(bytes)
614
615
            soft = False
615
616
            if len(record.key) > 1:
616
617
                prefix = record.key[0]
625
626
                        groups += 1
626
627
                last_prefix = prefix
627
628
            found_sha1, end_point = self._compressor.compress(record.key,
628
 
                lines, record.sha1, soft=soft)
 
629
                [bytes], record.sha1, soft=soft)
629
630
            if record.key[-1] is None:
630
631
                key = record.key[:-1] + ('sha1:' + found_sha1,)
631
632
            else: