← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/pack.py

  • Committer: Martin Packman
  • Date: 2012-01-05 09:50:04 UTC
  • mfrom: (6424 +trunk)
  • mto: This revision was merged to the branch mainline in revision 6426.
  • Revision ID: martin.packman@canonical.com-20120105095004-mia9xb7y0efmto0v
Merge bzr.dev to resolve conflicts in bzrlib.builtins

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/pack.py
20
20
doc/developers/container-format.txt.
21
21
"""
22
22
 
 
23
from __future__ import absolute_import
 
24
 
23
25
from cStringIO import StringIO
24
26
import re
25
27
 
75
77
        """Return the bytes to finish a container."""
76
78
        return "E"
77
79
 
78
 
    def bytes_record(self, bytes, names):
79
 
        """Return the bytes for a Bytes record with the given name and
80
 
        contents.
81
 
        """
 
80
    def bytes_header(self, length, names):
 
81
        """Return the header for a Bytes record."""
82
82
        # Kind marker
83
83
        byte_sections = ["B"]
84
84
        # Length
85
 
        byte_sections.append(str(len(bytes)) + "\n")
 
85
        byte_sections.append(str(length) + "\n")
86
86
        # Names
87
87
        for name_tuple in names:
88
88
            # Make sure we're writing valid names.  Note that we will leave a
92
92
            byte_sections.append('\x00'.join(name_tuple) + "\n")
93
93
        # End of headers
94
94
        byte_sections.append("\n")
95
 
        # Finally, the contents.
96
 
        byte_sections.append(bytes)
97
 
        # XXX: This causes a memory copy of bytes in size, but is usually
98
 
        # faster than two write calls (12 vs 13 seconds to output a gig of
99
 
        # 1k records.) - results may differ on significantly larger records
100
 
        # like .iso's but as they should be rare in any case and thus not
101
 
        # likely to be the common case. The biggest issue is causing extreme
102
 
        # memory pressure in that case. One possibly improvement here is to
103
 
        # check the size of the content before deciding to join here vs call
104
 
        # write twice.
105
95
        return ''.join(byte_sections)
106
96
 
 
97
    def bytes_record(self, bytes, names):
 
98
        """Return the bytes for a Bytes record with the given name and
 
99
        contents.
 
100
 
 
101
        If the content may be large, construct the header separately and then
 
102
        stream out the contents.
 
103
        """
 
104
        return self.bytes_header(len(bytes), names) + bytes
 
105
 
107
106
 
108
107
class ContainerWriter(object):
109
108
    """A class for writing containers to a file.
113
112
        introduced by the begin() and end() methods.
114
113
    """
115
114
 
 
115
    # Join up headers with the body if writing fewer than this many bytes:
 
116
    # trades off memory usage and copying to do less IO ops.
 
117
    _JOIN_WRITES_THRESHOLD = 100000
 
118
 
116
119
    def __init__(self, write_func):
117
120
        """Constructor.
118
121
 
151
154
            and thus are only suitable for use by a ContainerReader.
152
155
        """
153
156
        current_offset = self.current_offset
154
 
        serialised_record = self._serialiser.bytes_record(bytes, names)
155
 
        self.write_func(serialised_record)
 
157
        length = len(bytes)
 
158
        if length < self._JOIN_WRITES_THRESHOLD:
 
159
            self.write_func(self._serialiser.bytes_header(length, names)
 
160
                + bytes)
 
161
        else:
 
162
            self.write_func(self._serialiser.bytes_header(length, names))
 
163
            self.write_func(bytes)
156
164
        self.records_written += 1
157
165
        # return a memo of where we wrote data to allow random access.
158
166
        return current_offset, self.current_offset - current_offset
333
341
                # risk that the same unicode string has been encoded two
334
342
                # different ways.
335
343
                if name_tuple in all_names:
336
 
                    raise errors.DuplicateRecordNameError(name_tuple)
 
344
                    raise errors.DuplicateRecordNameError(name_tuple[0])
337
345
                all_names.add(name_tuple)
338
346
        excess_bytes = self.reader_func(1)
339
347
        if excess_bytes != '':