← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/chunk_writer.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2011-08-17 18:13:57 UTC
  • mfrom: (5268.7.29 transport-segments)
  • Revision ID: pqm@pqm.ubuntu.com-20110817181357-y5q5eth1hk8bl3om
(jelmer) Allow specifying the colocated branch to use in the branch URL,
 and retrieving the branch name using ControlDir._get_selected_branch.
 (Jelmer Vernooij)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/chunk_writer.py
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
#
17
17
 
18
18
"""ChunkWriter: write compressed data out with a fixed upper bound."""
47
47
    #    In testing, some values for bzr.dev::
48
48
    #        repack  time  MB   max   full
49
49
    #         1       7.5  4.6  1140  0
50
 
    #         2       8.4  4.2  1036  1          6.8
 
50
    #         2       8.4  4.2  1036  1
51
51
    #         3       9.8  4.1  1012  278
52
52
    #         4      10.8  4.1  728   945
53
53
    #        20      11.1  4.1  0     1012
54
54
    #        repack = 0
55
 
    #        zsync   time  MB    repack  max_z   time w/ add_node
56
 
    #         0       6.7  24.7  0       6270    5.0
57
 
    #         1       6.5  13.2  0       3342    4.3
58
 
    #         2       6.6   9.6  0       2414    4.9
59
 
    #         5       6.5   6.2  0       1549    4.8
60
 
    #         6       6.5   5.8  1       1435    4.8
61
 
    #         7       6.6   5.5  19      1337    4.8
62
 
    #         8       6.7   5.3  81      1220    4.4
63
 
    #        10       6.8   5.0  260     967     5.3
64
 
    #        11       6.8   4.9  366     839     5.3
65
 
    #        12       6.9   4.8  454     731     5.1
66
 
    #        15       7.2   4.7  704     450     5.8
67
 
    #        20       7.7   4.6  1133    7       5.8
 
55
    #        zsync   time  MB    repack  stop_for_z
 
56
    #         0       5.0  24.7  0       6270
 
57
    #         1       4.3  13.2  0       3342
 
58
    #         2       4.9   9.6  0       2414
 
59
    #         5       4.8   6.2  0       1549
 
60
    #         6       4.8   5.8  1       1435
 
61
    #         7       4.8   5.5  19      1337
 
62
    #         8       4.4   5.3  81      1220
 
63
    #        10       5.3   5.0  260     967
 
64
    #        11       5.3   4.9  366     839
 
65
    #        12       5.1   4.8  454     731
 
66
    #        15       5.8   4.7  704     450
 
67
    #        20       5.8   4.6  1133    7
68
68
 
69
69
    #    In testing, some values for mysql-unpacked::
70
70
    #                next_bytes estim
71
 
    #        repack  time  MB    hit_max full
72
 
    #         1      51.7  15.4  3913  0
73
 
    #         2      54.4  13.7  3467  0         35.4
74
 
    #        20      67.0  13.4  0     3380      46.7
 
71
    #        repack  time  MB    full    stop_for_repack
 
72
    #         1            15.4  0       3913
 
73
    #         2      35.4  13.7  0       346
 
74
    #        20      46.7  13.4  3380    0
75
75
    #        repack=0
76
 
    #        zsync                               time w/ add_node
77
 
    #         0      47.7 116.5  0       29782   29.5
78
 
    #         1      48.5  60.2  0       15356   27.8
79
 
    #         2      48.1  42.4  0       10822   27.8
80
 
    #         5      48.3  25.5  0       6491    26.8
81
 
    #         6      48.0  23.2  13      5896    27.3
82
 
    #         7      48.1  21.6  29      5451    27.5
83
 
    #         8      48.1  20.3  52      5108    27.1
84
 
    #        10      46.9  18.6  195     4526    29.4
85
 
    #        11      48.8  18.0  421     4143    29.2
86
 
    #        12      47.4  17.5  702     3738    28.0
87
 
    #        15      49.6  16.5  1223    2969    28.9
88
 
    #        20      48.9  15.7  2182    1810    29.6
89
 
    #        30            15.4  3891    23      31.4
90
 
 
91
 
    _max_repack = 0
92
 
    _max_zsync = 8
93
 
 
94
 
    def __init__(self, chunk_size, reserved=0):
 
76
    #        zsync                       stop_for_z
 
77
    #         0      29.5 116.5  0       29782
 
78
    #         1      27.8  60.2  0       15356
 
79
    #         2      27.8  42.4  0       10822
 
80
    #         5      26.8  25.5  0       6491
 
81
    #         6      27.3  23.2  13      5896
 
82
    #         7      27.5  21.6  29      5451
 
83
    #         8      27.1  20.3  52      5108
 
84
    #        10      29.4  18.6  195     4526
 
85
    #        11      29.2  18.0  421     4143
 
86
    #        12      28.0  17.5  702     3738
 
87
    #        15      28.9  16.5  1223    2969
 
88
    #        20      29.6  15.7  2182    1810
 
89
    #        30      31.4  15.4  3891    23
 
90
 
 
91
    # Tuple of (num_repack_attempts, num_zsync_attempts)
 
92
    # num_zsync_attempts only has meaning if num_repack_attempts is 0.
 
93
    _repack_opts_for_speed = (0, 8)
 
94
    _repack_opts_for_size = (20, 0)
 
95
 
 
96
    def __init__(self, chunk_size, reserved=0, optimize_for_size=False):
95
97
        """Create a ChunkWriter to write chunk_size chunks.
96
98
 
97
99
        :param chunk_size: The total byte count to emit at the end of the
110
112
        self.num_zsync = 0
111
113
        self.unused_bytes = None
112
114
        self.reserved_size = reserved
 
115
        # Default is to make building fast rather than compact
 
116
        self.set_optimize(for_size=optimize_for_size)
113
117
 
114
118
    def finish(self):
115
119
        """Finish the chunk.
118
122
        bytes that did not fit in the chunk.
119
123
 
120
124
        :return: (compressed_bytes, unused_bytes, num_nulls_needed)
121
 
            compressed_bytes    a list of bytes that were output from the
122
 
                                compressor. If the compressed length was not
123
 
                                exactly chunk_size, the final string will be a
124
 
                                string of all null bytes to pad this to
125
 
                                chunk_size
126
 
            unused_bytes        None, or the last bytes that were added, which
127
 
                                we could not fit.
128
 
            num_nulls_needed    How many nulls are padded at the end
 
125
 
 
126
            * compressed_bytes: a list of bytes that were output from the
 
127
              compressor. If the compressed length was not exactly chunk_size,
 
128
              the final string will be a string of all null bytes to pad this
 
129
              to chunk_size
 
130
            * unused_bytes: None, or the last bytes that were added, which we
 
131
              could not fit.
 
132
            * num_nulls_needed: How many nulls are padded at the end
129
133
        """
130
134
        self.bytes_in = None # Free the data cached so far, we don't need it
131
135
        out = self.compressor.flush(Z_FINISH)
141
145
            self.bytes_list.append("\x00" * nulls_needed)
142
146
        return self.bytes_list, self.unused_bytes, nulls_needed
143
147
 
 
148
    def set_optimize(self, for_size=True):
 
149
        """Change how we optimize our writes.
 
150
 
 
151
        :param for_size: If True, optimize for minimum space usage, otherwise
 
152
            optimize for fastest writing speed.
 
153
        :return: None
 
154
        """
 
155
        if for_size:
 
156
            opts = ChunkWriter._repack_opts_for_size
 
157
        else:
 
158
            opts = ChunkWriter._repack_opts_for_speed
 
159
        self._max_repack, self._max_zsync = opts
 
160
 
144
161
    def _recompress_all_bytes_in(self, extra_bytes=None):
145
162
        """Recompress the current bytes_in, and optionally more.
146
163
 
147
164
        :param extra_bytes: Optional, if supplied we will add it with
148
165
            Z_SYNC_FLUSH
149
166
        :return: (bytes_out, bytes_out_len, alt_compressed)
150
 
            bytes_out   is the compressed bytes returned from the compressor
151
 
            bytes_out_len the length of the compressed output
152
 
            compressor  An object with everything packed in so far, and
153
 
                        Z_SYNC_FLUSH called.
 
167
 
 
168
            * bytes_out: is the compressed bytes returned from the compressor
 
169
            * bytes_out_len: the length of the compressed output
 
170
            * compressor: An object with everything packed in so far, and
 
171
              Z_SYNC_FLUSH called.
154
172
        """
155
173
        compressor = zlib.compressobj()
156
174
        bytes_out = []