1
# Copyright (C) 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Container format for Bazaar data.
19
"Containers" and "records" are described in doc/developers/container-format.txt.
22
from cStringIO import StringIO
25
from bzrlib import errors
28
FORMAT_ONE = "Bazaar pack format 1 (introduced in 0.18)"
31
_whitespace_re = re.compile('[\t\n\x0b\x0c\r ]')
34
def _check_name(name):
35
"""Do some basic checking of 'name'.
37
At the moment, this just checks that there are no whitespace characters in a
40
:raises InvalidRecordError: if name is not valid.
41
:seealso: _check_name_encoding
43
if _whitespace_re.search(name) is not None:
44
raise errors.InvalidRecordError("%r is not a valid name." % (name,))
47
def _check_name_encoding(name):
48
"""Check that 'name' is valid UTF-8.
50
This is separate from _check_name because UTF-8 decoding is relatively
51
expensive, and we usually want to avoid it.
53
:raises InvalidRecordError: if name is not valid UTF-8.
57
except UnicodeDecodeError, e:
58
raise errors.InvalidRecordError(str(e))
61
class ContainerWriter(object):
62
"""A class for writing containers.
64
:attribute records_written: The number of user records added to the
65
container. This does not count the prelude or suffix of the container
66
introduced by the begin() and end() methods.
69
def __init__(self, write_func):
72
:param write_func: a callable that will be called when this
73
ContainerWriter needs to write some bytes.
75
self._write_func = write_func
76
self.current_offset = 0
77
self.records_written = 0
80
"""Begin writing a container."""
81
self.write_func(FORMAT_ONE + "\n")
83
def write_func(self, bytes):
84
self._write_func(bytes)
85
self.current_offset += len(bytes)
88
"""Finish writing a container."""
91
def add_bytes_record(self, bytes, names):
92
"""Add a Bytes record with the given names.
94
:param bytes: The bytes to insert.
95
:param names: The names to give the inserted bytes. Each name is
96
a tuple of bytestrings. The bytestrings may not contain
98
:return: An offset, length tuple. The offset is the offset
99
of the record within the container, and the length is the
100
length of data that will need to be read to reconstitute the
101
record. These offset and length can only be used with the pack
102
interface - they might be offset by headers or other such details
103
and thus are only suitable for use by a ContainerReader.
105
current_offset = self.current_offset
107
byte_sections = ["B"]
109
byte_sections.append(str(len(bytes)) + "\n")
111
for name_tuple in names:
112
# Make sure we're writing valid names. Note that we will leave a
113
# half-written record if a name is bad!
114
for name in name_tuple:
116
byte_sections.append('\x00'.join(name_tuple) + "\n")
118
byte_sections.append("\n")
119
# Finally, the contents.
120
byte_sections.append(bytes)
121
# XXX: This causes a memory copy of bytes in size, but is usually
122
# faster than two write calls (12 vs 13 seconds to output a gig of
123
# 1k records.) - results may differ on significantly larger records
124
# like .iso's but as they should be rare in any case and thus not
125
# likely to be the common case. The biggest issue is causing extreme
126
# memory pressure in that case. One possibly improvement here is to
127
# check the size of the content before deciding to join here vs call
129
self.write_func(''.join(byte_sections))
130
self.records_written += 1
131
# return a memo of where we wrote data to allow random access.
132
return current_offset, self.current_offset - current_offset
135
class ReadVFile(object):
136
"""Adapt a readv result iterator to a file like protocol."""
138
def __init__(self, readv_result):
139
self.readv_result = readv_result
140
# the most recent readv result block
144
if (self._string is None or
145
self._string.tell() == self._string_length):
146
length, data = self.readv_result.next()
147
self._string_length = len(data)
148
self._string = StringIO(data)
150
def read(self, length):
152
result = self._string.read(length)
153
if len(result) < length:
154
raise errors.BzrError('request for too much data from a readv hunk.')
158
"""Note that readline will not cross readv segments."""
160
result = self._string.readline()
161
if self._string.tell() == self._string_length and result[-1] != '\n':
162
raise errors.BzrError('short readline in the readvfile hunk.')
166
def make_readv_reader(transport, filename, requested_records):
167
"""Create a ContainerReader that will read selected records only.
169
:param transport: The transport the pack file is located on.
170
:param filename: The filename of the pack file.
171
:param requested_records: The record offset, length tuples as returned
172
by add_bytes_record for the desired records.
174
readv_blocks = [(0, len(FORMAT_ONE)+1)]
175
readv_blocks.extend(requested_records)
176
result = ContainerReader(ReadVFile(
177
transport.readv(filename, readv_blocks)))
181
class BaseReader(object):
183
def __init__(self, source_file):
186
:param source_file: a file-like object with `read` and `readline`
189
self._source = source_file
191
def reader_func(self, length=None):
192
return self._source.read(length)
194
def _read_line(self):
195
line = self._source.readline()
196
if not line.endswith('\n'):
197
raise errors.UnexpectedEndOfContainerError()
198
return line.rstrip('\n')
201
class ContainerReader(BaseReader):
202
"""A class for reading Bazaar's container format."""
204
def iter_records(self):
205
"""Iterate over the container, yielding each record as it is read.
207
Each yielded record will be a 2-tuple of (names, callable), where names
208
is a ``list`` and bytes is a function that takes one argument,
211
You **must not** call the callable after advancing the interator to the
212
next record. That is, this code is invalid::
214
record_iter = container.iter_records()
215
names1, callable1 = record_iter.next()
216
names2, callable2 = record_iter.next()
217
bytes1 = callable1(None)
219
As it will give incorrect results and invalidate the state of the
222
:raises ContainerError: if any sort of containter corruption is
223
detected, e.g. UnknownContainerFormatError is the format of the
224
container is unrecognised.
225
:seealso: ContainerReader.read
228
return self._iter_records()
230
def iter_record_objects(self):
231
"""Iterate over the container, yielding each record as it is read.
233
Each yielded record will be an object with ``read`` and ``validate``
234
methods. Like with iter_records, it is not safe to use a record object
235
after advancing the iterator to yield next record.
237
:raises ContainerError: if any sort of containter corruption is
238
detected, e.g. UnknownContainerFormatError is the format of the
239
container is unrecognised.
240
:seealso: iter_records
243
return self._iter_record_objects()
245
def _iter_records(self):
246
for record in self._iter_record_objects():
249
def _iter_record_objects(self):
251
record_kind = self.reader_func(1)
252
if record_kind == 'B':
254
reader = BytesRecordReader(self._source)
256
elif record_kind == 'E':
257
# End marker. There are no more records.
259
elif record_kind == '':
260
# End of stream encountered, but no End Marker record seen, so
261
# this container is incomplete.
262
raise errors.UnexpectedEndOfContainerError()
264
# Unknown record type.
265
raise errors.UnknownRecordTypeError(record_kind)
267
def _read_format(self):
268
format = self._read_line()
269
if format != FORMAT_ONE:
270
raise errors.UnknownContainerFormatError(format)
273
"""Validate this container and its records.
275
Validating consumes the data stream just like iter_records and
276
iter_record_objects, so you cannot call it after
277
iter_records/iter_record_objects.
279
:raises ContainerError: if something is invalid.
282
for record_names, read_bytes in self.iter_records():
284
for name_tuple in record_names:
285
for name in name_tuple:
286
_check_name_encoding(name)
287
# Check that the name is unique. Note that Python will refuse
288
# to decode non-shortest forms of UTF-8 encoding, so there is no
289
# risk that the same unicode string has been encoded two
291
if name_tuple in all_names:
292
raise errors.DuplicateRecordNameError(name_tuple)
293
all_names.add(name_tuple)
294
excess_bytes = self.reader_func(1)
295
if excess_bytes != '':
296
raise errors.ContainerHasExcessDataError(excess_bytes)
299
class BytesRecordReader(BaseReader):
304
You can either validate or read a record, you can't do both.
306
:returns: A tuple of (names, callable). The callable can be called
307
repeatedly to obtain the bytes for the record, with a max_length
308
argument. If max_length is None, returns all the bytes. Because
309
records can be arbitrarily large, using None is not recommended
310
unless you have reason to believe the content will fit in memory.
312
# Read the content length.
313
length_line = self._read_line()
315
length = int(length_line)
317
raise errors.InvalidRecordError(
318
"%r is not a valid length." % (length_line,))
320
# Read the list of names.
323
name_line = self._read_line()
326
name_tuple = tuple(name_line.split('\x00'))
327
for name in name_tuple:
329
names.append(name_tuple)
331
self._remaining_length = length
332
return names, self._content_reader
334
def _content_reader(self, max_length):
335
if max_length is None:
336
length_to_read = self._remaining_length
338
length_to_read = min(max_length, self._remaining_length)
339
self._remaining_length -= length_to_read
340
bytes = self.reader_func(length_to_read)
341
if len(bytes) != length_to_read:
342
raise errors.UnexpectedEndOfContainerError()
346
"""Validate this record.
348
You can either validate or read, you can't do both.
350
:raises ContainerError: if this record is invalid.
352
names, read_bytes = self.read()
353
for name_tuple in names:
354
for name in name_tuple:
355
_check_name_encoding(name)
added
removed
bzrlib/pack.py
