1
# Copyright (C) 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Container format for Bazaar data.
19
"Containers" and "records" are described in
20
doc/developers/container-format.txt.
23
from cStringIO import StringIO
26
from bzrlib import errors
29
FORMAT_ONE = "Bazaar pack format 1 (introduced in 0.18)"
32
_whitespace_re = re.compile('[\t\n\x0b\x0c\r ]')
35
def _check_name(name):
36
"""Do some basic checking of 'name'.
38
At the moment, this just checks that there are no whitespace characters in a
41
:raises InvalidRecordError: if name is not valid.
42
:seealso: _check_name_encoding
44
if _whitespace_re.search(name) is not None:
45
raise errors.InvalidRecordError("%r is not a valid name." % (name,))
48
def _check_name_encoding(name):
49
"""Check that 'name' is valid UTF-8.
51
This is separate from _check_name because UTF-8 decoding is relatively
52
expensive, and we usually want to avoid it.
54
:raises InvalidRecordError: if name is not valid UTF-8.
58
except UnicodeDecodeError, e:
59
raise errors.InvalidRecordError(str(e))
62
class ContainerSerialiser(object):
63
"""A helper class for serialising containers.
65
It simply returns bytes from method calls to 'begin', 'end' and
66
'bytes_record'. You may find ContainerWriter to be a more convenient
71
"""Return the bytes to begin a container."""
72
return FORMAT_ONE + "\n"
75
"""Return the bytes to finish a container."""
78
def bytes_record(self, bytes, names):
79
"""Return the bytes for a Bytes record with the given name and
85
byte_sections.append(str(len(bytes)) + "\n")
87
for name_tuple in names:
88
# Make sure we're writing valid names. Note that we will leave a
89
# half-written record if a name is bad!
90
for name in name_tuple:
92
byte_sections.append('\x00'.join(name_tuple) + "\n")
94
byte_sections.append("\n")
95
# Finally, the contents.
96
byte_sections.append(bytes)
97
# XXX: This causes a memory copy of bytes in size, but is usually
98
# faster than two write calls (12 vs 13 seconds to output a gig of
99
# 1k records.) - results may differ on significantly larger records
100
# like .iso's but as they should be rare in any case and thus not
101
# likely to be the common case. The biggest issue is causing extreme
102
# memory pressure in that case. One possibly improvement here is to
103
# check the size of the content before deciding to join here vs call
105
return ''.join(byte_sections)
108
class ContainerWriter(object):
109
"""A class for writing containers to a file.
111
:attribute records_written: The number of user records added to the
112
container. This does not count the prelude or suffix of the container
113
introduced by the begin() and end() methods.
116
def __init__(self, write_func):
119
:param write_func: a callable that will be called when this
120
ContainerWriter needs to write some bytes.
122
self._write_func = write_func
123
self.current_offset = 0
124
self.records_written = 0
125
self._serialiser = ContainerSerialiser()
128
"""Begin writing a container."""
129
self.write_func(self._serialiser.begin())
131
def write_func(self, bytes):
132
self._write_func(bytes)
133
self.current_offset += len(bytes)
136
"""Finish writing a container."""
137
self.write_func(self._serialiser.end())
139
def add_bytes_record(self, bytes, names):
140
"""Add a Bytes record with the given names.
142
:param bytes: The bytes to insert.
143
:param names: The names to give the inserted bytes. Each name is
144
a tuple of bytestrings. The bytestrings may not contain
146
:return: An offset, length tuple. The offset is the offset
147
of the record within the container, and the length is the
148
length of data that will need to be read to reconstitute the
149
record. These offset and length can only be used with the pack
150
interface - they might be offset by headers or other such details
151
and thus are only suitable for use by a ContainerReader.
153
current_offset = self.current_offset
154
serialised_record = self._serialiser.bytes_record(bytes, names)
155
self.write_func(serialised_record)
156
self.records_written += 1
157
# return a memo of where we wrote data to allow random access.
158
return current_offset, self.current_offset - current_offset
161
class ReadVFile(object):
162
"""Adapt a readv result iterator to a file like protocol."""
164
def __init__(self, readv_result):
165
self.readv_result = readv_result
166
# the most recent readv result block
170
if (self._string is None or
171
self._string.tell() == self._string_length):
172
length, data = self.readv_result.next()
173
self._string_length = len(data)
174
self._string = StringIO(data)
176
def read(self, length):
178
result = self._string.read(length)
179
if len(result) < length:
180
raise errors.BzrError('request for too much data from a readv hunk.')
184
"""Note that readline will not cross readv segments."""
186
result = self._string.readline()
187
if self._string.tell() == self._string_length and result[-1] != '\n':
188
raise errors.BzrError('short readline in the readvfile hunk.')
192
def make_readv_reader(transport, filename, requested_records):
193
"""Create a ContainerReader that will read selected records only.
195
:param transport: The transport the pack file is located on.
196
:param filename: The filename of the pack file.
197
:param requested_records: The record offset, length tuples as returned
198
by add_bytes_record for the desired records.
200
readv_blocks = [(0, len(FORMAT_ONE)+1)]
201
readv_blocks.extend(requested_records)
202
result = ContainerReader(ReadVFile(
203
transport.readv(filename, readv_blocks)))
207
class BaseReader(object):
209
def __init__(self, source_file):
212
:param source_file: a file-like object with `read` and `readline`
215
self._source = source_file
217
def reader_func(self, length=None):
218
return self._source.read(length)
220
def _read_line(self):
221
line = self._source.readline()
222
if not line.endswith('\n'):
223
raise errors.UnexpectedEndOfContainerError()
224
return line.rstrip('\n')
227
class ContainerReader(BaseReader):
228
"""A class for reading Bazaar's container format."""
230
def iter_records(self):
231
"""Iterate over the container, yielding each record as it is read.
233
Each yielded record will be a 2-tuple of (names, callable), where names
234
is a ``list`` and bytes is a function that takes one argument,
237
You **must not** call the callable after advancing the interator to the
238
next record. That is, this code is invalid::
240
record_iter = container.iter_records()
241
names1, callable1 = record_iter.next()
242
names2, callable2 = record_iter.next()
243
bytes1 = callable1(None)
245
As it will give incorrect results and invalidate the state of the
248
:raises ContainerError: if any sort of containter corruption is
249
detected, e.g. UnknownContainerFormatError is the format of the
250
container is unrecognised.
251
:seealso: ContainerReader.read
254
return self._iter_records()
256
def iter_record_objects(self):
257
"""Iterate over the container, yielding each record as it is read.
259
Each yielded record will be an object with ``read`` and ``validate``
260
methods. Like with iter_records, it is not safe to use a record object
261
after advancing the iterator to yield next record.
263
:raises ContainerError: if any sort of containter corruption is
264
detected, e.g. UnknownContainerFormatError is the format of the
265
container is unrecognised.
266
:seealso: iter_records
269
return self._iter_record_objects()
271
def _iter_records(self):
272
for record in self._iter_record_objects():
275
def _iter_record_objects(self):
277
record_kind = self.reader_func(1)
278
if record_kind == 'B':
280
reader = BytesRecordReader(self._source)
282
elif record_kind == 'E':
283
# End marker. There are no more records.
285
elif record_kind == '':
286
# End of stream encountered, but no End Marker record seen, so
287
# this container is incomplete.
288
raise errors.UnexpectedEndOfContainerError()
290
# Unknown record type.
291
raise errors.UnknownRecordTypeError(record_kind)
293
def _read_format(self):
294
format = self._read_line()
295
if format != FORMAT_ONE:
296
raise errors.UnknownContainerFormatError(format)
299
"""Validate this container and its records.
301
Validating consumes the data stream just like iter_records and
302
iter_record_objects, so you cannot call it after
303
iter_records/iter_record_objects.
305
:raises ContainerError: if something is invalid.
308
for record_names, read_bytes in self.iter_records():
310
for name_tuple in record_names:
311
for name in name_tuple:
312
_check_name_encoding(name)
313
# Check that the name is unique. Note that Python will refuse
314
# to decode non-shortest forms of UTF-8 encoding, so there is no
315
# risk that the same unicode string has been encoded two
317
if name_tuple in all_names:
318
raise errors.DuplicateRecordNameError(name_tuple)
319
all_names.add(name_tuple)
320
excess_bytes = self.reader_func(1)
321
if excess_bytes != '':
322
raise errors.ContainerHasExcessDataError(excess_bytes)
325
class BytesRecordReader(BaseReader):
330
You can either validate or read a record, you can't do both.
332
:returns: A tuple of (names, callable). The callable can be called
333
repeatedly to obtain the bytes for the record, with a max_length
334
argument. If max_length is None, returns all the bytes. Because
335
records can be arbitrarily large, using None is not recommended
336
unless you have reason to believe the content will fit in memory.
338
# Read the content length.
339
length_line = self._read_line()
341
length = int(length_line)
343
raise errors.InvalidRecordError(
344
"%r is not a valid length." % (length_line,))
346
# Read the list of names.
349
name_line = self._read_line()
352
name_tuple = tuple(name_line.split('\x00'))
353
for name in name_tuple:
355
names.append(name_tuple)
357
self._remaining_length = length
358
return names, self._content_reader
360
def _content_reader(self, max_length):
361
if max_length is None:
362
length_to_read = self._remaining_length
364
length_to_read = min(max_length, self._remaining_length)
365
self._remaining_length -= length_to_read
366
bytes = self.reader_func(length_to_read)
367
if len(bytes) != length_to_read:
368
raise errors.UnexpectedEndOfContainerError()
372
"""Validate this record.
374
You can either validate or read, you can't do both.
376
:raises ContainerError: if this record is invalid.
378
names, read_bytes = self.read()
379
for name_tuple in names:
380
for name in name_tuple:
381
_check_name_encoding(name)
385
class ContainerPushParser(object):
386
"""A "push" parser for container format 1.
388
It accepts bytes via the ``accept_bytes`` method, and parses them into
389
records which can be retrieved via the ``read_pending_records`` method.
394
self._state_handler = self._state_expecting_format_line
395
self._parsed_records = []
396
self._reset_current_record()
397
self.finished = False
399
def _reset_current_record(self):
400
self._current_record_length = None
401
self._current_record_names = []
403
def accept_bytes(self, bytes):
404
self._buffer += bytes
405
# Keep iterating the state machine until it stops consuming bytes from
407
last_buffer_length = None
408
cur_buffer_length = len(self._buffer)
409
while cur_buffer_length != last_buffer_length:
410
last_buffer_length = cur_buffer_length
411
self._state_handler()
412
cur_buffer_length = len(self._buffer)
414
def read_pending_records(self, max=None):
416
records = self._parsed_records[:max]
417
del self._parsed_records[:max]
420
records = self._parsed_records
421
self._parsed_records = []
424
def _consume_line(self):
425
"""Take a line out of the buffer, and return the line.
427
If a newline byte is not found in the buffer, the buffer is
428
unchanged and this returns None instead.
430
newline_pos = self._buffer.find('\n')
431
if newline_pos != -1:
432
line = self._buffer[:newline_pos]
433
self._buffer = self._buffer[newline_pos+1:]
438
def _state_expecting_format_line(self):
439
line = self._consume_line()
441
if line != FORMAT_ONE:
442
raise errors.UnknownContainerFormatError(line)
443
self._state_handler = self._state_expecting_record_type
445
def _state_expecting_record_type(self):
446
if len(self._buffer) >= 1:
447
record_type = self._buffer[0]
448
self._buffer = self._buffer[1:]
449
if record_type == 'B':
450
self._state_handler = self._state_expecting_length
451
elif record_type == 'E':
453
self._state_handler = self._state_expecting_nothing
455
raise errors.UnknownRecordTypeError(record_type)
457
def _state_expecting_length(self):
458
line = self._consume_line()
461
self._current_record_length = int(line)
463
raise errors.InvalidRecordError(
464
"%r is not a valid length." % (line,))
465
self._state_handler = self._state_expecting_name
467
def _state_expecting_name(self):
468
encoded_name_parts = self._consume_line()
469
if encoded_name_parts == '':
470
self._state_handler = self._state_expecting_body
471
elif encoded_name_parts:
472
name_parts = tuple(encoded_name_parts.split('\x00'))
473
for name_part in name_parts:
474
_check_name(name_part)
475
self._current_record_names.append(name_parts)
477
def _state_expecting_body(self):
478
if len(self._buffer) >= self._current_record_length:
479
body_bytes = self._buffer[:self._current_record_length]
480
self._buffer = self._buffer[self._current_record_length:]
481
record = (self._current_record_names, body_bytes)
482
self._parsed_records.append(record)
483
self._reset_current_record()
484
self._state_handler = self._state_expecting_record_type
486
def _state_expecting_nothing(self):
489
def read_size_hint(self):
491
if self._state_handler == self._state_expecting_body:
492
remaining = self._current_record_length - len(self._buffer)
495
return max(hint, remaining)
499
def iter_records_from_file(source_file):
500
parser = ContainerPushParser()
502
bytes = source_file.read(parser.read_size_hint())
503
parser.accept_bytes(bytes)
504
for record in parser.read_pending_records():
added
removed
bzrlib/pack.py
