1
# Copyright (C) 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from cStringIO import StringIO
28
revision as _mod_revision,
32
from bzrlib.bundle import bundle_data, serializer
33
from bzrlib.util import bencode
36
class BundleWriter(object):
37
"""Writer for bundle-format files.
39
This serves roughly the same purpose as ContainerReader, but acts as a
42
Provides ways of writing the specific record types supported this bundle
46
def __init__(self, fileobj):
47
self._container = pack.ContainerWriter(self._write_encoded)
48
self._fileobj = fileobj
49
self._compressor = bz2.BZ2Compressor()
51
def _write_encoded(self, bytes):
52
"""Write bzip2-encoded bytes to the file"""
53
self._fileobj.write(self._compressor.compress(bytes))
56
"""Start writing the bundle"""
57
self._fileobj.write(serializer._get_bundle_header(
58
serializer.v4_string))
59
self._fileobj.write('#\n')
60
self._container.begin()
63
"""Finish writing the bundle"""
65
self._fileobj.write(self._compressor.flush())
67
def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
68
revision_id, file_id):
69
"""Add a record for a multi-parent diff
71
:mp_bytes: A multi-parent diff, as a bytestring
72
:sha1: The sha1 hash of the fulltext
73
:parents: a list of revision-ids of the parents
74
:repo_kind: The kind of object in the repository. May be 'file' or
76
:revision_id: The revision id of the mpdiff being added.
77
:file_id: The file-id of the file, or None for inventories.
79
metadata = {'parents': parents,
80
'storage_kind': 'mpdiff',
82
self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
84
def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
85
"""Add a record for a fulltext
87
:bytes: The fulltext, as a bytestring
88
:parents: a list of revision-ids of the parents
89
:repo_kind: The kind of object in the repository. May be 'revision' or
91
:revision_id: The revision id of the fulltext being added.
93
metadata = {'parents': parents,
94
'storage_kind': 'mpdiff'}
95
self._add_record(bytes, {'parents': parents,
96
'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
98
def add_info_record(self, **kwargs):
99
"""Add an info record to the bundle
101
Any parameters may be supplied, except 'self' and 'storage_kind'.
102
Values must be lists, strings, integers, dicts, or a combination.
104
kwargs['storage_kind'] = 'header'
105
self._add_record(None, kwargs, 'info', None, None)
108
def encode_name(content_kind, revision_id, file_id=None):
109
"""Encode semantic ids as a container name"""
110
assert content_kind in ('revision', 'file', 'inventory', 'signature',
113
if content_kind == 'file':
114
assert file_id is not None
116
assert file_id is None
117
if content_kind == 'info':
118
assert revision_id is None
120
assert revision_id is not None
121
names = [n.replace('/', '//') for n in
122
(content_kind, revision_id, file_id) if n is not None]
123
return '/'.join(names)
125
def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
126
"""Add a bundle record to the container.
128
Most bundle records are recorded as header/body pairs, with the
129
body being nameless. Records with storage_kind 'header' have no
132
name = self.encode_name(repo_kind, revision_id, file_id)
133
encoded_metadata = bencode.bencode(metadata)
134
self._container.add_bytes_record(encoded_metadata, [name])
135
if metadata['storage_kind'] != 'header':
136
self._container.add_bytes_record(bytes, [])
139
class BundleReader(object):
140
"""Reader for bundle-format files.
142
This serves roughly the same purpose as ContainerReader, but acts as a
143
layer on top of it, providing metadata, a semantic name, and a record
147
def __init__(self, fileobj):
148
line = fileobj.readline()
151
self.patch_lines = []
152
self._container = pack.ContainerReader(
153
iterablefile.IterableFile(self.iter_decode(fileobj)))
156
def iter_decode(fileobj):
157
"""Iterate through decoded fragments of the file"""
158
decompressor = bz2.BZ2Decompressor()
160
yield decompressor.decompress(line)
163
def decode_name(name):
164
"""Decode a name from its container form into a semantic form
166
:retval: content_kind, revision_id, file_id
168
segments = re.split('(//?)', name)
170
for segment in segments:
177
content_kind = names[0]
181
revision_id = names[1]
184
return content_kind, revision_id, file_id
186
def iter_records(self):
187
"""Iterate through bundle records
189
:return: a generator of (bytes, metadata, content_kind, revision_id,
192
iterator = self._container.iter_records()
193
for names, meta_bytes in iterator:
195
raise errors.BadBundle('Record has %d names instead of 1'
197
metadata = bencode.bdecode(meta_bytes(None))
198
if metadata['storage_kind'] == 'header':
201
_unused, bytes = iterator.next()
203
yield (bytes, metadata) + self.decode_name(names[0])
206
class BundleSerializerV4(serializer.BundleSerializer):
207
"""Implement the high-level bundle interface"""
209
def write(self, repository, revision_ids, forced_bases, fileobj):
210
"""Write a bundle to a file-like object
212
For backwards-compatibility only
214
write_op = BundleWriteOperation.from_old_args(repository, revision_ids,
215
forced_bases, fileobj)
216
return write_op.do_write()
218
def write_bundle(self, repository, target, base, fileobj):
219
"""Write a bundle to a file object
221
:param repository: The repository to retrieve revision data from
222
:param target: The head revision to include ancestors of
223
:param base: The ancestor of the target to stop including acestors
225
:param fileobj: The file-like object to write to
227
write_op = BundleWriteOperation(base, target, repository, fileobj)
228
return write_op.do_write()
230
def read(self, file):
231
"""return a reader object for a given file"""
232
bundle = BundleInfoV4(file, self)
236
def get_source_serializer(info):
237
"""Retrieve the serializer for a given info object"""
238
return xml_serializer.format_registry.get(info['serializer'])
241
class BundleWriteOperation(object):
242
"""Perform the operation of writing revisions to a bundle"""
245
def from_old_args(cls, repository, revision_ids, forced_bases, fileobj):
246
"""Create a BundleWriteOperation from old-style arguments"""
247
base, target = cls.get_base_target(revision_ids, forced_bases,
249
return BundleWriteOperation(base, target, repository, fileobj,
252
def __init__(self, base, target, repository, fileobj, revision_ids=None):
255
self.repository = repository
256
bundle = BundleWriter(fileobj)
258
self.base_ancestry = set(repository.get_ancestry(base,
260
if revision_ids is not None:
261
self.revision_ids = revision_ids
263
revision_ids = set(repository.get_ancestry(target,
265
self.revision_ids = revision_ids.difference(self.base_ancestry)
268
"""Write all data to the bundle"""
272
self.write_revisions()
274
return self.revision_ids
276
def write_info(self):
277
"""Write format info"""
278
serializer_format = self.repository.get_serializer_format()
279
supports_rich_root = {True: 1, False: 0}[
280
self.repository.supports_rich_root()]
281
self.bundle.add_info_record(serializer=serializer_format,
282
supports_rich_root=supports_rich_root)
284
def iter_file_revisions(self):
285
"""Iterate through all relevant revisions of all files.
287
This is the correct implementation, but is not compatible with bzr.dev,
288
because certain old revisions were not converted correctly, and have
289
the wrong "revision" marker in inventories.
291
transaction = self.repository.get_transaction()
292
altered = self.repository.fileids_altered_by_revision_ids(
294
for file_id, file_revision_ids in altered.iteritems():
295
vf = self.repository.weave_store.get_weave(file_id, transaction)
296
yield vf, file_id, file_revision_ids
298
def iter_file_revisions_aggressive(self):
299
"""Iterate through all relevant revisions of all files.
301
This uses the standard iter_file_revisions to determine what revisions
302
are referred to by inventories, but then uses the versionedfile to
303
determine what the build-dependencies of each required revision.
305
All build dependencies which are not ancestors of the base revision
308
for vf, file_id, file_revision_ids in self.iter_file_revisions():
309
new_revision_ids = set()
310
pending = list(file_revision_ids)
311
while len(pending) > 0:
312
revision_id = pending.pop()
313
if revision_id in new_revision_ids:
315
if revision_id in self.base_ancestry:
317
new_revision_ids.add(revision_id)
318
pending.extend(vf.get_parents(revision_id))
319
yield vf, file_id, new_revision_ids
321
def write_files(self):
322
"""Write bundle records for all revisions of all files"""
323
for vf, file_id, revision_ids in self.iter_file_revisions_aggressive():
324
self.add_mp_records('file', file_id, vf, revision_ids)
326
def write_revisions(self):
327
"""Write bundle records for all revisions and signatures"""
328
inv_vf = self.repository.get_inventory_weave()
329
revision_order = list(multiparent.topo_iter(inv_vf, self.revision_ids))
330
if self.target is not None and self.target in self.revision_ids:
331
revision_order.remove(self.target)
332
revision_order.append(self.target)
333
self.add_mp_records('inventory', None, inv_vf, revision_order)
334
parents_list = self.repository.get_parents(revision_order)
335
for parents, revision_id in zip(parents_list, revision_order):
336
revision_text = self.repository.get_revision_xml(revision_id)
337
self.bundle.add_fulltext_record(revision_text, parents,
338
'revision', revision_id)
340
self.bundle.add_fulltext_record(
341
self.repository.get_signature_text(
342
revision_id), parents, 'signature', revision_id)
343
except errors.NoSuchRevision:
347
def get_base_target(revision_ids, forced_bases, repository):
348
"""Determine the base and target from old-style revision ids"""
349
if len(revision_ids) == 0:
351
target = revision_ids[0]
352
base = forced_bases.get(target)
354
parents = repository.get_revision(target).parent_ids
355
if len(parents) == 0:
356
base = _mod_revision.NULL_REVISION
361
def add_mp_records(self, repo_kind, file_id, vf, revision_ids):
362
"""Add multi-parent diff records to a bundle"""
363
revision_ids = list(multiparent.topo_iter(vf, revision_ids))
364
mpdiffs = vf.make_mpdiffs(revision_ids)
365
sha1s = vf.get_sha1s(revision_ids)
366
for mpdiff, revision_id, sha1, in zip(mpdiffs, revision_ids, sha1s):
367
parents = vf.get_parents(revision_id)
368
text = ''.join(mpdiff.to_patch())
369
self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
370
revision_id, file_id)
373
class BundleInfoV4(object):
375
"""Provide (most of) the BundleInfo interface"""
376
def __init__(self, fileobj, serializer):
377
self._fileobj = fileobj
378
self._serializer = serializer
379
self.__real_revisions = None
380
self.__revisions = None
382
def install(self, repository):
383
return self.install_revisions(repository)
385
def install_revisions(self, repository):
386
"""Install this bundle's revisions into the specified repository"""
387
repository.lock_write()
389
ri = RevisionInstaller(self.get_bundle_reader(),
390
self._serializer, repository)
395
def get_merge_request(self, target_repo):
396
"""Provide data for performing a merge
398
Returns suggested base, suggested target, and patch verification status
400
return None, self.target, 'inapplicable'
402
def get_bundle_reader(self):
403
self._fileobj.seek(0)
404
return BundleReader(self._fileobj)
406
def _get_real_revisions(self):
407
if self.__real_revisions is None:
408
self.__real_revisions = []
409
bundle_reader = self.get_bundle_reader()
410
for bytes, metadata, repo_kind, revision_id, file_id in \
411
bundle_reader.iter_records():
412
if repo_kind == 'info':
414
self._serializer.get_source_serializer(metadata)
415
if repo_kind == 'revision':
416
rev = serializer.read_revision_from_string(bytes)
417
self.__real_revisions.append(rev)
418
return self.__real_revisions
419
real_revisions = property(_get_real_revisions)
421
def _get_revisions(self):
422
if self.__revisions is None:
423
self.__revisions = []
424
for revision in self.real_revisions:
425
self.__revisions.append(
426
bundle_data.RevisionInfo.from_revision(revision))
427
return self.__revisions
429
revisions = property(_get_revisions)
431
def _get_target(self):
432
return self.revisions[-1].revision_id
434
target = property(_get_target)
437
class RevisionInstaller(object):
438
"""Installs revisions into a repository"""
440
def __init__(self, container, serializer, repository):
441
self._container = container
442
self._serializer = serializer
443
self._repository = repository
447
"""Perform the installation"""
449
current_versionedfile = None
450
pending_file_records = []
452
target_revision = None
453
for bytes, metadata, repo_kind, revision_id, file_id in\
454
self._container.iter_records():
455
if repo_kind == 'info':
456
assert self._info is None
457
self._handle_info(metadata)
458
if repo_kind != 'file':
459
self._install_mp_records(current_versionedfile,
460
pending_file_records)
462
current_versionedfile = None
463
pending_file_records = []
464
if repo_kind == 'inventory':
465
self._install_inventory(revision_id, metadata, bytes)
466
if repo_kind == 'revision':
467
target_revision = revision_id
468
self._install_revision(revision_id, metadata, bytes)
469
if repo_kind == 'signature':
470
self._install_signature(revision_id, metadata, bytes)
471
if repo_kind == 'file':
472
if file_id != current_file:
473
self._install_mp_records(current_versionedfile,
474
pending_file_records)
475
current_file = file_id
476
current_versionedfile = \
477
self._repository.weave_store.get_weave_or_empty(
478
file_id, self._repository.get_transaction())
479
pending_file_records = []
480
if revision_id in current_versionedfile:
482
pending_file_records.append((revision_id, metadata, bytes))
483
self._install_mp_records(current_versionedfile, pending_file_records)
484
return target_revision
486
def _handle_info(self, info):
487
"""Extract data from an info record"""
489
self._source_serializer = self._serializer.get_source_serializer(info)
490
if (info['supports_rich_root'] == 0 and
491
self._repository.supports_rich_root()):
492
self.update_root = True
494
self.update_root = False
496
def _install_mp_records(self, versionedfile, records):
497
if len(records) == 0:
499
d_func = multiparent.MultiParent.from_patch
500
vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
501
records if r not in versionedfile]
502
versionedfile.add_mpdiffs(vf_records)
504
def _install_inventory(self, revision_id, metadata, text):
505
vf = self._repository.get_inventory_weave()
506
if revision_id in vf:
508
parent_ids = metadata['parents']
509
if self._info['serializer'] == self._repository._serializer.format_num:
510
return self._install_mp_records(vf, [(revision_id, metadata,
512
parents = [self._repository.get_inventory(p)
514
parent_texts = [self._source_serializer.write_inventory_to_string(p)
516
target_lines = multiparent.MultiParent.from_patch(text).to_lines(
518
sha1 = osutils.sha_strings(target_lines)
519
if sha1 != metadata['sha1']:
520
raise errors.BadBundle("Can't convert to target format")
521
target_inv = self._source_serializer.read_inventory_from_string(
522
''.join(target_lines))
523
self._handle_root(target_inv, parent_ids)
525
self._repository.add_inventory(revision_id, target_inv, parent_ids)
526
except errors.UnsupportedInventoryKind:
527
raise errors.IncompatibleRevision(repr(self._repository))
529
def _handle_root(self, target_inv, parent_ids):
530
revision_id = target_inv.revision_id
532
target_inv.root.revision = revision_id
533
store = self._repository.weave_store
534
transaction = self._repository.get_transaction()
535
vf = store.get_weave_or_empty(target_inv.root.file_id, transaction)
536
vf.add_lines(revision_id, parent_ids, [])
537
elif not self._repository.supports_rich_root():
538
if target_inv.root.revision != revision_id:
539
raise errors.IncompatibleRevision(repr(self._repository))
542
def _install_revision(self, revision_id, metadata, text):
543
if self._repository.has_revision(revision_id):
545
self._repository._add_revision_text(revision_id, text)
547
def _install_signature(self, revision_id, metadata, text):
548
transaction = self._repository.get_transaction()
549
if self._repository._revision_store.has_signature(revision_id,
552
self._repository._revision_store.add_revision_signature_text(
553
revision_id, text, transaction)
added
removed
bzrlib/bundle/serializer/v4.py
