« back to all changes in this revision
Viewing changes to bzrlib/bundle/serializer/v4.py
- browse files at revision 2631
- compare with another revision
- download diff
- download tarball
- view history from revision 2631
- Committer: Canonical.com Patch Queue Manager
- Date: 2007-07-19 16:09:34 UTC
- mfrom: (2520.4.135 bzr.mpbundle)
- Revision ID: pqm@pqm.ubuntu.com-20070719160934-d51fyijw69oto88p
Add new bundle and merge-directive formats
- files added:
- bzrlib/bundle/serializer/v4.py
- files removed:
- bzrlib/bundle/common.py
- bzrlib/bundle/old
- files renamed:
- bzrlib/tests/blackbox/test_bundle.py => bzrlib/tests/blackbox/test_submit.py
- files modified:
- NEWS
added
removed
bzrlib/bundle/serializer/v4.py
1
# Copyright (C) 2007 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
17
from cStringIO import StringIO
18
import bz2
19
import re
20
21
from bzrlib import (
22
diff,
23
errors,
24
iterablefile,
25
multiparent,
26
osutils,
27
pack,
28
revision as _mod_revision,
29
trace,
30
xml_serializer,
31
)
32
from bzrlib.bundle import bundle_data, serializer
33
from bzrlib.util import bencode
34
35
36
class BundleWriter(object):
37
"""Writer for bundle-format files.
38
39
This serves roughly the same purpose as ContainerReader, but acts as a
40
layer on top of it.
41
42
Provides ways of writing the specific record types supported this bundle
43
format.
44
"""
45
46
def __init__(self, fileobj):
47
self._container = pack.ContainerWriter(self._write_encoded)
48
self._fileobj = fileobj
49
self._compressor = bz2.BZ2Compressor()
50
51
def _write_encoded(self, bytes):
52
"""Write bzip2-encoded bytes to the file"""
53
self._fileobj.write(self._compressor.compress(bytes))
54
55
def begin(self):
56
"""Start writing the bundle"""
57
self._fileobj.write(serializer._get_bundle_header(
58
serializer.v4_string))
59
self._fileobj.write('#\n')
60
self._container.begin()
61
62
def end(self):
63
"""Finish writing the bundle"""
64
self._container.end()
65
self._fileobj.write(self._compressor.flush())
66
67
def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
68
revision_id, file_id):
69
"""Add a record for a multi-parent diff
70
71
:mp_bytes: A multi-parent diff, as a bytestring
72
:sha1: The sha1 hash of the fulltext
73
:parents: a list of revision-ids of the parents
74
:repo_kind: The kind of object in the repository. May be 'file' or
75
'inventory'
76
:revision_id: The revision id of the mpdiff being added.
77
:file_id: The file-id of the file, or None for inventories.
78
"""
79
metadata = {'parents': parents,
80
'storage_kind': 'mpdiff',
81
'sha1': sha1}
82
self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
83
84
def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
85
"""Add a record for a fulltext
86
87
:bytes: The fulltext, as a bytestring
88
:parents: a list of revision-ids of the parents
89
:repo_kind: The kind of object in the repository. May be 'revision' or
90
'signature'
91
:revision_id: The revision id of the fulltext being added.
92
"""
93
metadata = {'parents': parents,
94
'storage_kind': 'mpdiff'}
95
self._add_record(bytes, {'parents': parents,
96
'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
97
98
def add_info_record(self, **kwargs):
99
"""Add an info record to the bundle
100
101
Any parameters may be supplied, except 'self' and 'storage_kind'.
102
Values must be lists, strings, integers, dicts, or a combination.
103
"""
104
kwargs['storage_kind'] = 'header'
105
self._add_record(None, kwargs, 'info', None, None)
106
107
@staticmethod
108
def encode_name(content_kind, revision_id, file_id=None):
109
"""Encode semantic ids as a container name"""
110
assert content_kind in ('revision', 'file', 'inventory', 'signature',
111
'info')
112
113
if content_kind == 'file':
114
assert file_id is not None
115
else:
116
assert file_id is None
117
if content_kind == 'info':
118
assert revision_id is None
119
else:
120
assert revision_id is not None
121
names = [n.replace('/', '//') for n in
122
(content_kind, revision_id, file_id) if n is not None]
123
return '/'.join(names)
124
125
def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
126
"""Add a bundle record to the container.
127
128
Most bundle records are recorded as header/body pairs, with the
129
body being nameless. Records with storage_kind 'header' have no
130
body.
131
"""
132
name = self.encode_name(repo_kind, revision_id, file_id)
133
encoded_metadata = bencode.bencode(metadata)
134
self._container.add_bytes_record(encoded_metadata, [name])
135
if metadata['storage_kind'] != 'header':
136
self._container.add_bytes_record(bytes, [])
137
138
139
class BundleReader(object):
140
"""Reader for bundle-format files.
141
142
This serves roughly the same purpose as ContainerReader, but acts as a
143
layer on top of it, providing metadata, a semantic name, and a record
144
body
145
"""
146
147
def __init__(self, fileobj):
148
line = fileobj.readline()
149
if line != '\n':
150
fileobj.readline()
151
self.patch_lines = []
152
self._container = pack.ContainerReader(
153
iterablefile.IterableFile(self.iter_decode(fileobj)))
154
155
@staticmethod
156
def iter_decode(fileobj):
157
"""Iterate through decoded fragments of the file"""
158
decompressor = bz2.BZ2Decompressor()
159
for line in fileobj:
160
yield decompressor.decompress(line)
161
162
@staticmethod
163
def decode_name(name):
164
"""Decode a name from its container form into a semantic form
165
166
:retval: content_kind, revision_id, file_id
167
"""
168
segments = re.split('(//?)', name)
169
names = ['']
170
for segment in segments:
171
if segment == '//':
172
names[-1] += '/'
173
elif segment == '/':
174
names.append('')
175
else:
176
names[-1] += segment
177
content_kind = names[0]
178
revision_id = None
179
file_id = None
180
if len(names) > 1:
181
revision_id = names[1]
182
if len(names) > 2:
183
file_id = names[2]
184
return content_kind, revision_id, file_id
185
186
def iter_records(self):
187
"""Iterate through bundle records
188
189
:return: a generator of (bytes, metadata, content_kind, revision_id,
190
file_id)
191
"""
192
iterator = self._container.iter_records()
193
for names, meta_bytes in iterator:
194
if len(names) != 1:
195
raise errors.BadBundle('Record has %d names instead of 1'
196
% len(names))
197
metadata = bencode.bdecode(meta_bytes(None))
198
if metadata['storage_kind'] == 'header':
199
bytes = None
200
else:
201
_unused, bytes = iterator.next()
202
bytes = bytes(None)
203
yield (bytes, metadata) + self.decode_name(names[0])
204
205
206
class BundleSerializerV4(serializer.BundleSerializer):
207
"""Implement the high-level bundle interface"""
208
209
def write(self, repository, revision_ids, forced_bases, fileobj):
210
"""Write a bundle to a file-like object
211
212
For backwards-compatibility only
213
"""
214
write_op = BundleWriteOperation.from_old_args(repository, revision_ids,
215
forced_bases, fileobj)
216
return write_op.do_write()
217
218
def write_bundle(self, repository, target, base, fileobj):
219
"""Write a bundle to a file object
220
221
:param repository: The repository to retrieve revision data from
222
:param target: The head revision to include ancestors of
223
:param base: The ancestor of the target to stop including acestors
224
at.
225
:param fileobj: The file-like object to write to
226
"""
227
write_op = BundleWriteOperation(base, target, repository, fileobj)
228
return write_op.do_write()
229
230
def read(self, file):
231
"""return a reader object for a given file"""
232
bundle = BundleInfoV4(file, self)
233
return bundle
234
235
@staticmethod
236
def get_source_serializer(info):
237
"""Retrieve the serializer for a given info object"""
238
return xml_serializer.format_registry.get(info['serializer'])
239
240
241
class BundleWriteOperation(object):
242
"""Perform the operation of writing revisions to a bundle"""
243
244
@classmethod
245
def from_old_args(cls, repository, revision_ids, forced_bases, fileobj):
246
"""Create a BundleWriteOperation from old-style arguments"""
247
base, target = cls.get_base_target(revision_ids, forced_bases,
248
repository)
249
return BundleWriteOperation(base, target, repository, fileobj,
250
revision_ids)
251
252
def __init__(self, base, target, repository, fileobj, revision_ids=None):
253
self.base = base
254
self.target = target
255
self.repository = repository
256
bundle = BundleWriter(fileobj)
257
self.bundle = bundle
258
self.base_ancestry = set(repository.get_ancestry(base,
259
topo_sorted=False))
260
if revision_ids is not None:
261
self.revision_ids = revision_ids
262
else:
263
revision_ids = set(repository.get_ancestry(target,
264
topo_sorted=False))
265
self.revision_ids = revision_ids.difference(self.base_ancestry)
266
267
def do_write(self):
268
"""Write all data to the bundle"""
269
self.bundle.begin()
270
self.write_info()
271
self.write_files()
272
self.write_revisions()
273
self.bundle.end()
274
return self.revision_ids
275
276
def write_info(self):
277
"""Write format info"""
278
serializer_format = self.repository.get_serializer_format()
279
supports_rich_root = {True: 1, False: 0}[
280
self.repository.supports_rich_root()]
281
self.bundle.add_info_record(serializer=serializer_format,
282
supports_rich_root=supports_rich_root)
283
284
def iter_file_revisions(self):
285
"""Iterate through all relevant revisions of all files.
286
287
This is the correct implementation, but is not compatible with bzr.dev,
288
because certain old revisions were not converted correctly, and have
289
the wrong "revision" marker in inventories.
290
"""
291
transaction = self.repository.get_transaction()
292
altered = self.repository.fileids_altered_by_revision_ids(
293
self.revision_ids)
294
for file_id, file_revision_ids in altered.iteritems():
295
vf = self.repository.weave_store.get_weave(file_id, transaction)
296
yield vf, file_id, file_revision_ids
297
298
def iter_file_revisions_aggressive(self):
299
"""Iterate through all relevant revisions of all files.
300
301
This uses the standard iter_file_revisions to determine what revisions
302
are referred to by inventories, but then uses the versionedfile to
303
determine what the build-dependencies of each required revision.
304
305
All build dependencies which are not ancestors of the base revision
306
are emitted.
307
"""
308
for vf, file_id, file_revision_ids in self.iter_file_revisions():
309
new_revision_ids = set()
310
pending = list(file_revision_ids)
311
while len(pending) > 0:
312
revision_id = pending.pop()
313
if revision_id in new_revision_ids:
314
continue
315
if revision_id in self.base_ancestry:
316
continue
317
new_revision_ids.add(revision_id)
318
pending.extend(vf.get_parents(revision_id))
319
yield vf, file_id, new_revision_ids
320
321
def write_files(self):
322
"""Write bundle records for all revisions of all files"""
323
for vf, file_id, revision_ids in self.iter_file_revisions_aggressive():
324
self.add_mp_records('file', file_id, vf, revision_ids)
325
326
def write_revisions(self):
327
"""Write bundle records for all revisions and signatures"""
328
inv_vf = self.repository.get_inventory_weave()
329
revision_order = list(multiparent.topo_iter(inv_vf, self.revision_ids))
330
if self.target is not None and self.target in self.revision_ids:
331
revision_order.remove(self.target)
332
revision_order.append(self.target)
333
self.add_mp_records('inventory', None, inv_vf, revision_order)
334
parents_list = self.repository.get_parents(revision_order)
335
for parents, revision_id in zip(parents_list, revision_order):
336
revision_text = self.repository.get_revision_xml(revision_id)
337
self.bundle.add_fulltext_record(revision_text, parents,
338
'revision', revision_id)
339
try:
340
self.bundle.add_fulltext_record(
341
self.repository.get_signature_text(
342
revision_id), parents, 'signature', revision_id)
343
except errors.NoSuchRevision:
344
pass
345
346
@staticmethod
347
def get_base_target(revision_ids, forced_bases, repository):
348
"""Determine the base and target from old-style revision ids"""
349
if len(revision_ids) == 0:
350
return None, None
351
target = revision_ids[0]
352
base = forced_bases.get(target)
353
if base is None:
354
parents = repository.get_revision(target).parent_ids
355
if len(parents) == 0:
356
base = _mod_revision.NULL_REVISION
357
else:
358
base = parents[0]
359
return base, target
360
361
def add_mp_records(self, repo_kind, file_id, vf, revision_ids):
362
"""Add multi-parent diff records to a bundle"""
363
revision_ids = list(multiparent.topo_iter(vf, revision_ids))
364
mpdiffs = vf.make_mpdiffs(revision_ids)
365
sha1s = vf.get_sha1s(revision_ids)
366
for mpdiff, revision_id, sha1, in zip(mpdiffs, revision_ids, sha1s):
367
parents = vf.get_parents(revision_id)
368
text = ''.join(mpdiff.to_patch())
369
self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
370
revision_id, file_id)
371
372
373
class BundleInfoV4(object):
374
375
"""Provide (most of) the BundleInfo interface"""
376
def __init__(self, fileobj, serializer):
377
self._fileobj = fileobj
378
self._serializer = serializer
379
self.__real_revisions = None
380
self.__revisions = None
381
382
def install(self, repository):
383
return self.install_revisions(repository)
384
385
def install_revisions(self, repository):
386
"""Install this bundle's revisions into the specified repository"""
387
repository.lock_write()
388
try:
389
ri = RevisionInstaller(self.get_bundle_reader(),
390
self._serializer, repository)
391
return ri.install()
392
finally:
393
repository.unlock()
394
395
def get_merge_request(self, target_repo):
396
"""Provide data for performing a merge
397
398
Returns suggested base, suggested target, and patch verification status
399
"""
400
return None, self.target, 'inapplicable'
401
402
def get_bundle_reader(self):
403
self._fileobj.seek(0)
404
return BundleReader(self._fileobj)
405
406
def _get_real_revisions(self):
407
if self.__real_revisions is None:
408
self.__real_revisions = []
409
bundle_reader = self.get_bundle_reader()
410
for bytes, metadata, repo_kind, revision_id, file_id in \
411
bundle_reader.iter_records():
412
if repo_kind == 'info':
413
serializer =\
414
self._serializer.get_source_serializer(metadata)
415
if repo_kind == 'revision':
416
rev = serializer.read_revision_from_string(bytes)
417
self.__real_revisions.append(rev)
418
return self.__real_revisions
419
real_revisions = property(_get_real_revisions)
420
421
def _get_revisions(self):
422
if self.__revisions is None:
423
self.__revisions = []
424
for revision in self.real_revisions:
425
self.__revisions.append(
426
bundle_data.RevisionInfo.from_revision(revision))
427
return self.__revisions
428
429
revisions = property(_get_revisions)
430
431
def _get_target(self):
432
return self.revisions[-1].revision_id
433
434
target = property(_get_target)
435
436
437
class RevisionInstaller(object):
438
"""Installs revisions into a repository"""
439
440
def __init__(self, container, serializer, repository):
441
self._container = container
442
self._serializer = serializer
443
self._repository = repository
444
self._info = None
445
446
def install(self):
447
"""Perform the installation"""
448
current_file = None
449
current_versionedfile = None
450
pending_file_records = []
451
added_inv = set()
452
target_revision = None
453
for bytes, metadata, repo_kind, revision_id, file_id in\
454
self._container.iter_records():
455
if repo_kind == 'info':
456
assert self._info is None
457
self._handle_info(metadata)
458
if repo_kind != 'file':
459
self._install_mp_records(current_versionedfile,
460
pending_file_records)
461
current_file = None
462
current_versionedfile = None
463
pending_file_records = []
464
if repo_kind == 'inventory':
465
self._install_inventory(revision_id, metadata, bytes)
466
if repo_kind == 'revision':
467
target_revision = revision_id
468
self._install_revision(revision_id, metadata, bytes)
469
if repo_kind == 'signature':
470
self._install_signature(revision_id, metadata, bytes)
471
if repo_kind == 'file':
472
if file_id != current_file:
473
self._install_mp_records(current_versionedfile,
474
pending_file_records)
475
current_file = file_id
476
current_versionedfile = \
477
self._repository.weave_store.get_weave_or_empty(
478
file_id, self._repository.get_transaction())
479
pending_file_records = []
480
if revision_id in current_versionedfile:
481
continue
482
pending_file_records.append((revision_id, metadata, bytes))
483
self._install_mp_records(current_versionedfile, pending_file_records)
484
return target_revision
485
486
def _handle_info(self, info):
487
"""Extract data from an info record"""
488
self._info = info
489
self._source_serializer = self._serializer.get_source_serializer(info)
490
if (info['supports_rich_root'] == 0 and
491
self._repository.supports_rich_root()):
492
self.update_root = True
493
else:
494
self.update_root = False
495
496
def _install_mp_records(self, versionedfile, records):
497
if len(records) == 0:
498
return
499
d_func = multiparent.MultiParent.from_patch
500
vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
501
records if r not in versionedfile]
502
versionedfile.add_mpdiffs(vf_records)
503
504
def _install_inventory(self, revision_id, metadata, text):
505
vf = self._repository.get_inventory_weave()
506
if revision_id in vf:
507
return
508
parent_ids = metadata['parents']
509
if self._info['serializer'] == self._repository._serializer.format_num:
510
return self._install_mp_records(vf, [(revision_id, metadata,
511
text)])
512
parents = [self._repository.get_inventory(p)
513
for p in parent_ids]
514
parent_texts = [self._source_serializer.write_inventory_to_string(p)
515
for p in parents]
516
target_lines = multiparent.MultiParent.from_patch(text).to_lines(
517
parent_texts)
518
sha1 = osutils.sha_strings(target_lines)
519
if sha1 != metadata['sha1']:
520
raise errors.BadBundle("Can't convert to target format")
521
target_inv = self._source_serializer.read_inventory_from_string(
522
''.join(target_lines))
523
self._handle_root(target_inv, parent_ids)
524
try:
525
self._repository.add_inventory(revision_id, target_inv, parent_ids)
526
except errors.UnsupportedInventoryKind:
527
raise errors.IncompatibleRevision(repr(self._repository))
528
529
def _handle_root(self, target_inv, parent_ids):
530
revision_id = target_inv.revision_id
531
if self.update_root:
532
target_inv.root.revision = revision_id
533
store = self._repository.weave_store
534
transaction = self._repository.get_transaction()
535
vf = store.get_weave_or_empty(target_inv.root.file_id, transaction)
536
vf.add_lines(revision_id, parent_ids, [])
537
elif not self._repository.supports_rich_root():
538
if target_inv.root.revision != revision_id:
539
raise errors.IncompatibleRevision(repr(self._repository))
540
541
542
def _install_revision(self, revision_id, metadata, text):
543
if self._repository.has_revision(revision_id):
544
return
545
self._repository._add_revision_text(revision_id, text)
546
547
def _install_signature(self, revision_id, metadata, text):
548
transaction = self._repository.get_transaction()
549
if self._repository._revision_store.has_signature(revision_id,
550
transaction):
551
return
552
self._repository._revision_store.add_revision_signature_text(
553
revision_id, text, transaction)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1