~bzr-pqm/bzr/bzr.dev

5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
1
# Copyright (C) 2007-2010 Canonical Ltd
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
16
6379.6.3 by Jelmer Vernooij
Use absolute_import.
17
from __future__ import absolute_import
18
2520.4.20 by Aaron Bentley
Compress and base64-encode bundle contents
19
from cStringIO import StringIO
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
20
import bz2
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
21
import re
2520.4.20 by Aaron Bentley
Compress and base64-encode bundle contents
22
2520.4.13 by Aaron Bentley
Use real container implementation
23
from bzrlib import (
2520.4.34 by Aaron Bentley
Add signature support
24
    errors,
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
25
    iterablefile,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
26
    lru_cache,
2520.4.13 by Aaron Bentley
Use real container implementation
27
    multiparent,
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
28
    osutils,
2520.4.13 by Aaron Bentley
Use real container implementation
29
    pack,
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
30
    revision as _mod_revision,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
31
    serializer,
2520.4.45 by Aaron Bentley
Handle inconsistencies in last-modified-revision between vf and inventory
32
    trace,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
33
    ui,
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
34
    versionedfile as _mod_versionedfile,
2520.4.13 by Aaron Bentley
Use real container implementation
35
    )
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
36
from bzrlib.bundle import bundle_data, serializer as bundle_serializer
6164.1.1 by Jelmer Vernooij
Move file_ids_altered_by to VersionedFileRepository.
37
from bzrlib.i18n import ngettext
2694.5.1 by Alexander Belchenko
pyrex bencode (without benchmarks)
38
from bzrlib import bencode
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
39
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
40
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
41
class _MPDiffInventoryGenerator(_mod_versionedfile._MPDiffGenerator):
42
    """Generate Inventory diffs serialized inventories."""
43
44
    def __init__(self, repo, inventory_keys):
45
        super(_MPDiffInventoryGenerator, self).__init__(repo.inventories,
46
            inventory_keys)
47
        self.repo = repo
48
        self.sha1s = {}
49
50
    def iter_diffs(self):
51
        """Compute the diffs one at a time."""
52
        # This is instead of compute_diffs() since we guarantee our ordering of
53
        # inventories, we don't have to do any buffering
54
        self._find_needed_keys()
55
        # We actually use a slightly different ordering. We grab all of the
56
        # parents first, and then grab the ordered requests.
57
        needed_ids = [k[-1] for k in self.present_parents]
58
        needed_ids.extend([k[-1] for k in self.ordered_keys])
59
        inv_to_str = self.repo._serializer.write_inventory_to_string
60
        for inv in self.repo.iter_inventories(needed_ids):
61
            revision_id = inv.revision_id
62
            key = (revision_id,)
63
            if key in self.present_parents:
64
                # Not a key we will transmit, which is a shame, since because
65
                # of that bundles don't work with stacked branches
66
                parent_ids = None
67
            else:
68
                parent_ids = [k[-1] for k in self.parent_map[key]]
69
            as_bytes = inv_to_str(inv)
70
            self._process_one_record(key, (as_bytes,))
71
            if parent_ids is None:
72
                continue
73
            diff = self.diffs.pop(key)
74
            sha1 = osutils.sha_string(as_bytes)
75
            yield revision_id, parent_ids, sha1, diff
76
77
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
78
class BundleWriter(object):
2520.4.118 by Aaron Bentley
Add docs
79
    """Writer for bundle-format files.
80
81
    This serves roughly the same purpose as ContainerReader, but acts as a
82
    layer on top of it.
83
2520.4.123 by Aaron Bentley
Cleanup of bundle code
84
    Provides ways of writing the specific record types supported this bundle
2520.4.118 by Aaron Bentley
Add docs
85
    format.
86
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
87
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
88
    def __init__(self, fileobj):
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
89
        self._container = pack.ContainerWriter(self._write_encoded)
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
90
        self._fileobj = fileobj
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
91
        self._compressor = bz2.BZ2Compressor()
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
92
2520.4.118 by Aaron Bentley
Add docs
93
    def _write_encoded(self, bytes):
94
        """Write bzip2-encoded bytes to the file"""
95
        self._fileobj.write(self._compressor.compress(bytes))
96
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
97
    def begin(self):
2520.4.118 by Aaron Bentley
Add docs
98
        """Start writing the bundle"""
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
99
        self._fileobj.write(bundle_serializer._get_bundle_header(
100
            bundle_serializer.v4_string))
2520.4.24 by Aaron Bentley
Move heading writing above container beginning
101
        self._fileobj.write('#\n')
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
102
        self._container.begin()
103
104
    def end(self):
2520.4.118 by Aaron Bentley
Add docs
105
        """Finish writing the bundle"""
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
106
        self._container.end()
2520.4.76 by Aaron Bentley
Move base64-encoding into merge directives
107
        self._fileobj.write(self._compressor.flush())
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
108
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
109
    def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
110
                               revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
111
        """Add a record for a multi-parent diff
112
113
        :mp_bytes: A multi-parent diff, as a bytestring
2520.4.123 by Aaron Bentley
Cleanup of bundle code
114
        :sha1: The sha1 hash of the fulltext
2520.4.118 by Aaron Bentley
Add docs
115
        :parents: a list of revision-ids of the parents
116
        :repo_kind: The kind of object in the repository.  May be 'file' or
117
            'inventory'
118
        :revision_id: The revision id of the mpdiff being added.
119
        :file_id: The file-id of the file, or None for inventories.
120
        """
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
121
        metadata = {'parents': parents,
122
                    'storage_kind': 'mpdiff',
123
                    'sha1': sha1}
124
        self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
125
2520.4.123 by Aaron Bentley
Cleanup of bundle code
126
    def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
2520.4.118 by Aaron Bentley
Add docs
127
        """Add a record for a fulltext
128
129
        :bytes: The fulltext, as a bytestring
130
        :parents: a list of revision-ids of the parents
131
        :repo_kind: The kind of object in the repository.  May be 'revision' or
132
            'signature'
133
        :revision_id: The revision id of the fulltext being added.
134
        """
135
        metadata = {'parents': parents,
2520.5.3 by Aaron Bentley
fix sha1 in bundle format 4
136
                    'storage_kind': 'mpdiff'}
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
137
        self._add_record(bytes, {'parents': parents,
2520.4.123 by Aaron Bentley
Cleanup of bundle code
138
            'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
139
2520.4.95 by Aaron Bentley
Add support for header/info records
140
    def add_info_record(self, **kwargs):
2520.4.118 by Aaron Bentley
Add docs
141
        """Add an info record to the bundle
142
143
        Any parameters may be supplied, except 'self' and 'storage_kind'.
144
        Values must be lists, strings, integers, dicts, or a combination.
145
        """
2520.4.95 by Aaron Bentley
Add support for header/info records
146
        kwargs['storage_kind'] = 'header'
147
        self._add_record(None, kwargs, 'info', None, None)
148
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
149
    @staticmethod
2520.4.68 by Aaron Bentley
Change name separators to all-slash
150
    def encode_name(content_kind, revision_id, file_id=None):
2520.4.118 by Aaron Bentley
Add docs
151
        """Encode semantic ids as a container name"""
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
152
        if content_kind not in ('revision', 'file', 'inventory', 'signature',
153
                'info'):
154
            raise ValueError(content_kind)
2520.4.118 by Aaron Bentley
Add docs
155
        if content_kind == 'file':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
156
            if file_id is None:
157
                raise AssertionError()
2520.4.118 by Aaron Bentley
Add docs
158
        else:
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
159
            if file_id is not None:
160
                raise AssertionError()
2520.4.95 by Aaron Bentley
Add support for header/info records
161
        if content_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
162
            if revision_id is not None:
163
                raise AssertionError()
164
        elif revision_id is None:
165
            raise AssertionError()
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
166
        names = [n.replace('/', '//') for n in
167
                 (content_kind, revision_id, file_id) if n is not None]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
168
        return '/'.join(names)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
169
2520.4.56 by Aaron Bentley
Begin adding support for arbitrary metadata
170
    def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
171
        """Add a bundle record to the container.
172
173
        Most bundle records are recorded as header/body pairs, with the
174
        body being nameless.  Records with storage_kind 'header' have no
175
        body.
176
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
177
        name = self.encode_name(repo_kind, revision_id, file_id)
2520.4.95 by Aaron Bentley
Add support for header/info records
178
        encoded_metadata = bencode.bencode(metadata)
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
179
        self._container.add_bytes_record(encoded_metadata, [(name, )])
2520.4.95 by Aaron Bentley
Add support for header/info records
180
        if metadata['storage_kind'] != 'header':
181
            self._container.add_bytes_record(bytes, [])
2520.4.13 by Aaron Bentley
Use real container implementation
182
2520.4.7 by Aaron Bentley
Fix patch deserialization
183
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
184
class BundleReader(object):
2520.4.118 by Aaron Bentley
Add docs
185
    """Reader for bundle-format files.
186
187
    This serves roughly the same purpose as ContainerReader, but acts as a
188
    layer on top of it, providing metadata, a semantic name, and a record
189
    body
190
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
191
4543.2.14 by John Arbash Meinel
Clarify some comments, fix up a debugging change.
192
    def __init__(self, fileobj, stream_input=True):
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
193
        """Constructor
194
195
        :param fileobj: a file containing a bzip-encoded container
2520.4.148 by Aaron Bentley
Updates from review
196
        :param stream_input: If True, the BundleReader stream input rather than
197
            reading it all into memory at once.  Reading it into memory all at
198
            once is (currently) faster.
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
199
        """
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
200
        line = fileobj.readline()
201
        if line != '\n':
202
            fileobj.readline()
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
203
        self.patch_lines = []
2520.4.148 by Aaron Bentley
Updates from review
204
        if stream_input:
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
205
            source_file = iterablefile.IterableFile(self.iter_decode(fileobj))
206
        else:
207
            source_file = StringIO(bz2.decompress(fileobj.read()))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
208
        self._container_file = source_file
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
209
210
    @staticmethod
211
    def iter_decode(fileobj):
2520.4.118 by Aaron Bentley
Add docs
212
        """Iterate through decoded fragments of the file"""
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
213
        decompressor = bz2.BZ2Decompressor()
214
        for line in fileobj:
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
215
            try:
216
                yield decompressor.decompress(line)
217
            except EOFError:
218
                return
2520.4.22 by Aaron Bentley
Create ContainerReader
219
220
    @staticmethod
221
    def decode_name(name):
2520.4.118 by Aaron Bentley
Add docs
222
        """Decode a name from its container form into a semantic form
223
224
        :retval: content_kind, revision_id, file_id
225
        """
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
226
        segments = re.split('(//?)', name)
227
        names = ['']
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
228
        for segment in segments:
229
            if segment == '//':
230
                names[-1] += '/'
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
231
            elif segment == '/':
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
232
                names.append('')
233
            else:
234
                names[-1] += segment
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
235
        content_kind = names[0]
2520.4.95 by Aaron Bentley
Add support for header/info records
236
        revision_id = None
237
        file_id = None
238
        if len(names) > 1:
239
            revision_id = names[1]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
240
        if len(names) > 2:
241
            file_id = names[2]
242
        return content_kind, revision_id, file_id
2520.4.22 by Aaron Bentley
Create ContainerReader
243
244
    def iter_records(self):
2520.4.118 by Aaron Bentley
Add docs
245
        """Iterate through bundle records
246
247
        :return: a generator of (bytes, metadata, content_kind, revision_id,
248
            file_id)
249
        """
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
250
        iterator = pack.iter_records_from_file(self._container_file)
251
        for names, bytes in iterator:
2520.4.131 by Aaron Bentley
Raise BadBundle for records with wrong number of names
252
            if len(names) != 1:
253
                raise errors.BadBundle('Record has %d names instead of 1'
254
                                       % len(names))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
255
            metadata = bencode.bdecode(bytes)
2520.4.95 by Aaron Bentley
Add support for header/info records
256
            if metadata['storage_kind'] == 'header':
257
                bytes = None
258
            else:
259
                _unused, bytes = iterator.next()
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
260
            yield (bytes, metadata) + self.decode_name(names[0][0])
2520.4.22 by Aaron Bentley
Create ContainerReader
261
262
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
263
class BundleSerializerV4(bundle_serializer.BundleSerializer):
2520.4.118 by Aaron Bentley
Add docs
264
    """Implement the high-level bundle interface"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
265
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
266
    def write(self, repository, revision_ids, forced_bases, fileobj):
2520.4.118 by Aaron Bentley
Add docs
267
        """Write a bundle to a file-like object
268
269
        For backwards-compatibility only
270
        """
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
271
        write_op = BundleWriteOperation.from_old_args(repository, revision_ids,
272
                                                      forced_bases, fileobj)
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
273
        return write_op.do_write()
274
275
    def write_bundle(self, repository, target, base, fileobj):
2520.4.118 by Aaron Bentley
Add docs
276
        """Write a bundle to a file object
277
278
        :param repository: The repository to retrieve revision data from
279
        :param target: The head revision to include ancestors of
280
        :param base: The ancestor of the target to stop including acestors
281
            at.
282
        :param fileobj: The file-like object to write to
283
        """
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
284
        write_op =  BundleWriteOperation(base, target, repository, fileobj)
285
        return write_op.do_write()
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
286
287
    def read(self, file):
2520.4.118 by Aaron Bentley
Add docs
288
        """return a reader object for a given file"""
2520.4.72 by Aaron Bentley
Rename format to 4alpha
289
        bundle = BundleInfoV4(file, self)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
290
        return bundle
291
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
292
    @staticmethod
293
    def get_source_serializer(info):
2520.4.118 by Aaron Bentley
Add docs
294
        """Retrieve the serializer for a given info object"""
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
295
        return serializer.format_registry.get(info['serializer'])
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
296
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
297
298
class BundleWriteOperation(object):
2520.4.118 by Aaron Bentley
Add docs
299
    """Perform the operation of writing revisions to a bundle"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
300
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
301
    @classmethod
302
    def from_old_args(cls, repository, revision_ids, forced_bases, fileobj):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
303
        """Create a BundleWriteOperation from old-style arguments"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
304
        base, target = cls.get_base_target(revision_ids, forced_bases,
305
                                           repository)
306
        return BundleWriteOperation(base, target, repository, fileobj,
307
                                    revision_ids)
308
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
309
    def __init__(self, base, target, repository, fileobj, revision_ids=None):
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
310
        self.base = base
311
        self.target = target
312
        self.repository = repository
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
313
        bundle = BundleWriter(fileobj)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
314
        self.bundle = bundle
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
315
        if revision_ids is not None:
316
            self.revision_ids = revision_ids
317
        else:
4154.1.1 by Ian Clatworthy
make send use graph.find_difference() instead of walking all of history twice
318
            graph = repository.get_graph()
4154.1.3 by Ian Clatworthy
strip ghosts so test_bundle_with_ghosts works again
319
            revision_ids = graph.find_unique_ancestors(target, [base])
320
            # Strip ghosts
321
            parents = graph.get_parent_map(revision_ids)
322
            self.revision_ids = [r for r in revision_ids if r in parents]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
323
        self.revision_keys = set([(revid,) for revid in self.revision_ids])
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
324
325
    def do_write(self):
2520.4.118 by Aaron Bentley
Add docs
326
        """Write all data to the bundle"""
6143.1.3 by Jonathan Riddell
more plurals
327
        trace.note(ngettext('Bundling %d revision.', 'Bundling %d revisions.',
328
                            len(self.revision_ids)), len(self.revision_ids))
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
329
        self.repository.lock_read()
330
        try:
331
            self.bundle.begin()
332
            self.write_info()
333
            self.write_files()
334
            self.write_revisions()
335
            self.bundle.end()
336
        finally:
337
            self.repository.unlock()
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
338
        return self.revision_ids
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
339
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
340
    def write_info(self):
2520.4.118 by Aaron Bentley
Add docs
341
        """Write format info"""
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
342
        serializer_format = self.repository.get_serializer_format()
2520.4.99 by Aaron Bentley
Test conversion across models
343
        supports_rich_root = {True: 1, False: 0}[
344
            self.repository.supports_rich_root()]
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
345
        self.bundle.add_info_record(serializer=serializer_format,
2520.4.99 by Aaron Bentley
Test conversion across models
346
                                    supports_rich_root=supports_rich_root)
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
347
2520.4.51 by Aaron Bentley
Split iteration through file revisions into a method, so we can vary it
348
    def write_files(self):
2520.4.118 by Aaron Bentley
Add docs
349
        """Write bundle records for all revisions of all files"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
350
        text_keys = []
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
351
        altered_fileids = self.repository.fileids_altered_by_revision_ids(
352
                self.revision_ids)
353
        for file_id, revision_ids in altered_fileids.iteritems():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
354
            for revision_id in revision_ids:
355
                text_keys.append((file_id, revision_id))
3350.6.10 by Martin Pool
VersionedFiles review cleanups
356
        self._add_mp_records_keys('file', self.repository.texts, text_keys)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
357
358
    def write_revisions(self):
2520.4.118 by Aaron Bentley
Add docs
359
        """Write bundle records for all revisions and signatures"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
360
        inv_vf = self.repository.inventories
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
361
        topological_order = [key[-1] for key in multiparent.topo_iter_keys(
362
                                inv_vf, self.revision_keys)]
363
        revision_order = topological_order
2520.4.75 by Aaron Bentley
Fix traceback on empty bundles.
364
        if self.target is not None and self.target in self.revision_ids:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
365
            # Make sure the target revision is always the last entry
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
366
            revision_order = list(topological_order)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
367
            revision_order.remove(self.target)
368
            revision_order.append(self.target)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
369
        if self.repository._serializer.support_altered_by_hack:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
370
            # Repositories that support_altered_by_hack means that
371
            # inventories.make_mpdiffs() contains all the data about the tree
372
            # shape. Formats without support_altered_by_hack require
373
            # chk_bytes/etc, so we use a different code path.
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
374
            self._add_mp_records_keys('inventory', inv_vf,
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
375
                                      [(revid,) for revid in topological_order])
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
376
        else:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
377
            # Inventories should always be added in pure-topological order, so
378
            # that we can apply the mpdiff for the child to the parent texts.
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
379
            self._add_inventory_mpdiffs_from_serializer(topological_order)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
380
        self._add_revision_texts(revision_order)
381
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
382
    def _add_inventory_mpdiffs_from_serializer(self, revision_order):
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
383
        """Generate mpdiffs by serializing inventories.
384
385
        The current repository only has part of the tree shape information in
386
        the 'inventories' vf. So we use serializer.write_inventory_to_string to
387
        get a 'full' representation of the tree shape, and then generate
388
        mpdiffs on that data stream. This stream can then be reconstructed on
389
        the other side.
390
        """
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
391
        inventory_key_order = [(r,) for r in revision_order]
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
392
        generator = _MPDiffInventoryGenerator(self.repository,
393
                                              inventory_key_order)
394
        for revision_id, parent_ids, sha1, diff in generator.iter_diffs():
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
395
            text = ''.join(diff.to_patch())
396
            self.bundle.add_multiparent_record(text, sha1, parent_ids,
397
                                               'inventory', revision_id, None)
398
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
399
    def _add_revision_texts(self, revision_order):
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
400
        parent_map = self.repository.get_parent_map(revision_order)
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
401
        revision_to_str = self.repository._serializer.write_revision_to_string
402
        revisions = self.repository.get_revisions(revision_order)
403
        for revision in revisions:
404
            revision_id = revision.revision_id
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
405
            parents = parent_map.get(revision_id, None)
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
406
            revision_text = revision_to_str(revision)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
407
            self.bundle.add_fulltext_record(revision_text, parents,
2520.4.123 by Aaron Bentley
Cleanup of bundle code
408
                                       'revision', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
409
            try:
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
410
                self.bundle.add_fulltext_record(
411
                    self.repository.get_signature_text(
2520.4.123 by Aaron Bentley
Cleanup of bundle code
412
                    revision_id), parents, 'signature', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
413
            except errors.NoSuchRevision:
414
                pass
415
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
416
    @staticmethod
417
    def get_base_target(revision_ids, forced_bases, repository):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
418
        """Determine the base and target from old-style revision ids"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
419
        if len(revision_ids) == 0:
420
            return None, None
421
        target = revision_ids[0]
422
        base = forced_bases.get(target)
423
        if base is None:
424
            parents = repository.get_revision(target).parent_ids
425
            if len(parents) == 0:
426
                base = _mod_revision.NULL_REVISION
427
            else:
428
                base = parents[0]
429
        return base, target
430
3350.6.10 by Martin Pool
VersionedFiles review cleanups
431
    def _add_mp_records_keys(self, repo_kind, vf, keys):
2520.4.118 by Aaron Bentley
Add docs
432
        """Add multi-parent diff records to a bundle"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
433
        ordered_keys = list(multiparent.topo_iter_keys(vf, keys))
434
        mpdiffs = vf.make_mpdiffs(ordered_keys)
435
        sha1s = vf.get_sha1s(ordered_keys)
436
        parent_map = vf.get_parent_map(ordered_keys)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
437
        for mpdiff, item_key, in zip(mpdiffs, ordered_keys):
438
            sha1 = sha1s[item_key]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
439
            parents = [key[-1] for key in parent_map[item_key]]
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
440
            text = ''.join(mpdiff.to_patch())
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
441
            # Infer file id records as appropriate.
442
            if len(item_key) == 2:
443
                file_id = item_key[0]
444
            else:
445
                file_id = None
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
446
            self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
447
                                               item_key[-1], file_id)
2520.4.6 by Aaron Bentley
Get installation started
448
449
2520.4.72 by Aaron Bentley
Rename format to 4alpha
450
class BundleInfoV4(object):
2520.4.6 by Aaron Bentley
Get installation started
451
2520.4.118 by Aaron Bentley
Add docs
452
    """Provide (most of) the BundleInfo interface"""
2520.4.6 by Aaron Bentley
Get installation started
453
    def __init__(self, fileobj, serializer):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
454
        self._fileobj = fileobj
455
        self._serializer = serializer
456
        self.__real_revisions = None
457
        self.__revisions = None
458
459
    def install(self, repository):
460
        return self.install_revisions(repository)
461
2520.4.148 by Aaron Bentley
Updates from review
462
    def install_revisions(self, repository, stream_input=True):
463
        """Install this bundle's revisions into the specified repository
464
465
        :param target_repo: The repository to install into
466
        :param stream_input: If True, will stream input rather than reading it
467
            all into memory at once.  Reading it into memory all at once is
468
            (currently) faster.
469
        """
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
470
        repository.lock_write()
471
        try:
2520.4.148 by Aaron Bentley
Updates from review
472
            ri = RevisionInstaller(self.get_bundle_reader(stream_input),
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
473
                                   self._serializer, repository)
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
474
            return ri.install()
475
        finally:
476
            repository.unlock()
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
477
2520.4.109 by Aaron Bentley
start work on directive cherry-picking
478
    def get_merge_request(self, target_repo):
479
        """Provide data for performing a merge
480
481
        Returns suggested base, suggested target, and patch verification status
482
        """
483
        return None, self.target, 'inapplicable'
484
2520.4.148 by Aaron Bentley
Updates from review
485
    def get_bundle_reader(self, stream_input=True):
486
        """Return a new BundleReader for the associated bundle
487
488
        :param stream_input: If True, the BundleReader stream input rather than
489
            reading it all into memory at once.  Reading it into memory all at
490
            once is (currently) faster.
491
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
492
        self._fileobj.seek(0)
2520.4.148 by Aaron Bentley
Updates from review
493
        return BundleReader(self._fileobj, stream_input)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
494
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
495
    def _get_real_revisions(self):
496
        if self.__real_revisions is None:
497
            self.__real_revisions = []
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
498
            bundle_reader = self.get_bundle_reader()
2520.4.102 by Aaron Bentley
rename parents to metadata
499
            for bytes, metadata, repo_kind, revision_id, file_id in \
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
500
                bundle_reader.iter_records():
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
501
                if repo_kind == 'info':
502
                    serializer =\
2520.4.102 by Aaron Bentley
rename parents to metadata
503
                        self._serializer.get_source_serializer(metadata)
2520.4.22 by Aaron Bentley
Create ContainerReader
504
                if repo_kind == 'revision':
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
505
                    rev = serializer.read_revision_from_string(bytes)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
506
                    self.__real_revisions.append(rev)
507
        return self.__real_revisions
508
    real_revisions = property(_get_real_revisions)
509
510
    def _get_revisions(self):
511
        if self.__revisions is None:
512
            self.__revisions = []
513
            for revision in self.real_revisions:
2520.4.33 by Aaron Bentley
remove test dependencies on serialization minutia
514
                self.__revisions.append(
515
                    bundle_data.RevisionInfo.from_revision(revision))
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
516
        return self.__revisions
517
518
    revisions = property(_get_revisions)
519
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
520
    def _get_target(self):
521
        return self.revisions[-1].revision_id
522
523
    target = property(_get_target)
524
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
525
526
class RevisionInstaller(object):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
527
    """Installs revisions into a repository"""
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
528
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
529
    def __init__(self, container, serializer, repository):
530
        self._container = container
2520.4.6 by Aaron Bentley
Get installation started
531
        self._serializer = serializer
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
532
        self._repository = repository
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
533
        self._info = None
2520.4.99 by Aaron Bentley
Test conversion across models
534
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
535
    def install(self):
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
536
        """Perform the installation.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
537
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
538
        Must be called with the Repository locked.
539
        """
540
        self._repository.start_write_group()
541
        try:
2856.1.2 by Robert Collins
Review feedback.
542
            result = self._install_in_write_group()
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
543
        except:
544
            self._repository.abort_write_group()
545
            raise
546
        self._repository.commit_write_group()
547
        return result
548
2856.1.2 by Robert Collins
Review feedback.
549
    def _install_in_write_group(self):
2520.4.6 by Aaron Bentley
Get installation started
550
        current_file = None
551
        current_versionedfile = None
552
        pending_file_records = []
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
553
        inventory_vf = None
554
        pending_inventory_records = []
2520.4.8 by Aaron Bentley
Serialize inventory
555
        added_inv = set()
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
556
        target_revision = None
2520.4.58 by Aaron Bentley
Propogate support for metadata to iter_revisions, add storage kind
557
        for bytes, metadata, repo_kind, revision_id, file_id in\
2520.4.22 by Aaron Bentley
Create ContainerReader
558
            self._container.iter_records():
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
559
            if repo_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
560
                if self._info is not None:
561
                    raise AssertionError()
2520.4.123 by Aaron Bentley
Cleanup of bundle code
562
                self._handle_info(metadata)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
563
            if (pending_file_records and
564
                (repo_kind, file_id) != ('file', current_file)):
565
                # Flush the data for a single file - prevents memory
566
                # spiking due to buffering all files in memory.
567
                self._install_mp_records_keys(self._repository.texts,
568
                    pending_file_records)
2520.4.8 by Aaron Bentley
Serialize inventory
569
                current_file = None
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
570
                del pending_file_records[:]
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
571
            if len(pending_inventory_records) > 0 and repo_kind != 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
572
                self._install_inventory_records(pending_inventory_records)
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
573
                pending_inventory_records = []
574
            if repo_kind == 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
575
                pending_inventory_records.append(((revision_id,), metadata, bytes))
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
576
            if repo_kind == 'revision':
577
                target_revision = revision_id
578
                self._install_revision(revision_id, metadata, bytes)
579
            if repo_kind == 'signature':
580
                self._install_signature(revision_id, metadata, bytes)
2520.4.22 by Aaron Bentley
Create ContainerReader
581
            if repo_kind == 'file':
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
582
                current_file = file_id
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
583
                pending_file_records.append(((file_id, revision_id), metadata, bytes))
584
        self._install_mp_records_keys(self._repository.texts, pending_file_records)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
585
        return target_revision
2520.4.6 by Aaron Bentley
Get installation started
586
2520.4.123 by Aaron Bentley
Cleanup of bundle code
587
    def _handle_info(self, info):
588
        """Extract data from an info record"""
589
        self._info = info
590
        self._source_serializer = self._serializer.get_source_serializer(info)
591
        if (info['supports_rich_root'] == 0 and
592
            self._repository.supports_rich_root()):
593
            self.update_root = True
594
        else:
595
            self.update_root = False
596
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
597
    def _install_mp_records(self, versionedfile, records):
2520.4.61 by Aaron Bentley
Do bulk insertion of records
598
        if len(records) == 0:
599
            return
600
        d_func = multiparent.MultiParent.from_patch
601
        vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
602
                      records if r not in versionedfile]
603
        versionedfile.add_mpdiffs(vf_records)
2520.4.8 by Aaron Bentley
Serialize inventory
604
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
605
    def _install_mp_records_keys(self, versionedfile, records):
606
        d_func = multiparent.MultiParent.from_patch
607
        vf_records = []
608
        for key, meta, text in records:
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
609
            # Adapt to tuple interface: A length two key is a file_id,
610
            # revision_id pair, a length 1 key is a
611
            # revision/signature/inventory. We need to do this because
612
            # the metadata extraction from the bundle has not yet been updated
613
            # to use the consistent tuple interface itself.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
614
            if len(key) == 2:
615
                prefix = key[:1]
616
            else:
617
                prefix = ()
618
            parents = [prefix + (parent,) for parent in meta['parents']]
619
            vf_records.append((key, parents, meta['sha1'], d_func(text)))
620
        versionedfile.add_mpdiffs(vf_records)
621
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
622
    def _get_parent_inventory_texts(self, inventory_text_cache,
623
                                    inventory_cache, parent_ids):
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
624
        cached_parent_texts = {}
625
        remaining_parent_ids = []
626
        for parent_id in parent_ids:
627
            p_text = inventory_text_cache.get(parent_id, None)
628
            if p_text is None:
629
                remaining_parent_ids.append(parent_id)
630
            else:
631
                cached_parent_texts[parent_id] = p_text
632
        ghosts = ()
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
633
        # TODO: Use inventory_cache to grab inventories we already have in
634
        #       memory
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
635
        if remaining_parent_ids:
636
            # first determine what keys are actually present in the local
637
            # inventories object (don't use revisions as they haven't been
638
            # installed yet.)
639
            parent_keys = [(r,) for r in remaining_parent_ids]
640
            present_parent_map = self._repository.inventories.get_parent_map(
641
                                        parent_keys)
642
            present_parent_ids = []
643
            ghosts = set()
644
            for p_id in remaining_parent_ids:
645
                if (p_id,) in present_parent_map:
646
                    present_parent_ids.append(p_id)
647
                else:
648
                    ghosts.add(p_id)
649
            to_string = self._source_serializer.write_inventory_to_string
650
            for parent_inv in self._repository.iter_inventories(
651
                                    present_parent_ids):
652
                p_text = to_string(parent_inv)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
653
                inventory_cache[parent_inv.revision_id] = parent_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
654
                cached_parent_texts[parent_inv.revision_id] = p_text
655
                inventory_text_cache[parent_inv.revision_id] = p_text
656
657
        parent_texts = [cached_parent_texts[parent_id]
658
                        for parent_id in parent_ids
659
                         if parent_id not in ghosts]
660
        return parent_texts
661
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
662
    def _install_inventory_records(self, records):
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
663
        if (self._info['serializer'] == self._repository._serializer.format_num
664
            and self._repository._serializer.support_altered_by_hack):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
665
            return self._install_mp_records_keys(self._repository.inventories,
666
                records)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
667
        # Use a 10MB text cache, since these are string xml inventories. Note
668
        # that 10MB is fairly small for large projects (a single inventory can
669
        # be >5MB). Another possibility is to cache 10-20 inventory texts
670
        # instead
671
        inventory_text_cache = lru_cache.LRUSizeCache(10*1024*1024)
4543.2.21 by John Arbash Meinel
A few more tiny tweaks to comments, etc.
672
        # Also cache the in-memory representation. This allows us to create
673
        # inventory deltas to apply rather than calling add_inventory from
674
        # scratch each time.
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
675
        inventory_cache = lru_cache.LRUCache(10)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
676
        pb = ui.ui_factory.nested_progress_bar()
677
        try:
678
            num_records = len(records)
679
            for idx, (key, metadata, bytes) in enumerate(records):
680
                pb.update('installing inventory', idx, num_records)
681
                revision_id = key[-1]
682
                parent_ids = metadata['parents']
683
                # Note: This assumes the local ghosts are identical to the
684
                #       ghosts in the source, as the Bundle serialization
685
                #       format doesn't record ghosts.
686
                p_texts = self._get_parent_inventory_texts(inventory_text_cache,
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
687
                                                           inventory_cache,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
688
                                                           parent_ids)
689
                # Why does to_lines() take strings as the source, it seems that
690
                # it would have to cast to a list of lines, which we get back
691
                # as lines and then cast back to a string.
692
                target_lines = multiparent.MultiParent.from_patch(bytes
693
                            ).to_lines(p_texts)
694
                inv_text = ''.join(target_lines)
695
                del target_lines
696
                sha1 = osutils.sha_string(inv_text)
697
                if sha1 != metadata['sha1']:
698
                    raise errors.BadBundle("Can't convert to target format")
699
                # Add this to the cache so we don't have to extract it again.
700
                inventory_text_cache[revision_id] = inv_text
701
                target_inv = self._source_serializer.read_inventory_from_string(
702
                    inv_text)
703
                self._handle_root(target_inv, parent_ids)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
704
                parent_inv = None
705
                if parent_ids:
706
                    parent_inv = inventory_cache.get(parent_ids[0], None)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
707
                try:
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
708
                    if parent_inv is None:
709
                        self._repository.add_inventory(revision_id, target_inv,
710
                                                       parent_ids)
711
                    else:
712
                        delta = target_inv._make_delta(parent_inv)
713
                        self._repository.add_inventory_by_delta(parent_ids[0],
714
                            delta, revision_id, parent_ids)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
715
                except errors.UnsupportedInventoryKind:
716
                    raise errors.IncompatibleRevision(repr(self._repository))
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
717
                inventory_cache[revision_id] = target_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
718
        finally:
719
            pb.finished()
2520.4.99 by Aaron Bentley
Test conversion across models
720
721
    def _handle_root(self, target_inv, parent_ids):
722
        revision_id = target_inv.revision_id
723
        if self.update_root:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
724
            text_key = (target_inv.root.file_id, revision_id)
725
            parent_keys = [(target_inv.root.file_id, parent) for
726
                parent in parent_ids]
727
            self._repository.texts.add_lines(text_key, parent_keys, [])
2520.4.99 by Aaron Bentley
Test conversion across models
728
        elif not self._repository.supports_rich_root():
729
            if target_inv.root.revision != revision_id:
730
                raise errors.IncompatibleRevision(repr(self._repository))
731
2520.4.59 by Aaron Bentley
Push metadata down the stack
732
    def _install_revision(self, revision_id, metadata, text):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
733
        if self._repository.has_revision(revision_id):
734
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
735
        revision = self._source_serializer.read_revision_from_string(text)
736
        self._repository.add_revision(revision.revision_id, revision)
2520.4.34 by Aaron Bentley
Add signature support
737
2520.4.59 by Aaron Bentley
Push metadata down the stack
738
    def _install_signature(self, revision_id, metadata, text):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
739
        transaction = self._repository.get_transaction()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
740
        if self._repository.has_signature_for_revision_id(revision_id):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
741
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
742
        self._repository.add_signature_text(revision_id, text)