~bzr-pqm/bzr/bzr.dev

2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
1
# Copyright (C) 2007 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
16
2520.4.20 by Aaron Bentley
Compress and base64-encode bundle contents
17
from cStringIO import StringIO
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
18
import bz2
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
19
import re
2520.4.20 by Aaron Bentley
Compress and base64-encode bundle contents
20
2520.4.13 by Aaron Bentley
Use real container implementation
21
from bzrlib import (
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
22
    diff,
2520.4.34 by Aaron Bentley
Add signature support
23
    errors,
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
24
    iterablefile,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
25
    lru_cache,
2520.4.13 by Aaron Bentley
Use real container implementation
26
    multiparent,
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
27
    osutils,
2520.4.13 by Aaron Bentley
Use real container implementation
28
    pack,
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
29
    revision as _mod_revision,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
30
    serializer,
2520.4.45 by Aaron Bentley
Handle inconsistencies in last-modified-revision between vf and inventory
31
    trace,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
32
    ui,
2520.4.13 by Aaron Bentley
Use real container implementation
33
    )
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
34
from bzrlib.bundle import bundle_data, serializer as bundle_serializer
2694.5.1 by Alexander Belchenko
pyrex bencode (without benchmarks)
35
from bzrlib import bencode
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
36
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
37
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
38
class BundleWriter(object):
2520.4.118 by Aaron Bentley
Add docs
39
    """Writer for bundle-format files.
40
41
    This serves roughly the same purpose as ContainerReader, but acts as a
42
    layer on top of it.
43
2520.4.123 by Aaron Bentley
Cleanup of bundle code
44
    Provides ways of writing the specific record types supported this bundle
2520.4.118 by Aaron Bentley
Add docs
45
    format.
46
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
47
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
48
    def __init__(self, fileobj):
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
49
        self._container = pack.ContainerWriter(self._write_encoded)
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
50
        self._fileobj = fileobj
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
51
        self._compressor = bz2.BZ2Compressor()
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
52
2520.4.118 by Aaron Bentley
Add docs
53
    def _write_encoded(self, bytes):
54
        """Write bzip2-encoded bytes to the file"""
55
        self._fileobj.write(self._compressor.compress(bytes))
56
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
57
    def begin(self):
2520.4.118 by Aaron Bentley
Add docs
58
        """Start writing the bundle"""
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
59
        self._fileobj.write(bundle_serializer._get_bundle_header(
60
            bundle_serializer.v4_string))
2520.4.24 by Aaron Bentley
Move heading writing above container beginning
61
        self._fileobj.write('#\n')
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
62
        self._container.begin()
63
64
    def end(self):
2520.4.118 by Aaron Bentley
Add docs
65
        """Finish writing the bundle"""
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
66
        self._container.end()
2520.4.76 by Aaron Bentley
Move base64-encoding into merge directives
67
        self._fileobj.write(self._compressor.flush())
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
68
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
69
    def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
70
                               revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
71
        """Add a record for a multi-parent diff
72
73
        :mp_bytes: A multi-parent diff, as a bytestring
2520.4.123 by Aaron Bentley
Cleanup of bundle code
74
        :sha1: The sha1 hash of the fulltext
2520.4.118 by Aaron Bentley
Add docs
75
        :parents: a list of revision-ids of the parents
76
        :repo_kind: The kind of object in the repository.  May be 'file' or
77
            'inventory'
78
        :revision_id: The revision id of the mpdiff being added.
79
        :file_id: The file-id of the file, or None for inventories.
80
        """
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
81
        metadata = {'parents': parents,
82
                    'storage_kind': 'mpdiff',
83
                    'sha1': sha1}
84
        self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
85
2520.4.123 by Aaron Bentley
Cleanup of bundle code
86
    def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
2520.4.118 by Aaron Bentley
Add docs
87
        """Add a record for a fulltext
88
89
        :bytes: The fulltext, as a bytestring
90
        :parents: a list of revision-ids of the parents
91
        :repo_kind: The kind of object in the repository.  May be 'revision' or
92
            'signature'
93
        :revision_id: The revision id of the fulltext being added.
94
        """
95
        metadata = {'parents': parents,
2520.5.3 by Aaron Bentley
fix sha1 in bundle format 4
96
                    'storage_kind': 'mpdiff'}
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
97
        self._add_record(bytes, {'parents': parents,
2520.4.123 by Aaron Bentley
Cleanup of bundle code
98
            'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
99
2520.4.95 by Aaron Bentley
Add support for header/info records
100
    def add_info_record(self, **kwargs):
2520.4.118 by Aaron Bentley
Add docs
101
        """Add an info record to the bundle
102
103
        Any parameters may be supplied, except 'self' and 'storage_kind'.
104
        Values must be lists, strings, integers, dicts, or a combination.
105
        """
2520.4.95 by Aaron Bentley
Add support for header/info records
106
        kwargs['storage_kind'] = 'header'
107
        self._add_record(None, kwargs, 'info', None, None)
108
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
109
    @staticmethod
2520.4.68 by Aaron Bentley
Change name separators to all-slash
110
    def encode_name(content_kind, revision_id, file_id=None):
2520.4.118 by Aaron Bentley
Add docs
111
        """Encode semantic ids as a container name"""
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
112
        if content_kind not in ('revision', 'file', 'inventory', 'signature',
113
                'info'):
114
            raise ValueError(content_kind)
2520.4.118 by Aaron Bentley
Add docs
115
        if content_kind == 'file':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
116
            if file_id is None:
117
                raise AssertionError()
2520.4.118 by Aaron Bentley
Add docs
118
        else:
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
119
            if file_id is not None:
120
                raise AssertionError()
2520.4.95 by Aaron Bentley
Add support for header/info records
121
        if content_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
122
            if revision_id is not None:
123
                raise AssertionError()
124
        elif revision_id is None:
125
            raise AssertionError()
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
126
        names = [n.replace('/', '//') for n in
127
                 (content_kind, revision_id, file_id) if n is not None]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
128
        return '/'.join(names)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
129
2520.4.56 by Aaron Bentley
Begin adding support for arbitrary metadata
130
    def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
131
        """Add a bundle record to the container.
132
133
        Most bundle records are recorded as header/body pairs, with the
134
        body being nameless.  Records with storage_kind 'header' have no
135
        body.
136
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
137
        name = self.encode_name(repo_kind, revision_id, file_id)
2520.4.95 by Aaron Bentley
Add support for header/info records
138
        encoded_metadata = bencode.bencode(metadata)
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
139
        self._container.add_bytes_record(encoded_metadata, [(name, )])
2520.4.95 by Aaron Bentley
Add support for header/info records
140
        if metadata['storage_kind'] != 'header':
141
            self._container.add_bytes_record(bytes, [])
2520.4.13 by Aaron Bentley
Use real container implementation
142
2520.4.7 by Aaron Bentley
Fix patch deserialization
143
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
144
class BundleReader(object):
2520.4.118 by Aaron Bentley
Add docs
145
    """Reader for bundle-format files.
146
147
    This serves roughly the same purpose as ContainerReader, but acts as a
148
    layer on top of it, providing metadata, a semantic name, and a record
149
    body
150
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
151
4543.2.14 by John Arbash Meinel
Clarify some comments, fix up a debugging change.
152
    def __init__(self, fileobj, stream_input=True):
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
153
        """Constructor
154
155
        :param fileobj: a file containing a bzip-encoded container
2520.4.148 by Aaron Bentley
Updates from review
156
        :param stream_input: If True, the BundleReader stream input rather than
157
            reading it all into memory at once.  Reading it into memory all at
158
            once is (currently) faster.
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
159
        """
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
160
        line = fileobj.readline()
161
        if line != '\n':
162
            fileobj.readline()
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
163
        self.patch_lines = []
2520.4.148 by Aaron Bentley
Updates from review
164
        if stream_input:
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
165
            source_file = iterablefile.IterableFile(self.iter_decode(fileobj))
166
        else:
167
            source_file = StringIO(bz2.decompress(fileobj.read()))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
168
        self._container_file = source_file
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
169
170
    @staticmethod
171
    def iter_decode(fileobj):
2520.4.118 by Aaron Bentley
Add docs
172
        """Iterate through decoded fragments of the file"""
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
173
        decompressor = bz2.BZ2Decompressor()
174
        for line in fileobj:
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
175
            try:
176
                yield decompressor.decompress(line)
177
            except EOFError:
178
                return
2520.4.22 by Aaron Bentley
Create ContainerReader
179
180
    @staticmethod
181
    def decode_name(name):
2520.4.118 by Aaron Bentley
Add docs
182
        """Decode a name from its container form into a semantic form
183
184
        :retval: content_kind, revision_id, file_id
185
        """
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
186
        segments = re.split('(//?)', name)
187
        names = ['']
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
188
        for segment in segments:
189
            if segment == '//':
190
                names[-1] += '/'
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
191
            elif segment == '/':
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
192
                names.append('')
193
            else:
194
                names[-1] += segment
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
195
        content_kind = names[0]
2520.4.95 by Aaron Bentley
Add support for header/info records
196
        revision_id = None
197
        file_id = None
198
        if len(names) > 1:
199
            revision_id = names[1]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
200
        if len(names) > 2:
201
            file_id = names[2]
202
        return content_kind, revision_id, file_id
2520.4.22 by Aaron Bentley
Create ContainerReader
203
204
    def iter_records(self):
2520.4.118 by Aaron Bentley
Add docs
205
        """Iterate through bundle records
206
207
        :return: a generator of (bytes, metadata, content_kind, revision_id,
208
            file_id)
209
        """
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
210
        iterator = pack.iter_records_from_file(self._container_file)
211
        for names, bytes in iterator:
2520.4.131 by Aaron Bentley
Raise BadBundle for records with wrong number of names
212
            if len(names) != 1:
213
                raise errors.BadBundle('Record has %d names instead of 1'
214
                                       % len(names))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
215
            metadata = bencode.bdecode(bytes)
2520.4.95 by Aaron Bentley
Add support for header/info records
216
            if metadata['storage_kind'] == 'header':
217
                bytes = None
218
            else:
219
                _unused, bytes = iterator.next()
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
220
            yield (bytes, metadata) + self.decode_name(names[0][0])
2520.4.22 by Aaron Bentley
Create ContainerReader
221
222
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
223
class BundleSerializerV4(bundle_serializer.BundleSerializer):
2520.4.118 by Aaron Bentley
Add docs
224
    """Implement the high-level bundle interface"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
225
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
226
    def write(self, repository, revision_ids, forced_bases, fileobj):
2520.4.118 by Aaron Bentley
Add docs
227
        """Write a bundle to a file-like object
228
229
        For backwards-compatibility only
230
        """
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
231
        write_op = BundleWriteOperation.from_old_args(repository, revision_ids,
232
                                                      forced_bases, fileobj)
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
233
        return write_op.do_write()
234
235
    def write_bundle(self, repository, target, base, fileobj):
2520.4.118 by Aaron Bentley
Add docs
236
        """Write a bundle to a file object
237
238
        :param repository: The repository to retrieve revision data from
239
        :param target: The head revision to include ancestors of
240
        :param base: The ancestor of the target to stop including acestors
241
            at.
242
        :param fileobj: The file-like object to write to
243
        """
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
244
        write_op =  BundleWriteOperation(base, target, repository, fileobj)
245
        return write_op.do_write()
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
246
247
    def read(self, file):
2520.4.118 by Aaron Bentley
Add docs
248
        """return a reader object for a given file"""
2520.4.72 by Aaron Bentley
Rename format to 4alpha
249
        bundle = BundleInfoV4(file, self)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
250
        return bundle
251
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
252
    @staticmethod
253
    def get_source_serializer(info):
2520.4.118 by Aaron Bentley
Add docs
254
        """Retrieve the serializer for a given info object"""
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
255
        return serializer.format_registry.get(info['serializer'])
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
256
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
257
258
class BundleWriteOperation(object):
2520.4.118 by Aaron Bentley
Add docs
259
    """Perform the operation of writing revisions to a bundle"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
260
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
261
    @classmethod
262
    def from_old_args(cls, repository, revision_ids, forced_bases, fileobj):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
263
        """Create a BundleWriteOperation from old-style arguments"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
264
        base, target = cls.get_base_target(revision_ids, forced_bases,
265
                                           repository)
266
        return BundleWriteOperation(base, target, repository, fileobj,
267
                                    revision_ids)
268
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
269
    def __init__(self, base, target, repository, fileobj, revision_ids=None):
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
270
        self.base = base
271
        self.target = target
272
        self.repository = repository
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
273
        bundle = BundleWriter(fileobj)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
274
        self.bundle = bundle
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
275
        if revision_ids is not None:
276
            self.revision_ids = revision_ids
277
        else:
4154.1.1 by Ian Clatworthy
make send use graph.find_difference() instead of walking all of history twice
278
            graph = repository.get_graph()
4154.1.3 by Ian Clatworthy
strip ghosts so test_bundle_with_ghosts works again
279
            revision_ids = graph.find_unique_ancestors(target, [base])
280
            # Strip ghosts
281
            parents = graph.get_parent_map(revision_ids)
282
            self.revision_ids = [r for r in revision_ids if r in parents]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
283
        self.revision_keys = set([(revid,) for revid in self.revision_ids])
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
284
285
    def do_write(self):
2520.4.118 by Aaron Bentley
Add docs
286
        """Write all data to the bundle"""
3794.4.1 by Aaron Bentley
Note the number of revisions being bundled (abentley)
287
        trace.note('Bundling %d revision(s).', len(self.revision_ids))
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
288
        self.repository.lock_read()
289
        try:
290
            self.bundle.begin()
291
            self.write_info()
292
            self.write_files()
293
            self.write_revisions()
294
            self.bundle.end()
295
        finally:
296
            self.repository.unlock()
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
297
        return self.revision_ids
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
298
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
299
    def write_info(self):
2520.4.118 by Aaron Bentley
Add docs
300
        """Write format info"""
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
301
        serializer_format = self.repository.get_serializer_format()
2520.4.99 by Aaron Bentley
Test conversion across models
302
        supports_rich_root = {True: 1, False: 0}[
303
            self.repository.supports_rich_root()]
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
304
        self.bundle.add_info_record(serializer=serializer_format,
2520.4.99 by Aaron Bentley
Test conversion across models
305
                                    supports_rich_root=supports_rich_root)
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
306
2520.4.51 by Aaron Bentley
Split iteration through file revisions into a method, so we can vary it
307
    def write_files(self):
2520.4.118 by Aaron Bentley
Add docs
308
        """Write bundle records for all revisions of all files"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
309
        text_keys = []
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
310
        altered_fileids = self.repository.fileids_altered_by_revision_ids(
311
                self.revision_ids)
312
        for file_id, revision_ids in altered_fileids.iteritems():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
313
            for revision_id in revision_ids:
314
                text_keys.append((file_id, revision_id))
3350.6.10 by Martin Pool
VersionedFiles review cleanups
315
        self._add_mp_records_keys('file', self.repository.texts, text_keys)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
316
317
    def write_revisions(self):
2520.4.118 by Aaron Bentley
Add docs
318
        """Write bundle records for all revisions and signatures"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
319
        inv_vf = self.repository.inventories
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
320
        topological_order = [key[-1] for key in multiparent.topo_iter_keys(
321
                                inv_vf, self.revision_keys)]
322
        revision_order = topological_order
2520.4.75 by Aaron Bentley
Fix traceback on empty bundles.
323
        if self.target is not None and self.target in self.revision_ids:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
324
            # Make sure the target revision is always the last entry
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
325
            revision_order = list(topological_order)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
326
            revision_order.remove(self.target)
327
            revision_order.append(self.target)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
328
        if self.repository._serializer.support_altered_by_hack:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
329
            # Repositories that support_altered_by_hack means that
330
            # inventories.make_mpdiffs() contains all the data about the tree
331
            # shape. Formats without support_altered_by_hack require
332
            # chk_bytes/etc, so we use a different code path.
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
333
            self._add_mp_records_keys('inventory', inv_vf,
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
334
                                      [(revid,) for revid in topological_order])
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
335
        else:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
336
            # Inventories should always be added in pure-topological order, so
337
            # that we can apply the mpdiff for the child to the parent texts.
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
338
            self._add_inventory_mpdiffs_from_serializer(topological_order)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
339
        self._add_revision_texts(revision_order)
340
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
341
    def _add_inventory_mpdiffs_from_serializer(self, revision_order):
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
342
        """Generate mpdiffs by serializing inventories.
343
344
        The current repository only has part of the tree shape information in
345
        the 'inventories' vf. So we use serializer.write_inventory_to_string to
346
        get a 'full' representation of the tree shape, and then generate
347
        mpdiffs on that data stream. This stream can then be reconstructed on
348
        the other side.
349
        """
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
350
        inventory_key_order = [(r,) for r in revision_order]
351
        parent_map = self.repository.inventories.get_parent_map(
352
                            inventory_key_order)
4543.2.5 by John Arbash Meinel
Fix issues with keys/ids and ghost handling.
353
        missing_keys = set(inventory_key_order).difference(parent_map)
354
        if missing_keys:
355
            raise errors.RevisionNotPresent(list(missing_keys)[0],
356
                                            self.repository.inventories)
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
357
        inv_to_str = self.repository._serializer.write_inventory_to_string
358
        # Make sure that we grab the parent texts first
359
        just_parents = set()
360
        map(just_parents.update, parent_map.itervalues())
4543.2.5 by John Arbash Meinel
Fix issues with keys/ids and ghost handling.
361
        just_parents.difference_update(parent_map)
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
362
        # Ignore ghost parents
363
        present_parents = self.repository.inventories.get_parent_map(
364
                            just_parents)
4543.2.5 by John Arbash Meinel
Fix issues with keys/ids and ghost handling.
365
        ghost_keys = just_parents.difference(present_parents)
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
366
        needed_inventories = list(present_parents) + inventory_key_order
367
        needed_inventories = [k[-1] for k in needed_inventories]
368
        all_lines = {}
369
        for inv in self.repository.iter_inventories(needed_inventories):
370
            revision_id = inv.revision_id
371
            key = (revision_id,)
372
            as_bytes = inv_to_str(inv)
373
            # The sha1 is validated as the xml/textual form, not as the
374
            # form-in-the-repository
375
            sha1 = osutils.sha_string(as_bytes)
376
            as_lines = osutils.split_lines(as_bytes)
4543.2.5 by John Arbash Meinel
Fix issues with keys/ids and ghost handling.
377
            del as_bytes
378
            all_lines[key] = as_lines
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
379
            if key in just_parents:
380
                # We don't transmit those entries
381
                continue
382
            # Create an mpdiff for this text, and add it to the output
383
            parent_keys = parent_map[key]
4543.2.5 by John Arbash Meinel
Fix issues with keys/ids and ghost handling.
384
            # See the comment in VF.make_mpdiffs about how this effects
385
            # ordering when there are ghosts present. I think we have a latent
386
            # bug
387
            parent_lines = [all_lines[p_key] for p_key in parent_keys
388
                            if p_key not in ghost_keys]
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
389
            diff = multiparent.MultiParent.from_lines(
390
                as_lines, parent_lines)
391
            text = ''.join(diff.to_patch())
4543.2.5 by John Arbash Meinel
Fix issues with keys/ids and ghost handling.
392
            parent_ids = [k[-1] for k in parent_keys]
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
393
            self.bundle.add_multiparent_record(text, sha1, parent_ids,
394
                                               'inventory', revision_id, None)
395
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
396
    def _add_revision_texts(self, revision_order):
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
397
        parent_map = self.repository.get_parent_map(revision_order)
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
398
        revision_to_str = self.repository._serializer.write_revision_to_string
399
        revisions = self.repository.get_revisions(revision_order)
400
        for revision in revisions:
401
            revision_id = revision.revision_id
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
402
            parents = parent_map.get(revision_id, None)
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
403
            revision_text = revision_to_str(revision)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
404
            self.bundle.add_fulltext_record(revision_text, parents,
2520.4.123 by Aaron Bentley
Cleanup of bundle code
405
                                       'revision', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
406
            try:
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
407
                self.bundle.add_fulltext_record(
408
                    self.repository.get_signature_text(
2520.4.123 by Aaron Bentley
Cleanup of bundle code
409
                    revision_id), parents, 'signature', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
410
            except errors.NoSuchRevision:
411
                pass
412
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
413
    @staticmethod
414
    def get_base_target(revision_ids, forced_bases, repository):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
415
        """Determine the base and target from old-style revision ids"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
416
        if len(revision_ids) == 0:
417
            return None, None
418
        target = revision_ids[0]
419
        base = forced_bases.get(target)
420
        if base is None:
421
            parents = repository.get_revision(target).parent_ids
422
            if len(parents) == 0:
423
                base = _mod_revision.NULL_REVISION
424
            else:
425
                base = parents[0]
426
        return base, target
427
3350.6.10 by Martin Pool
VersionedFiles review cleanups
428
    def _add_mp_records_keys(self, repo_kind, vf, keys):
2520.4.118 by Aaron Bentley
Add docs
429
        """Add multi-parent diff records to a bundle"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
430
        ordered_keys = list(multiparent.topo_iter_keys(vf, keys))
431
        mpdiffs = vf.make_mpdiffs(ordered_keys)
432
        sha1s = vf.get_sha1s(ordered_keys)
433
        parent_map = vf.get_parent_map(ordered_keys)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
434
        for mpdiff, item_key, in zip(mpdiffs, ordered_keys):
435
            sha1 = sha1s[item_key]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
436
            parents = [key[-1] for key in parent_map[item_key]]
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
437
            text = ''.join(mpdiff.to_patch())
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
438
            # Infer file id records as appropriate.
439
            if len(item_key) == 2:
440
                file_id = item_key[0]
441
            else:
442
                file_id = None
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
443
            self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
444
                                               item_key[-1], file_id)
2520.4.6 by Aaron Bentley
Get installation started
445
446
2520.4.72 by Aaron Bentley
Rename format to 4alpha
447
class BundleInfoV4(object):
2520.4.6 by Aaron Bentley
Get installation started
448
2520.4.118 by Aaron Bentley
Add docs
449
    """Provide (most of) the BundleInfo interface"""
2520.4.6 by Aaron Bentley
Get installation started
450
    def __init__(self, fileobj, serializer):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
451
        self._fileobj = fileobj
452
        self._serializer = serializer
453
        self.__real_revisions = None
454
        self.__revisions = None
455
456
    def install(self, repository):
457
        return self.install_revisions(repository)
458
2520.4.148 by Aaron Bentley
Updates from review
459
    def install_revisions(self, repository, stream_input=True):
460
        """Install this bundle's revisions into the specified repository
461
462
        :param target_repo: The repository to install into
463
        :param stream_input: If True, will stream input rather than reading it
464
            all into memory at once.  Reading it into memory all at once is
465
            (currently) faster.
466
        """
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
467
        repository.lock_write()
468
        try:
2520.4.148 by Aaron Bentley
Updates from review
469
            ri = RevisionInstaller(self.get_bundle_reader(stream_input),
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
470
                                   self._serializer, repository)
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
471
            return ri.install()
472
        finally:
473
            repository.unlock()
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
474
2520.4.109 by Aaron Bentley
start work on directive cherry-picking
475
    def get_merge_request(self, target_repo):
476
        """Provide data for performing a merge
477
478
        Returns suggested base, suggested target, and patch verification status
479
        """
480
        return None, self.target, 'inapplicable'
481
2520.4.148 by Aaron Bentley
Updates from review
482
    def get_bundle_reader(self, stream_input=True):
483
        """Return a new BundleReader for the associated bundle
484
485
        :param stream_input: If True, the BundleReader stream input rather than
486
            reading it all into memory at once.  Reading it into memory all at
487
            once is (currently) faster.
488
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
489
        self._fileobj.seek(0)
2520.4.148 by Aaron Bentley
Updates from review
490
        return BundleReader(self._fileobj, stream_input)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
491
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
492
    def _get_real_revisions(self):
493
        if self.__real_revisions is None:
494
            self.__real_revisions = []
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
495
            bundle_reader = self.get_bundle_reader()
2520.4.102 by Aaron Bentley
rename parents to metadata
496
            for bytes, metadata, repo_kind, revision_id, file_id in \
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
497
                bundle_reader.iter_records():
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
498
                if repo_kind == 'info':
499
                    serializer =\
2520.4.102 by Aaron Bentley
rename parents to metadata
500
                        self._serializer.get_source_serializer(metadata)
2520.4.22 by Aaron Bentley
Create ContainerReader
501
                if repo_kind == 'revision':
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
502
                    rev = serializer.read_revision_from_string(bytes)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
503
                    self.__real_revisions.append(rev)
504
        return self.__real_revisions
505
    real_revisions = property(_get_real_revisions)
506
507
    def _get_revisions(self):
508
        if self.__revisions is None:
509
            self.__revisions = []
510
            for revision in self.real_revisions:
2520.4.33 by Aaron Bentley
remove test dependencies on serialization minutia
511
                self.__revisions.append(
512
                    bundle_data.RevisionInfo.from_revision(revision))
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
513
        return self.__revisions
514
515
    revisions = property(_get_revisions)
516
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
517
    def _get_target(self):
518
        return self.revisions[-1].revision_id
519
520
    target = property(_get_target)
521
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
522
523
class RevisionInstaller(object):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
524
    """Installs revisions into a repository"""
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
525
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
526
    def __init__(self, container, serializer, repository):
527
        self._container = container
2520.4.6 by Aaron Bentley
Get installation started
528
        self._serializer = serializer
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
529
        self._repository = repository
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
530
        self._info = None
2520.4.99 by Aaron Bentley
Test conversion across models
531
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
532
    def install(self):
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
533
        """Perform the installation.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
534
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
535
        Must be called with the Repository locked.
536
        """
537
        self._repository.start_write_group()
538
        try:
2856.1.2 by Robert Collins
Review feedback.
539
            result = self._install_in_write_group()
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
540
        except:
541
            self._repository.abort_write_group()
542
            raise
543
        self._repository.commit_write_group()
544
        return result
545
2856.1.2 by Robert Collins
Review feedback.
546
    def _install_in_write_group(self):
2520.4.6 by Aaron Bentley
Get installation started
547
        current_file = None
548
        current_versionedfile = None
549
        pending_file_records = []
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
550
        inventory_vf = None
551
        pending_inventory_records = []
2520.4.8 by Aaron Bentley
Serialize inventory
552
        added_inv = set()
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
553
        target_revision = None
2520.4.58 by Aaron Bentley
Propogate support for metadata to iter_revisions, add storage kind
554
        for bytes, metadata, repo_kind, revision_id, file_id in\
2520.4.22 by Aaron Bentley
Create ContainerReader
555
            self._container.iter_records():
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
556
            if repo_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
557
                if self._info is not None:
558
                    raise AssertionError()
2520.4.123 by Aaron Bentley
Cleanup of bundle code
559
                self._handle_info(metadata)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
560
            if (pending_file_records and
561
                (repo_kind, file_id) != ('file', current_file)):
562
                # Flush the data for a single file - prevents memory
563
                # spiking due to buffering all files in memory.
564
                self._install_mp_records_keys(self._repository.texts,
565
                    pending_file_records)
2520.4.8 by Aaron Bentley
Serialize inventory
566
                current_file = None
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
567
                del pending_file_records[:]
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
568
            if len(pending_inventory_records) > 0 and repo_kind != 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
569
                self._install_inventory_records(pending_inventory_records)
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
570
                pending_inventory_records = []
571
            if repo_kind == 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
572
                pending_inventory_records.append(((revision_id,), metadata, bytes))
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
573
            if repo_kind == 'revision':
574
                target_revision = revision_id
575
                self._install_revision(revision_id, metadata, bytes)
576
            if repo_kind == 'signature':
577
                self._install_signature(revision_id, metadata, bytes)
2520.4.22 by Aaron Bentley
Create ContainerReader
578
            if repo_kind == 'file':
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
579
                current_file = file_id
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
580
                pending_file_records.append(((file_id, revision_id), metadata, bytes))
581
        self._install_mp_records_keys(self._repository.texts, pending_file_records)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
582
        return target_revision
2520.4.6 by Aaron Bentley
Get installation started
583
2520.4.123 by Aaron Bentley
Cleanup of bundle code
584
    def _handle_info(self, info):
585
        """Extract data from an info record"""
586
        self._info = info
587
        self._source_serializer = self._serializer.get_source_serializer(info)
588
        if (info['supports_rich_root'] == 0 and
589
            self._repository.supports_rich_root()):
590
            self.update_root = True
591
        else:
592
            self.update_root = False
593
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
594
    def _install_mp_records(self, versionedfile, records):
2520.4.61 by Aaron Bentley
Do bulk insertion of records
595
        if len(records) == 0:
596
            return
597
        d_func = multiparent.MultiParent.from_patch
598
        vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
599
                      records if r not in versionedfile]
600
        versionedfile.add_mpdiffs(vf_records)
2520.4.8 by Aaron Bentley
Serialize inventory
601
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
602
    def _install_mp_records_keys(self, versionedfile, records):
603
        d_func = multiparent.MultiParent.from_patch
604
        vf_records = []
605
        for key, meta, text in records:
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
606
            # Adapt to tuple interface: A length two key is a file_id,
607
            # revision_id pair, a length 1 key is a
608
            # revision/signature/inventory. We need to do this because
609
            # the metadata extraction from the bundle has not yet been updated
610
            # to use the consistent tuple interface itself.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
611
            if len(key) == 2:
612
                prefix = key[:1]
613
            else:
614
                prefix = ()
615
            parents = [prefix + (parent,) for parent in meta['parents']]
616
            vf_records.append((key, parents, meta['sha1'], d_func(text)))
617
        versionedfile.add_mpdiffs(vf_records)
618
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
619
    def _get_parent_inventory_texts(self, inventory_text_cache,
620
                                    inventory_cache, parent_ids):
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
621
        cached_parent_texts = {}
622
        remaining_parent_ids = []
623
        for parent_id in parent_ids:
624
            p_text = inventory_text_cache.get(parent_id, None)
625
            if p_text is None:
626
                remaining_parent_ids.append(parent_id)
627
            else:
628
                cached_parent_texts[parent_id] = p_text
629
        ghosts = ()
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
630
        # TODO: Use inventory_cache to grab inventories we already have in
631
        #       memory
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
632
        if remaining_parent_ids:
633
            # first determine what keys are actually present in the local
634
            # inventories object (don't use revisions as they haven't been
635
            # installed yet.)
636
            parent_keys = [(r,) for r in remaining_parent_ids]
637
            present_parent_map = self._repository.inventories.get_parent_map(
638
                                        parent_keys)
639
            present_parent_ids = []
640
            ghosts = set()
641
            for p_id in remaining_parent_ids:
642
                if (p_id,) in present_parent_map:
643
                    present_parent_ids.append(p_id)
644
                else:
645
                    ghosts.add(p_id)
646
            to_string = self._source_serializer.write_inventory_to_string
647
            for parent_inv in self._repository.iter_inventories(
648
                                    present_parent_ids):
649
                p_text = to_string(parent_inv)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
650
                inventory_cache[parent_inv.revision_id] = parent_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
651
                cached_parent_texts[parent_inv.revision_id] = p_text
652
                inventory_text_cache[parent_inv.revision_id] = p_text
653
654
        parent_texts = [cached_parent_texts[parent_id]
655
                        for parent_id in parent_ids
656
                         if parent_id not in ghosts]
657
        return parent_texts
658
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
659
    def _install_inventory_records(self, records):
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
660
        if (self._info['serializer'] == self._repository._serializer.format_num
661
            and self._repository._serializer.support_altered_by_hack):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
662
            return self._install_mp_records_keys(self._repository.inventories,
663
                records)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
664
        # Use a 10MB text cache, since these are string xml inventories. Note
665
        # that 10MB is fairly small for large projects (a single inventory can
666
        # be >5MB). Another possibility is to cache 10-20 inventory texts
667
        # instead
668
        inventory_text_cache = lru_cache.LRUSizeCache(10*1024*1024)
4543.2.21 by John Arbash Meinel
A few more tiny tweaks to comments, etc.
669
        # Also cache the in-memory representation. This allows us to create
670
        # inventory deltas to apply rather than calling add_inventory from
671
        # scratch each time.
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
672
        inventory_cache = lru_cache.LRUCache(10)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
673
        pb = ui.ui_factory.nested_progress_bar()
674
        try:
675
            num_records = len(records)
676
            for idx, (key, metadata, bytes) in enumerate(records):
677
                pb.update('installing inventory', idx, num_records)
678
                revision_id = key[-1]
679
                parent_ids = metadata['parents']
680
                # Note: This assumes the local ghosts are identical to the
681
                #       ghosts in the source, as the Bundle serialization
682
                #       format doesn't record ghosts.
683
                p_texts = self._get_parent_inventory_texts(inventory_text_cache,
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
684
                                                           inventory_cache,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
685
                                                           parent_ids)
686
                # Why does to_lines() take strings as the source, it seems that
687
                # it would have to cast to a list of lines, which we get back
688
                # as lines and then cast back to a string.
689
                target_lines = multiparent.MultiParent.from_patch(bytes
690
                            ).to_lines(p_texts)
691
                inv_text = ''.join(target_lines)
692
                del target_lines
693
                sha1 = osutils.sha_string(inv_text)
694
                if sha1 != metadata['sha1']:
695
                    raise errors.BadBundle("Can't convert to target format")
696
                # Add this to the cache so we don't have to extract it again.
697
                inventory_text_cache[revision_id] = inv_text
698
                target_inv = self._source_serializer.read_inventory_from_string(
699
                    inv_text)
700
                self._handle_root(target_inv, parent_ids)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
701
                parent_inv = None
702
                if parent_ids:
703
                    parent_inv = inventory_cache.get(parent_ids[0], None)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
704
                try:
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
705
                    if parent_inv is None:
706
                        self._repository.add_inventory(revision_id, target_inv,
707
                                                       parent_ids)
708
                    else:
709
                        delta = target_inv._make_delta(parent_inv)
710
                        self._repository.add_inventory_by_delta(parent_ids[0],
711
                            delta, revision_id, parent_ids)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
712
                except errors.UnsupportedInventoryKind:
713
                    raise errors.IncompatibleRevision(repr(self._repository))
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
714
                inventory_cache[revision_id] = target_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
715
        finally:
716
            pb.finished()
2520.4.99 by Aaron Bentley
Test conversion across models
717
718
    def _handle_root(self, target_inv, parent_ids):
719
        revision_id = target_inv.revision_id
720
        if self.update_root:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
721
            text_key = (target_inv.root.file_id, revision_id)
722
            parent_keys = [(target_inv.root.file_id, parent) for
723
                parent in parent_ids]
724
            self._repository.texts.add_lines(text_key, parent_keys, [])
2520.4.99 by Aaron Bentley
Test conversion across models
725
        elif not self._repository.supports_rich_root():
726
            if target_inv.root.revision != revision_id:
727
                raise errors.IncompatibleRevision(repr(self._repository))
728
2520.4.59 by Aaron Bentley
Push metadata down the stack
729
    def _install_revision(self, revision_id, metadata, text):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
730
        if self._repository.has_revision(revision_id):
731
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
732
        revision = self._source_serializer.read_revision_from_string(text)
733
        self._repository.add_revision(revision.revision_id, revision)
2520.4.34 by Aaron Bentley
Add signature support
734
2520.4.59 by Aaron Bentley
Push metadata down the stack
735
    def _install_signature(self, revision_id, metadata, text):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
736
        transaction = self._repository.get_transaction()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
737
        if self._repository.has_signature_for_revision_id(revision_id):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
738
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
739
        self._repository.add_signature_text(revision_id, text)