5374.2.5
by John Arbash Meinel
Rework things a bit so the logic can be shared. |
1 |
# Copyright (C) 2007-2010 Canonical Ltd
|
2520.4.85
by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!) |
2 |
#
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
#
|
|
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
#
|
|
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
4183.7.1
by Sabin Iacob
update FSF mailing address |
15 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
2520.4.85
by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!) |
16 |
|
6379.6.3
by Jelmer Vernooij
Use absolute_import. |
17 |
from __future__ import absolute_import |
18 |
||
2520.4.20
by Aaron Bentley
Compress and base64-encode bundle contents |
19 |
from cStringIO import StringIO |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
20 |
import bz2 |
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
21 |
import re |
2520.4.20
by Aaron Bentley
Compress and base64-encode bundle contents |
22 |
|
2520.4.13
by Aaron Bentley
Use real container implementation |
23 |
from bzrlib import ( |
2520.4.34
by Aaron Bentley
Add signature support |
24 |
errors, |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
25 |
iterablefile, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
26 |
lru_cache, |
2520.4.13
by Aaron Bentley
Use real container implementation |
27 |
multiparent, |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
28 |
osutils, |
2520.4.13
by Aaron Bentley
Use real container implementation |
29 |
pack, |
2520.4.40
by Aaron Bentley
Add human-readable diff to bundles |
30 |
revision as _mod_revision, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
31 |
serializer, |
2520.4.45
by Aaron Bentley
Handle inconsistencies in last-modified-revision between vf and inventory |
32 |
trace, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
33 |
ui, |
5374.2.5
by John Arbash Meinel
Rework things a bit so the logic can be shared. |
34 |
versionedfile as _mod_versionedfile, |
2520.4.13
by Aaron Bentley
Use real container implementation |
35 |
)
|
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
36 |
from bzrlib.bundle import bundle_data, serializer as bundle_serializer |
6164.1.1
by Jelmer Vernooij
Move file_ids_altered_by to VersionedFileRepository. |
37 |
from bzrlib.i18n import ngettext |
2694.5.1
by Alexander Belchenko
pyrex bencode (without benchmarks) |
38 |
from bzrlib import bencode |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
39 |
|
2520.4.4
by Aaron Bentley
Get basis support for a new bundle format in place |
40 |
|
5374.2.5
by John Arbash Meinel
Rework things a bit so the logic can be shared. |
41 |
class _MPDiffInventoryGenerator(_mod_versionedfile._MPDiffGenerator): |
42 |
"""Generate Inventory diffs serialized inventories."""
|
|
43 |
||
44 |
def __init__(self, repo, inventory_keys): |
|
45 |
super(_MPDiffInventoryGenerator, self).__init__(repo.inventories, |
|
46 |
inventory_keys) |
|
47 |
self.repo = repo |
|
48 |
self.sha1s = {} |
|
49 |
||
50 |
def iter_diffs(self): |
|
51 |
"""Compute the diffs one at a time."""
|
|
52 |
# This is instead of compute_diffs() since we guarantee our ordering of
|
|
53 |
# inventories, we don't have to do any buffering
|
|
54 |
self._find_needed_keys() |
|
55 |
# We actually use a slightly different ordering. We grab all of the
|
|
56 |
# parents first, and then grab the ordered requests.
|
|
57 |
needed_ids = [k[-1] for k in self.present_parents] |
|
58 |
needed_ids.extend([k[-1] for k in self.ordered_keys]) |
|
59 |
inv_to_str = self.repo._serializer.write_inventory_to_string |
|
60 |
for inv in self.repo.iter_inventories(needed_ids): |
|
61 |
revision_id = inv.revision_id |
|
62 |
key = (revision_id,) |
|
63 |
if key in self.present_parents: |
|
64 |
# Not a key we will transmit, which is a shame, since because
|
|
65 |
# of that bundles don't work with stacked branches
|
|
66 |
parent_ids = None |
|
67 |
else: |
|
68 |
parent_ids = [k[-1] for k in self.parent_map[key]] |
|
69 |
as_bytes = inv_to_str(inv) |
|
70 |
self._process_one_record(key, (as_bytes,)) |
|
71 |
if parent_ids is None: |
|
72 |
continue
|
|
73 |
diff = self.diffs.pop(key) |
|
74 |
sha1 = osutils.sha_string(as_bytes) |
|
75 |
yield revision_id, parent_ids, sha1, diff |
|
76 |
||
77 |
||
2520.4.25
by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader |
78 |
class BundleWriter(object): |
2520.4.118
by Aaron Bentley
Add docs |
79 |
"""Writer for bundle-format files.
|
80 |
||
81 |
This serves roughly the same purpose as ContainerReader, but acts as a
|
|
82 |
layer on top of it.
|
|
83 |
||
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
84 |
Provides ways of writing the specific record types supported this bundle
|
2520.4.118
by Aaron Bentley
Add docs |
85 |
format.
|
86 |
"""
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
87 |
|
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
88 |
def __init__(self, fileobj): |
2520.4.27
by Aaron Bentley
Use less memory when writing bzip-encoded files |
89 |
self._container = pack.ContainerWriter(self._write_encoded) |
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
90 |
self._fileobj = fileobj |
2520.4.27
by Aaron Bentley
Use less memory when writing bzip-encoded files |
91 |
self._compressor = bz2.BZ2Compressor() |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
92 |
|
2520.4.118
by Aaron Bentley
Add docs |
93 |
def _write_encoded(self, bytes): |
94 |
"""Write bzip2-encoded bytes to the file"""
|
|
95 |
self._fileobj.write(self._compressor.compress(bytes)) |
|
96 |
||
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
97 |
def begin(self): |
2520.4.118
by Aaron Bentley
Add docs |
98 |
"""Start writing the bundle"""
|
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
99 |
self._fileobj.write(bundle_serializer._get_bundle_header( |
100 |
bundle_serializer.v4_string)) |
|
2520.4.24
by Aaron Bentley
Move heading writing above container beginning |
101 |
self._fileobj.write('#\n') |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
102 |
self._container.begin() |
103 |
||
104 |
def end(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
105 |
"""Finish writing the bundle"""
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
106 |
self._container.end() |
2520.4.76
by Aaron Bentley
Move base64-encoding into merge directives |
107 |
self._fileobj.write(self._compressor.flush()) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
108 |
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
109 |
def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind, |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
110 |
revision_id, file_id): |
2520.4.118
by Aaron Bentley
Add docs |
111 |
"""Add a record for a multi-parent diff
|
112 |
||
113 |
:mp_bytes: A multi-parent diff, as a bytestring
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
114 |
:sha1: The sha1 hash of the fulltext
|
2520.4.118
by Aaron Bentley
Add docs |
115 |
:parents: a list of revision-ids of the parents
|
116 |
:repo_kind: The kind of object in the repository. May be 'file' or
|
|
117 |
'inventory'
|
|
118 |
:revision_id: The revision id of the mpdiff being added.
|
|
119 |
:file_id: The file-id of the file, or None for inventories.
|
|
120 |
"""
|
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
121 |
metadata = {'parents': parents, |
122 |
'storage_kind': 'mpdiff', |
|
123 |
'sha1': sha1} |
|
124 |
self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id) |
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
125 |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
126 |
def add_fulltext_record(self, bytes, parents, repo_kind, revision_id): |
2520.4.118
by Aaron Bentley
Add docs |
127 |
"""Add a record for a fulltext
|
128 |
||
129 |
:bytes: The fulltext, as a bytestring
|
|
130 |
:parents: a list of revision-ids of the parents
|
|
131 |
:repo_kind: The kind of object in the repository. May be 'revision' or
|
|
132 |
'signature'
|
|
133 |
:revision_id: The revision id of the fulltext being added.
|
|
134 |
"""
|
|
135 |
metadata = {'parents': parents, |
|
2520.5.3
by Aaron Bentley
fix sha1 in bundle format 4 |
136 |
'storage_kind': 'mpdiff'} |
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
137 |
self._add_record(bytes, {'parents': parents, |
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
138 |
'storage_kind': 'fulltext'}, repo_kind, revision_id, None) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
139 |
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
140 |
def add_info_record(self, **kwargs): |
2520.4.118
by Aaron Bentley
Add docs |
141 |
"""Add an info record to the bundle
|
142 |
||
143 |
Any parameters may be supplied, except 'self' and 'storage_kind'.
|
|
144 |
Values must be lists, strings, integers, dicts, or a combination.
|
|
145 |
"""
|
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
146 |
kwargs['storage_kind'] = 'header' |
147 |
self._add_record(None, kwargs, 'info', None, None) |
|
148 |
||
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
149 |
@staticmethod
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
150 |
def encode_name(content_kind, revision_id, file_id=None): |
2520.4.118
by Aaron Bentley
Add docs |
151 |
"""Encode semantic ids as a container name"""
|
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
152 |
if content_kind not in ('revision', 'file', 'inventory', 'signature', |
153 |
'info'): |
|
154 |
raise ValueError(content_kind) |
|
2520.4.118
by Aaron Bentley
Add docs |
155 |
if content_kind == 'file': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
156 |
if file_id is None: |
157 |
raise AssertionError() |
|
2520.4.118
by Aaron Bentley
Add docs |
158 |
else: |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
159 |
if file_id is not None: |
160 |
raise AssertionError() |
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
161 |
if content_kind == 'info': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
162 |
if revision_id is not None: |
163 |
raise AssertionError() |
|
164 |
elif revision_id is None: |
|
165 |
raise AssertionError() |
|
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
166 |
names = [n.replace('/', '//') for n in |
167 |
(content_kind, revision_id, file_id) if n is not None] |
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
168 |
return '/'.join(names) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
169 |
|
2520.4.56
by Aaron Bentley
Begin adding support for arbitrary metadata |
170 |
def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id): |
2520.4.118
by Aaron Bentley
Add docs |
171 |
"""Add a bundle record to the container.
|
172 |
||
173 |
Most bundle records are recorded as header/body pairs, with the
|
|
174 |
body being nameless. Records with storage_kind 'header' have no
|
|
175 |
body.
|
|
176 |
"""
|
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
177 |
name = self.encode_name(repo_kind, revision_id, file_id) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
178 |
encoded_metadata = bencode.bencode(metadata) |
2682.1.1
by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings |
179 |
self._container.add_bytes_record(encoded_metadata, [(name, )]) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
180 |
if metadata['storage_kind'] != 'header': |
181 |
self._container.add_bytes_record(bytes, []) |
|
2520.4.13
by Aaron Bentley
Use real container implementation |
182 |
|
2520.4.7
by Aaron Bentley
Fix patch deserialization |
183 |
|
2520.4.25
by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader |
184 |
class BundleReader(object): |
2520.4.118
by Aaron Bentley
Add docs |
185 |
"""Reader for bundle-format files.
|
186 |
||
187 |
This serves roughly the same purpose as ContainerReader, but acts as a
|
|
188 |
layer on top of it, providing metadata, a semantic name, and a record
|
|
189 |
body
|
|
190 |
"""
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
191 |
|
4543.2.14
by John Arbash Meinel
Clarify some comments, fix up a debugging change. |
192 |
def __init__(self, fileobj, stream_input=True): |
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
193 |
"""Constructor
|
194 |
||
195 |
:param fileobj: a file containing a bzip-encoded container
|
|
2520.4.148
by Aaron Bentley
Updates from review |
196 |
:param stream_input: If True, the BundleReader stream input rather than
|
197 |
reading it all into memory at once. Reading it into memory all at
|
|
198 |
once is (currently) faster.
|
|
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
199 |
"""
|
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
200 |
line = fileobj.readline() |
201 |
if line != '\n': |
|
202 |
fileobj.readline() |
|
2520.4.40
by Aaron Bentley
Add human-readable diff to bundles |
203 |
self.patch_lines = [] |
2520.4.148
by Aaron Bentley
Updates from review |
204 |
if stream_input: |
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
205 |
source_file = iterablefile.IterableFile(self.iter_decode(fileobj)) |
206 |
else: |
|
207 |
source_file = StringIO(bz2.decompress(fileobj.read())) |
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
208 |
self._container_file = source_file |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
209 |
|
210 |
@staticmethod
|
|
211 |
def iter_decode(fileobj): |
|
2520.4.118
by Aaron Bentley
Add docs |
212 |
"""Iterate through decoded fragments of the file"""
|
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
213 |
decompressor = bz2.BZ2Decompressor() |
214 |
for line in fileobj: |
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
215 |
try: |
216 |
yield decompressor.decompress(line) |
|
217 |
except EOFError: |
|
218 |
return
|
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
219 |
|
220 |
@staticmethod
|
|
221 |
def decode_name(name): |
|
2520.4.118
by Aaron Bentley
Add docs |
222 |
"""Decode a name from its container form into a semantic form
|
223 |
||
224 |
:retval: content_kind, revision_id, file_id
|
|
225 |
"""
|
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
226 |
segments = re.split('(//?)', name) |
227 |
names = [''] |
|
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
228 |
for segment in segments: |
229 |
if segment == '//': |
|
230 |
names[-1] += '/' |
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
231 |
elif segment == '/': |
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
232 |
names.append('') |
233 |
else: |
|
234 |
names[-1] += segment |
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
235 |
content_kind = names[0] |
2520.4.95
by Aaron Bentley
Add support for header/info records |
236 |
revision_id = None |
237 |
file_id = None |
|
238 |
if len(names) > 1: |
|
239 |
revision_id = names[1] |
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
240 |
if len(names) > 2: |
241 |
file_id = names[2] |
|
242 |
return content_kind, revision_id, file_id |
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
243 |
|
244 |
def iter_records(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
245 |
"""Iterate through bundle records
|
246 |
||
247 |
:return: a generator of (bytes, metadata, content_kind, revision_id,
|
|
248 |
file_id)
|
|
249 |
"""
|
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
250 |
iterator = pack.iter_records_from_file(self._container_file) |
251 |
for names, bytes in iterator: |
|
2520.4.131
by Aaron Bentley
Raise BadBundle for records with wrong number of names |
252 |
if len(names) != 1: |
253 |
raise errors.BadBundle('Record has %d names instead of 1' |
|
254 |
% len(names)) |
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
255 |
metadata = bencode.bdecode(bytes) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
256 |
if metadata['storage_kind'] == 'header': |
257 |
bytes = None |
|
258 |
else: |
|
259 |
_unused, bytes = iterator.next() |
|
2682.1.1
by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings |
260 |
yield (bytes, metadata) + self.decode_name(names[0][0]) |
2520.4.22
by Aaron Bentley
Create ContainerReader |
261 |
|
262 |
||
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
263 |
class BundleSerializerV4(bundle_serializer.BundleSerializer): |
2520.4.118
by Aaron Bentley
Add docs |
264 |
"""Implement the high-level bundle interface"""
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
265 |
|
2520.4.4
by Aaron Bentley
Get basis support for a new bundle format in place |
266 |
def write(self, repository, revision_ids, forced_bases, fileobj): |
2520.4.118
by Aaron Bentley
Add docs |
267 |
"""Write a bundle to a file-like object
|
268 |
||
269 |
For backwards-compatibility only
|
|
270 |
"""
|
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
271 |
write_op = BundleWriteOperation.from_old_args(repository, revision_ids, |
272 |
forced_bases, fileobj) |
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
273 |
return write_op.do_write() |
274 |
||
275 |
def write_bundle(self, repository, target, base, fileobj): |
|
2520.4.118
by Aaron Bentley
Add docs |
276 |
"""Write a bundle to a file object
|
277 |
||
278 |
:param repository: The repository to retrieve revision data from
|
|
279 |
:param target: The head revision to include ancestors of
|
|
280 |
:param base: The ancestor of the target to stop including acestors
|
|
281 |
at.
|
|
282 |
:param fileobj: The file-like object to write to
|
|
283 |
"""
|
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
284 |
write_op = BundleWriteOperation(base, target, repository, fileobj) |
285 |
return write_op.do_write() |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
286 |
|
287 |
def read(self, file): |
|
2520.4.118
by Aaron Bentley
Add docs |
288 |
"""return a reader object for a given file"""
|
2520.4.72
by Aaron Bentley
Rename format to 4alpha |
289 |
bundle = BundleInfoV4(file, self) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
290 |
return bundle |
291 |
||
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
292 |
@staticmethod
|
293 |
def get_source_serializer(info): |
|
2520.4.118
by Aaron Bentley
Add docs |
294 |
"""Retrieve the serializer for a given info object"""
|
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
295 |
return serializer.format_registry.get(info['serializer']) |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
296 |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
297 |
|
298 |
class BundleWriteOperation(object): |
|
2520.4.118
by Aaron Bentley
Add docs |
299 |
"""Perform the operation of writing revisions to a bundle"""
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
300 |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
301 |
@classmethod
|
302 |
def from_old_args(cls, repository, revision_ids, forced_bases, fileobj): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
303 |
"""Create a BundleWriteOperation from old-style arguments"""
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
304 |
base, target = cls.get_base_target(revision_ids, forced_bases, |
305 |
repository) |
|
306 |
return BundleWriteOperation(base, target, repository, fileobj, |
|
307 |
revision_ids) |
|
308 |
||
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
309 |
def __init__(self, base, target, repository, fileobj, revision_ids=None): |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
310 |
self.base = base |
311 |
self.target = target |
|
312 |
self.repository = repository |
|
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
313 |
bundle = BundleWriter(fileobj) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
314 |
self.bundle = bundle |
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
315 |
if revision_ids is not None: |
316 |
self.revision_ids = revision_ids |
|
317 |
else: |
|
4154.1.1
by Ian Clatworthy
make send use graph.find_difference() instead of walking all of history twice |
318 |
graph = repository.get_graph() |
4154.1.3
by Ian Clatworthy
strip ghosts so test_bundle_with_ghosts works again |
319 |
revision_ids = graph.find_unique_ancestors(target, [base]) |
320 |
# Strip ghosts
|
|
321 |
parents = graph.get_parent_map(revision_ids) |
|
322 |
self.revision_ids = [r for r in revision_ids if r in parents] |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
323 |
self.revision_keys = set([(revid,) for revid in self.revision_ids]) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
324 |
|
325 |
def do_write(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
326 |
"""Write all data to the bundle"""
|
6143.1.3
by Jonathan Riddell
more plurals |
327 |
trace.note(ngettext('Bundling %d revision.', 'Bundling %d revisions.', |
328 |
len(self.revision_ids)), len(self.revision_ids)) |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
329 |
self.repository.lock_read() |
330 |
try: |
|
331 |
self.bundle.begin() |
|
332 |
self.write_info() |
|
333 |
self.write_files() |
|
334 |
self.write_revisions() |
|
335 |
self.bundle.end() |
|
336 |
finally: |
|
337 |
self.repository.unlock() |
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
338 |
return self.revision_ids |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
339 |
|
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
340 |
def write_info(self): |
2520.4.118
by Aaron Bentley
Add docs |
341 |
"""Write format info"""
|
2520.4.113
by Aaron Bentley
Avoid peeking at Repository._serializer |
342 |
serializer_format = self.repository.get_serializer_format() |
2520.4.99
by Aaron Bentley
Test conversion across models |
343 |
supports_rich_root = {True: 1, False: 0}[ |
344 |
self.repository.supports_rich_root()] |
|
2520.4.113
by Aaron Bentley
Avoid peeking at Repository._serializer |
345 |
self.bundle.add_info_record(serializer=serializer_format, |
2520.4.99
by Aaron Bentley
Test conversion across models |
346 |
supports_rich_root=supports_rich_root) |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
347 |
|
2520.4.51
by Aaron Bentley
Split iteration through file revisions into a method, so we can vary it |
348 |
def write_files(self): |
2520.4.118
by Aaron Bentley
Add docs |
349 |
"""Write bundle records for all revisions of all files"""
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
350 |
text_keys = [] |
3350.6.7
by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where. |
351 |
altered_fileids = self.repository.fileids_altered_by_revision_ids( |
352 |
self.revision_ids) |
|
353 |
for file_id, revision_ids in altered_fileids.iteritems(): |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
354 |
for revision_id in revision_ids: |
355 |
text_keys.append((file_id, revision_id)) |
|
3350.6.10
by Martin Pool
VersionedFiles review cleanups |
356 |
self._add_mp_records_keys('file', self.repository.texts, text_keys) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
357 |
|
358 |
def write_revisions(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
359 |
"""Write bundle records for all revisions and signatures"""
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
360 |
inv_vf = self.repository.inventories |
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
361 |
topological_order = [key[-1] for key in multiparent.topo_iter_keys( |
362 |
inv_vf, self.revision_keys)] |
|
363 |
revision_order = topological_order |
|
2520.4.75
by Aaron Bentley
Fix traceback on empty bundles. |
364 |
if self.target is not None and self.target in self.revision_ids: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
365 |
# Make sure the target revision is always the last entry
|
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
366 |
revision_order = list(topological_order) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
367 |
revision_order.remove(self.target) |
368 |
revision_order.append(self.target) |
|
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
369 |
if self.repository._serializer.support_altered_by_hack: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
370 |
# Repositories that support_altered_by_hack means that
|
371 |
# inventories.make_mpdiffs() contains all the data about the tree
|
|
372 |
# shape. Formats without support_altered_by_hack require
|
|
373 |
# chk_bytes/etc, so we use a different code path.
|
|
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
374 |
self._add_mp_records_keys('inventory', inv_vf, |
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
375 |
[(revid,) for revid in topological_order]) |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
376 |
else: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
377 |
# Inventories should always be added in pure-topological order, so
|
378 |
# that we can apply the mpdiff for the child to the parent texts.
|
|
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
379 |
self._add_inventory_mpdiffs_from_serializer(topological_order) |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
380 |
self._add_revision_texts(revision_order) |
381 |
||
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
382 |
def _add_inventory_mpdiffs_from_serializer(self, revision_order): |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
383 |
"""Generate mpdiffs by serializing inventories.
|
384 |
||
385 |
The current repository only has part of the tree shape information in
|
|
386 |
the 'inventories' vf. So we use serializer.write_inventory_to_string to
|
|
387 |
get a 'full' representation of the tree shape, and then generate
|
|
388 |
mpdiffs on that data stream. This stream can then be reconstructed on
|
|
389 |
the other side.
|
|
390 |
"""
|
|
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
391 |
inventory_key_order = [(r,) for r in revision_order] |
5374.2.5
by John Arbash Meinel
Rework things a bit so the logic can be shared. |
392 |
generator = _MPDiffInventoryGenerator(self.repository, |
393 |
inventory_key_order) |
|
394 |
for revision_id, parent_ids, sha1, diff in generator.iter_diffs(): |
|
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
395 |
text = ''.join(diff.to_patch()) |
396 |
self.bundle.add_multiparent_record(text, sha1, parent_ids, |
|
397 |
'inventory', revision_id, None) |
|
398 |
||
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
399 |
def _add_revision_texts(self, revision_order): |
3099.3.5
by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly. |
400 |
parent_map = self.repository.get_parent_map(revision_order) |
4202.3.1
by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together. |
401 |
revision_to_str = self.repository._serializer.write_revision_to_string |
402 |
revisions = self.repository.get_revisions(revision_order) |
|
403 |
for revision in revisions: |
|
404 |
revision_id = revision.revision_id |
|
3099.3.5
by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly. |
405 |
parents = parent_map.get(revision_id, None) |
4202.3.1
by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together. |
406 |
revision_text = revision_to_str(revision) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
407 |
self.bundle.add_fulltext_record(revision_text, parents, |
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
408 |
'revision', revision_id) |
2520.4.34
by Aaron Bentley
Add signature support |
409 |
try: |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
410 |
self.bundle.add_fulltext_record( |
411 |
self.repository.get_signature_text( |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
412 |
revision_id), parents, 'signature', revision_id) |
2520.4.34
by Aaron Bentley
Add signature support |
413 |
except errors.NoSuchRevision: |
414 |
pass
|
|
415 |
||
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
416 |
@staticmethod
|
417 |
def get_base_target(revision_ids, forced_bases, repository): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
418 |
"""Determine the base and target from old-style revision ids"""
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
419 |
if len(revision_ids) == 0: |
420 |
return None, None |
|
421 |
target = revision_ids[0] |
|
422 |
base = forced_bases.get(target) |
|
423 |
if base is None: |
|
424 |
parents = repository.get_revision(target).parent_ids |
|
425 |
if len(parents) == 0: |
|
426 |
base = _mod_revision.NULL_REVISION |
|
427 |
else: |
|
428 |
base = parents[0] |
|
429 |
return base, target |
|
430 |
||
3350.6.10
by Martin Pool
VersionedFiles review cleanups |
431 |
def _add_mp_records_keys(self, repo_kind, vf, keys): |
2520.4.118
by Aaron Bentley
Add docs |
432 |
"""Add multi-parent diff records to a bundle"""
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
433 |
ordered_keys = list(multiparent.topo_iter_keys(vf, keys)) |
434 |
mpdiffs = vf.make_mpdiffs(ordered_keys) |
|
435 |
sha1s = vf.get_sha1s(ordered_keys) |
|
436 |
parent_map = vf.get_parent_map(ordered_keys) |
|
3350.8.3
by Robert Collins
VF.get_sha1s needed changing to be stackable. |
437 |
for mpdiff, item_key, in zip(mpdiffs, ordered_keys): |
438 |
sha1 = sha1s[item_key] |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
439 |
parents = [key[-1] for key in parent_map[item_key]] |
2520.4.41
by Aaron Bentley
Accelerate mpdiff generation |
440 |
text = ''.join(mpdiff.to_patch()) |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
441 |
# Infer file id records as appropriate.
|
442 |
if len(item_key) == 2: |
|
443 |
file_id = item_key[0] |
|
444 |
else: |
|
445 |
file_id = None |
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
446 |
self.bundle.add_multiparent_record(text, sha1, parents, repo_kind, |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
447 |
item_key[-1], file_id) |
2520.4.6
by Aaron Bentley
Get installation started |
448 |
|
449 |
||
2520.4.72
by Aaron Bentley
Rename format to 4alpha |
450 |
class BundleInfoV4(object): |
2520.4.6
by Aaron Bentley
Get installation started |
451 |
|
2520.4.118
by Aaron Bentley
Add docs |
452 |
"""Provide (most of) the BundleInfo interface"""
|
2520.4.6
by Aaron Bentley
Get installation started |
453 |
def __init__(self, fileobj, serializer): |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
454 |
self._fileobj = fileobj |
455 |
self._serializer = serializer |
|
456 |
self.__real_revisions = None |
|
457 |
self.__revisions = None |
|
458 |
||
459 |
def install(self, repository): |
|
460 |
return self.install_revisions(repository) |
|
461 |
||
2520.4.148
by Aaron Bentley
Updates from review |
462 |
def install_revisions(self, repository, stream_input=True): |
463 |
"""Install this bundle's revisions into the specified repository
|
|
464 |
||
465 |
:param target_repo: The repository to install into
|
|
466 |
:param stream_input: If True, will stream input rather than reading it
|
|
467 |
all into memory at once. Reading it into memory all at once is
|
|
468 |
(currently) faster.
|
|
469 |
"""
|
|
2520.4.18
by Aaron Bentley
Generate mpdiffs for inventory |
470 |
repository.lock_write() |
471 |
try: |
|
2520.4.148
by Aaron Bentley
Updates from review |
472 |
ri = RevisionInstaller(self.get_bundle_reader(stream_input), |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
473 |
self._serializer, repository) |
2520.4.18
by Aaron Bentley
Generate mpdiffs for inventory |
474 |
return ri.install() |
475 |
finally: |
|
476 |
repository.unlock() |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
477 |
|
2520.4.109
by Aaron Bentley
start work on directive cherry-picking |
478 |
def get_merge_request(self, target_repo): |
479 |
"""Provide data for performing a merge
|
|
480 |
||
481 |
Returns suggested base, suggested target, and patch verification status
|
|
482 |
"""
|
|
483 |
return None, self.target, 'inapplicable' |
|
484 |
||
2520.4.148
by Aaron Bentley
Updates from review |
485 |
def get_bundle_reader(self, stream_input=True): |
486 |
"""Return a new BundleReader for the associated bundle
|
|
487 |
||
488 |
:param stream_input: If True, the BundleReader stream input rather than
|
|
489 |
reading it all into memory at once. Reading it into memory all at
|
|
490 |
once is (currently) faster.
|
|
491 |
"""
|
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
492 |
self._fileobj.seek(0) |
2520.4.148
by Aaron Bentley
Updates from review |
493 |
return BundleReader(self._fileobj, stream_input) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
494 |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
495 |
def _get_real_revisions(self): |
496 |
if self.__real_revisions is None: |
|
497 |
self.__real_revisions = [] |
|
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
498 |
bundle_reader = self.get_bundle_reader() |
2520.4.102
by Aaron Bentley
rename parents to metadata |
499 |
for bytes, metadata, repo_kind, revision_id, file_id in \ |
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
500 |
bundle_reader.iter_records(): |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
501 |
if repo_kind == 'info': |
502 |
serializer =\ |
|
2520.4.102
by Aaron Bentley
rename parents to metadata |
503 |
self._serializer.get_source_serializer(metadata) |
2520.4.22
by Aaron Bentley
Create ContainerReader |
504 |
if repo_kind == 'revision': |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
505 |
rev = serializer.read_revision_from_string(bytes) |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
506 |
self.__real_revisions.append(rev) |
507 |
return self.__real_revisions |
|
508 |
real_revisions = property(_get_real_revisions) |
|
509 |
||
510 |
def _get_revisions(self): |
|
511 |
if self.__revisions is None: |
|
512 |
self.__revisions = [] |
|
513 |
for revision in self.real_revisions: |
|
2520.4.33
by Aaron Bentley
remove test dependencies on serialization minutia |
514 |
self.__revisions.append( |
515 |
bundle_data.RevisionInfo.from_revision(revision)) |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
516 |
return self.__revisions |
517 |
||
518 |
revisions = property(_get_revisions) |
|
519 |
||
2520.4.29
by Aaron Bentley
Reactivate some testing, fix topo_iter |
520 |
def _get_target(self): |
521 |
return self.revisions[-1].revision_id |
|
522 |
||
523 |
target = property(_get_target) |
|
524 |
||
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
525 |
|
526 |
class RevisionInstaller(object): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
527 |
"""Installs revisions into a repository"""
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
528 |
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
529 |
def __init__(self, container, serializer, repository): |
530 |
self._container = container |
|
2520.4.6
by Aaron Bentley
Get installation started |
531 |
self._serializer = serializer |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
532 |
self._repository = repository |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
533 |
self._info = None |
2520.4.99
by Aaron Bentley
Test conversion across models |
534 |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
535 |
def install(self): |
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
536 |
"""Perform the installation.
|
3943.8.1
by Marius Kruger
remove all trailing whitespace from bzr source |
537 |
|
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
538 |
Must be called with the Repository locked.
|
539 |
"""
|
|
540 |
self._repository.start_write_group() |
|
541 |
try: |
|
2856.1.2
by Robert Collins
Review feedback. |
542 |
result = self._install_in_write_group() |
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
543 |
except: |
544 |
self._repository.abort_write_group() |
|
545 |
raise
|
|
546 |
self._repository.commit_write_group() |
|
547 |
return result |
|
548 |
||
2856.1.2
by Robert Collins
Review feedback. |
549 |
def _install_in_write_group(self): |
2520.4.6
by Aaron Bentley
Get installation started |
550 |
current_file = None |
551 |
current_versionedfile = None |
|
552 |
pending_file_records = [] |
|
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
553 |
inventory_vf = None |
554 |
pending_inventory_records = [] |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
555 |
added_inv = set() |
2520.4.29
by Aaron Bentley
Reactivate some testing, fix topo_iter |
556 |
target_revision = None |
2520.4.58
by Aaron Bentley
Propogate support for metadata to iter_revisions, add storage kind |
557 |
for bytes, metadata, repo_kind, revision_id, file_id in\ |
2520.4.22
by Aaron Bentley
Create ContainerReader |
558 |
self._container.iter_records(): |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
559 |
if repo_kind == 'info': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
560 |
if self._info is not None: |
561 |
raise AssertionError() |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
562 |
self._handle_info(metadata) |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
563 |
if (pending_file_records and |
564 |
(repo_kind, file_id) != ('file', current_file)): |
|
565 |
# Flush the data for a single file - prevents memory
|
|
566 |
# spiking due to buffering all files in memory.
|
|
567 |
self._install_mp_records_keys(self._repository.texts, |
|
568 |
pending_file_records) |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
569 |
current_file = None |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
570 |
del pending_file_records[:] |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
571 |
if len(pending_inventory_records) > 0 and repo_kind != 'inventory': |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
572 |
self._install_inventory_records(pending_inventory_records) |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
573 |
pending_inventory_records = [] |
574 |
if repo_kind == 'inventory': |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
575 |
pending_inventory_records.append(((revision_id,), metadata, bytes)) |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
576 |
if repo_kind == 'revision': |
577 |
target_revision = revision_id |
|
578 |
self._install_revision(revision_id, metadata, bytes) |
|
579 |
if repo_kind == 'signature': |
|
580 |
self._install_signature(revision_id, metadata, bytes) |
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
581 |
if repo_kind == 'file': |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
582 |
current_file = file_id |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
583 |
pending_file_records.append(((file_id, revision_id), metadata, bytes)) |
584 |
self._install_mp_records_keys(self._repository.texts, pending_file_records) |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
585 |
return target_revision |
2520.4.6
by Aaron Bentley
Get installation started |
586 |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
587 |
def _handle_info(self, info): |
588 |
"""Extract data from an info record"""
|
|
589 |
self._info = info |
|
590 |
self._source_serializer = self._serializer.get_source_serializer(info) |
|
591 |
if (info['supports_rich_root'] == 0 and |
|
592 |
self._repository.supports_rich_root()): |
|
593 |
self.update_root = True |
|
594 |
else: |
|
595 |
self.update_root = False |
|
596 |
||
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
597 |
def _install_mp_records(self, versionedfile, records): |
2520.4.61
by Aaron Bentley
Do bulk insertion of records |
598 |
if len(records) == 0: |
599 |
return
|
|
600 |
d_func = multiparent.MultiParent.from_patch |
|
601 |
vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in |
|
602 |
records if r not in versionedfile] |
|
603 |
versionedfile.add_mpdiffs(vf_records) |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
604 |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
605 |
def _install_mp_records_keys(self, versionedfile, records): |
606 |
d_func = multiparent.MultiParent.from_patch |
|
607 |
vf_records = [] |
|
608 |
for key, meta, text in records: |
|
3350.6.7
by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where. |
609 |
# Adapt to tuple interface: A length two key is a file_id,
|
610 |
# revision_id pair, a length 1 key is a
|
|
611 |
# revision/signature/inventory. We need to do this because
|
|
612 |
# the metadata extraction from the bundle has not yet been updated
|
|
613 |
# to use the consistent tuple interface itself.
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
614 |
if len(key) == 2: |
615 |
prefix = key[:1] |
|
616 |
else: |
|
617 |
prefix = () |
|
618 |
parents = [prefix + (parent,) for parent in meta['parents']] |
|
619 |
vf_records.append((key, parents, meta['sha1'], d_func(text))) |
|
620 |
versionedfile.add_mpdiffs(vf_records) |
|
621 |
||
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
622 |
def _get_parent_inventory_texts(self, inventory_text_cache, |
623 |
inventory_cache, parent_ids): |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
624 |
cached_parent_texts = {} |
625 |
remaining_parent_ids = [] |
|
626 |
for parent_id in parent_ids: |
|
627 |
p_text = inventory_text_cache.get(parent_id, None) |
|
628 |
if p_text is None: |
|
629 |
remaining_parent_ids.append(parent_id) |
|
630 |
else: |
|
631 |
cached_parent_texts[parent_id] = p_text |
|
632 |
ghosts = () |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
633 |
# TODO: Use inventory_cache to grab inventories we already have in
|
634 |
# memory
|
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
635 |
if remaining_parent_ids: |
636 |
# first determine what keys are actually present in the local
|
|
637 |
# inventories object (don't use revisions as they haven't been
|
|
638 |
# installed yet.)
|
|
639 |
parent_keys = [(r,) for r in remaining_parent_ids] |
|
640 |
present_parent_map = self._repository.inventories.get_parent_map( |
|
641 |
parent_keys) |
|
642 |
present_parent_ids = [] |
|
643 |
ghosts = set() |
|
644 |
for p_id in remaining_parent_ids: |
|
645 |
if (p_id,) in present_parent_map: |
|
646 |
present_parent_ids.append(p_id) |
|
647 |
else: |
|
648 |
ghosts.add(p_id) |
|
649 |
to_string = self._source_serializer.write_inventory_to_string |
|
650 |
for parent_inv in self._repository.iter_inventories( |
|
651 |
present_parent_ids): |
|
652 |
p_text = to_string(parent_inv) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
653 |
inventory_cache[parent_inv.revision_id] = parent_inv |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
654 |
cached_parent_texts[parent_inv.revision_id] = p_text |
655 |
inventory_text_cache[parent_inv.revision_id] = p_text |
|
656 |
||
657 |
parent_texts = [cached_parent_texts[parent_id] |
|
658 |
for parent_id in parent_ids |
|
659 |
if parent_id not in ghosts] |
|
660 |
return parent_texts |
|
661 |
||
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
662 |
def _install_inventory_records(self, records): |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
663 |
if (self._info['serializer'] == self._repository._serializer.format_num |
664 |
and self._repository._serializer.support_altered_by_hack): |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
665 |
return self._install_mp_records_keys(self._repository.inventories, |
666 |
records) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
667 |
# Use a 10MB text cache, since these are string xml inventories. Note
|
668 |
# that 10MB is fairly small for large projects (a single inventory can
|
|
669 |
# be >5MB). Another possibility is to cache 10-20 inventory texts
|
|
670 |
# instead
|
|
671 |
inventory_text_cache = lru_cache.LRUSizeCache(10*1024*1024) |
|
4543.2.21
by John Arbash Meinel
A few more tiny tweaks to comments, etc. |
672 |
# Also cache the in-memory representation. This allows us to create
|
673 |
# inventory deltas to apply rather than calling add_inventory from
|
|
674 |
# scratch each time.
|
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
675 |
inventory_cache = lru_cache.LRUCache(10) |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
676 |
pb = ui.ui_factory.nested_progress_bar() |
677 |
try: |
|
678 |
num_records = len(records) |
|
679 |
for idx, (key, metadata, bytes) in enumerate(records): |
|
680 |
pb.update('installing inventory', idx, num_records) |
|
681 |
revision_id = key[-1] |
|
682 |
parent_ids = metadata['parents'] |
|
683 |
# Note: This assumes the local ghosts are identical to the
|
|
684 |
# ghosts in the source, as the Bundle serialization
|
|
685 |
# format doesn't record ghosts.
|
|
686 |
p_texts = self._get_parent_inventory_texts(inventory_text_cache, |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
687 |
inventory_cache, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
688 |
parent_ids) |
689 |
# Why does to_lines() take strings as the source, it seems that
|
|
690 |
# it would have to cast to a list of lines, which we get back
|
|
691 |
# as lines and then cast back to a string.
|
|
692 |
target_lines = multiparent.MultiParent.from_patch(bytes |
|
693 |
).to_lines(p_texts) |
|
694 |
inv_text = ''.join(target_lines) |
|
695 |
del target_lines |
|
696 |
sha1 = osutils.sha_string(inv_text) |
|
697 |
if sha1 != metadata['sha1']: |
|
698 |
raise errors.BadBundle("Can't convert to target format") |
|
699 |
# Add this to the cache so we don't have to extract it again.
|
|
700 |
inventory_text_cache[revision_id] = inv_text |
|
701 |
target_inv = self._source_serializer.read_inventory_from_string( |
|
702 |
inv_text) |
|
703 |
self._handle_root(target_inv, parent_ids) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
704 |
parent_inv = None |
705 |
if parent_ids: |
|
706 |
parent_inv = inventory_cache.get(parent_ids[0], None) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
707 |
try: |
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
708 |
if parent_inv is None: |
709 |
self._repository.add_inventory(revision_id, target_inv, |
|
710 |
parent_ids) |
|
711 |
else: |
|
712 |
delta = target_inv._make_delta(parent_inv) |
|
713 |
self._repository.add_inventory_by_delta(parent_ids[0], |
|
714 |
delta, revision_id, parent_ids) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
715 |
except errors.UnsupportedInventoryKind: |
716 |
raise errors.IncompatibleRevision(repr(self._repository)) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
717 |
inventory_cache[revision_id] = target_inv |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
718 |
finally: |
719 |
pb.finished() |
|
2520.4.99
by Aaron Bentley
Test conversion across models |
720 |
|
721 |
def _handle_root(self, target_inv, parent_ids): |
|
722 |
revision_id = target_inv.revision_id |
|
723 |
if self.update_root: |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
724 |
text_key = (target_inv.root.file_id, revision_id) |
725 |
parent_keys = [(target_inv.root.file_id, parent) for |
|
726 |
parent in parent_ids] |
|
727 |
self._repository.texts.add_lines(text_key, parent_keys, []) |
|
2520.4.99
by Aaron Bentley
Test conversion across models |
728 |
elif not self._repository.supports_rich_root(): |
729 |
if target_inv.root.revision != revision_id: |
|
730 |
raise errors.IncompatibleRevision(repr(self._repository)) |
|
731 |
||
2520.4.59
by Aaron Bentley
Push metadata down the stack |
732 |
def _install_revision(self, revision_id, metadata, text): |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
733 |
if self._repository.has_revision(revision_id): |
734 |
return
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
735 |
revision = self._source_serializer.read_revision_from_string(text) |
736 |
self._repository.add_revision(revision.revision_id, revision) |
|
2520.4.34
by Aaron Bentley
Add signature support |
737 |
|
2520.4.59
by Aaron Bentley
Push metadata down the stack |
738 |
def _install_signature(self, revision_id, metadata, text): |
2520.4.100
by Aaron Bentley
Fix repeat signature installs |
739 |
transaction = self._repository.get_transaction() |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
740 |
if self._repository.has_signature_for_revision_id(revision_id): |
2520.4.100
by Aaron Bentley
Fix repeat signature installs |
741 |
return
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
742 |
self._repository.add_signature_text(revision_id, text) |