2520.4.85
by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!) |
1 |
# Copyright (C) 2007 Canonical Ltd
|
2 |
#
|
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
#
|
|
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
#
|
|
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
4183.7.1
by Sabin Iacob
update FSF mailing address |
15 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
2520.4.85
by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!) |
16 |
|
2520.4.20
by Aaron Bentley
Compress and base64-encode bundle contents |
17 |
from cStringIO import StringIO |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
18 |
import bz2 |
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
19 |
import re |
2520.4.20
by Aaron Bentley
Compress and base64-encode bundle contents |
20 |
|
2520.4.13
by Aaron Bentley
Use real container implementation |
21 |
from bzrlib import ( |
2520.4.40
by Aaron Bentley
Add human-readable diff to bundles |
22 |
diff, |
2520.4.34
by Aaron Bentley
Add signature support |
23 |
errors, |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
24 |
iterablefile, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
25 |
lru_cache, |
2520.4.13
by Aaron Bentley
Use real container implementation |
26 |
multiparent, |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
27 |
osutils, |
2520.4.13
by Aaron Bentley
Use real container implementation |
28 |
pack, |
2520.4.40
by Aaron Bentley
Add human-readable diff to bundles |
29 |
revision as _mod_revision, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
30 |
serializer, |
2520.4.45
by Aaron Bentley
Handle inconsistencies in last-modified-revision between vf and inventory |
31 |
trace, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
32 |
ui, |
2520.4.13
by Aaron Bentley
Use real container implementation |
33 |
)
|
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
34 |
from bzrlib.bundle import bundle_data, serializer as bundle_serializer |
2694.5.1
by Alexander Belchenko
pyrex bencode (without benchmarks) |
35 |
from bzrlib import bencode |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
36 |
|
2520.4.4
by Aaron Bentley
Get basis support for a new bundle format in place |
37 |
|
2520.4.25
by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader |
38 |
class BundleWriter(object): |
2520.4.118
by Aaron Bentley
Add docs |
39 |
"""Writer for bundle-format files.
|
40 |
||
41 |
This serves roughly the same purpose as ContainerReader, but acts as a
|
|
42 |
layer on top of it.
|
|
43 |
||
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
44 |
Provides ways of writing the specific record types supported this bundle
|
2520.4.118
by Aaron Bentley
Add docs |
45 |
format.
|
46 |
"""
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
47 |
|
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
48 |
def __init__(self, fileobj): |
2520.4.27
by Aaron Bentley
Use less memory when writing bzip-encoded files |
49 |
self._container = pack.ContainerWriter(self._write_encoded) |
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
50 |
self._fileobj = fileobj |
2520.4.27
by Aaron Bentley
Use less memory when writing bzip-encoded files |
51 |
self._compressor = bz2.BZ2Compressor() |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
52 |
|
2520.4.118
by Aaron Bentley
Add docs |
53 |
def _write_encoded(self, bytes): |
54 |
"""Write bzip2-encoded bytes to the file"""
|
|
55 |
self._fileobj.write(self._compressor.compress(bytes)) |
|
56 |
||
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
57 |
def begin(self): |
2520.4.118
by Aaron Bentley
Add docs |
58 |
"""Start writing the bundle"""
|
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
59 |
self._fileobj.write(bundle_serializer._get_bundle_header( |
60 |
bundle_serializer.v4_string)) |
|
2520.4.24
by Aaron Bentley
Move heading writing above container beginning |
61 |
self._fileobj.write('#\n') |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
62 |
self._container.begin() |
63 |
||
64 |
def end(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
65 |
"""Finish writing the bundle"""
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
66 |
self._container.end() |
2520.4.76
by Aaron Bentley
Move base64-encoding into merge directives |
67 |
self._fileobj.write(self._compressor.flush()) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
68 |
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
69 |
def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind, |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
70 |
revision_id, file_id): |
2520.4.118
by Aaron Bentley
Add docs |
71 |
"""Add a record for a multi-parent diff
|
72 |
||
73 |
:mp_bytes: A multi-parent diff, as a bytestring
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
74 |
:sha1: The sha1 hash of the fulltext
|
2520.4.118
by Aaron Bentley
Add docs |
75 |
:parents: a list of revision-ids of the parents
|
76 |
:repo_kind: The kind of object in the repository. May be 'file' or
|
|
77 |
'inventory'
|
|
78 |
:revision_id: The revision id of the mpdiff being added.
|
|
79 |
:file_id: The file-id of the file, or None for inventories.
|
|
80 |
"""
|
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
81 |
metadata = {'parents': parents, |
82 |
'storage_kind': 'mpdiff', |
|
83 |
'sha1': sha1} |
|
84 |
self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id) |
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
85 |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
86 |
def add_fulltext_record(self, bytes, parents, repo_kind, revision_id): |
2520.4.118
by Aaron Bentley
Add docs |
87 |
"""Add a record for a fulltext
|
88 |
||
89 |
:bytes: The fulltext, as a bytestring
|
|
90 |
:parents: a list of revision-ids of the parents
|
|
91 |
:repo_kind: The kind of object in the repository. May be 'revision' or
|
|
92 |
'signature'
|
|
93 |
:revision_id: The revision id of the fulltext being added.
|
|
94 |
"""
|
|
95 |
metadata = {'parents': parents, |
|
2520.5.3
by Aaron Bentley
fix sha1 in bundle format 4 |
96 |
'storage_kind': 'mpdiff'} |
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
97 |
self._add_record(bytes, {'parents': parents, |
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
98 |
'storage_kind': 'fulltext'}, repo_kind, revision_id, None) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
99 |
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
100 |
def add_info_record(self, **kwargs): |
2520.4.118
by Aaron Bentley
Add docs |
101 |
"""Add an info record to the bundle
|
102 |
||
103 |
Any parameters may be supplied, except 'self' and 'storage_kind'.
|
|
104 |
Values must be lists, strings, integers, dicts, or a combination.
|
|
105 |
"""
|
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
106 |
kwargs['storage_kind'] = 'header' |
107 |
self._add_record(None, kwargs, 'info', None, None) |
|
108 |
||
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
109 |
@staticmethod
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
110 |
def encode_name(content_kind, revision_id, file_id=None): |
2520.4.118
by Aaron Bentley
Add docs |
111 |
"""Encode semantic ids as a container name"""
|
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
112 |
if content_kind not in ('revision', 'file', 'inventory', 'signature', |
113 |
'info'): |
|
114 |
raise ValueError(content_kind) |
|
2520.4.118
by Aaron Bentley
Add docs |
115 |
if content_kind == 'file': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
116 |
if file_id is None: |
117 |
raise AssertionError() |
|
2520.4.118
by Aaron Bentley
Add docs |
118 |
else: |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
119 |
if file_id is not None: |
120 |
raise AssertionError() |
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
121 |
if content_kind == 'info': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
122 |
if revision_id is not None: |
123 |
raise AssertionError() |
|
124 |
elif revision_id is None: |
|
125 |
raise AssertionError() |
|
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
126 |
names = [n.replace('/', '//') for n in |
127 |
(content_kind, revision_id, file_id) if n is not None] |
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
128 |
return '/'.join(names) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
129 |
|
2520.4.56
by Aaron Bentley
Begin adding support for arbitrary metadata |
130 |
def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id): |
2520.4.118
by Aaron Bentley
Add docs |
131 |
"""Add a bundle record to the container.
|
132 |
||
133 |
Most bundle records are recorded as header/body pairs, with the
|
|
134 |
body being nameless. Records with storage_kind 'header' have no
|
|
135 |
body.
|
|
136 |
"""
|
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
137 |
name = self.encode_name(repo_kind, revision_id, file_id) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
138 |
encoded_metadata = bencode.bencode(metadata) |
2682.1.1
by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings |
139 |
self._container.add_bytes_record(encoded_metadata, [(name, )]) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
140 |
if metadata['storage_kind'] != 'header': |
141 |
self._container.add_bytes_record(bytes, []) |
|
2520.4.13
by Aaron Bentley
Use real container implementation |
142 |
|
2520.4.7
by Aaron Bentley
Fix patch deserialization |
143 |
|
2520.4.25
by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader |
144 |
class BundleReader(object): |
2520.4.118
by Aaron Bentley
Add docs |
145 |
"""Reader for bundle-format files.
|
146 |
||
147 |
This serves roughly the same purpose as ContainerReader, but acts as a
|
|
148 |
layer on top of it, providing metadata, a semantic name, and a record
|
|
149 |
body
|
|
150 |
"""
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
151 |
|
4543.2.14
by John Arbash Meinel
Clarify some comments, fix up a debugging change. |
152 |
def __init__(self, fileobj, stream_input=True): |
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
153 |
"""Constructor
|
154 |
||
155 |
:param fileobj: a file containing a bzip-encoded container
|
|
2520.4.148
by Aaron Bentley
Updates from review |
156 |
:param stream_input: If True, the BundleReader stream input rather than
|
157 |
reading it all into memory at once. Reading it into memory all at
|
|
158 |
once is (currently) faster.
|
|
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
159 |
"""
|
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
160 |
line = fileobj.readline() |
161 |
if line != '\n': |
|
162 |
fileobj.readline() |
|
2520.4.40
by Aaron Bentley
Add human-readable diff to bundles |
163 |
self.patch_lines = [] |
2520.4.148
by Aaron Bentley
Updates from review |
164 |
if stream_input: |
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
165 |
source_file = iterablefile.IterableFile(self.iter_decode(fileobj)) |
166 |
else: |
|
167 |
source_file = StringIO(bz2.decompress(fileobj.read())) |
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
168 |
self._container_file = source_file |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
169 |
|
170 |
@staticmethod
|
|
171 |
def iter_decode(fileobj): |
|
2520.4.118
by Aaron Bentley
Add docs |
172 |
"""Iterate through decoded fragments of the file"""
|
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
173 |
decompressor = bz2.BZ2Decompressor() |
174 |
for line in fileobj: |
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
175 |
try: |
176 |
yield decompressor.decompress(line) |
|
177 |
except EOFError: |
|
178 |
return
|
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
179 |
|
180 |
@staticmethod
|
|
181 |
def decode_name(name): |
|
2520.4.118
by Aaron Bentley
Add docs |
182 |
"""Decode a name from its container form into a semantic form
|
183 |
||
184 |
:retval: content_kind, revision_id, file_id
|
|
185 |
"""
|
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
186 |
segments = re.split('(//?)', name) |
187 |
names = [''] |
|
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
188 |
for segment in segments: |
189 |
if segment == '//': |
|
190 |
names[-1] += '/' |
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
191 |
elif segment == '/': |
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
192 |
names.append('') |
193 |
else: |
|
194 |
names[-1] += segment |
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
195 |
content_kind = names[0] |
2520.4.95
by Aaron Bentley
Add support for header/info records |
196 |
revision_id = None |
197 |
file_id = None |
|
198 |
if len(names) > 1: |
|
199 |
revision_id = names[1] |
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
200 |
if len(names) > 2: |
201 |
file_id = names[2] |
|
202 |
return content_kind, revision_id, file_id |
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
203 |
|
204 |
def iter_records(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
205 |
"""Iterate through bundle records
|
206 |
||
207 |
:return: a generator of (bytes, metadata, content_kind, revision_id,
|
|
208 |
file_id)
|
|
209 |
"""
|
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
210 |
iterator = pack.iter_records_from_file(self._container_file) |
211 |
for names, bytes in iterator: |
|
2520.4.131
by Aaron Bentley
Raise BadBundle for records with wrong number of names |
212 |
if len(names) != 1: |
213 |
raise errors.BadBundle('Record has %d names instead of 1' |
|
214 |
% len(names)) |
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
215 |
metadata = bencode.bdecode(bytes) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
216 |
if metadata['storage_kind'] == 'header': |
217 |
bytes = None |
|
218 |
else: |
|
219 |
_unused, bytes = iterator.next() |
|
2682.1.1
by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings |
220 |
yield (bytes, metadata) + self.decode_name(names[0][0]) |
2520.4.22
by Aaron Bentley
Create ContainerReader |
221 |
|
222 |
||
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
223 |
class BundleSerializerV4(bundle_serializer.BundleSerializer): |
2520.4.118
by Aaron Bentley
Add docs |
224 |
"""Implement the high-level bundle interface"""
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
225 |
|
2520.4.4
by Aaron Bentley
Get basis support for a new bundle format in place |
226 |
def write(self, repository, revision_ids, forced_bases, fileobj): |
2520.4.118
by Aaron Bentley
Add docs |
227 |
"""Write a bundle to a file-like object
|
228 |
||
229 |
For backwards-compatibility only
|
|
230 |
"""
|
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
231 |
write_op = BundleWriteOperation.from_old_args(repository, revision_ids, |
232 |
forced_bases, fileobj) |
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
233 |
return write_op.do_write() |
234 |
||
235 |
def write_bundle(self, repository, target, base, fileobj): |
|
2520.4.118
by Aaron Bentley
Add docs |
236 |
"""Write a bundle to a file object
|
237 |
||
238 |
:param repository: The repository to retrieve revision data from
|
|
239 |
:param target: The head revision to include ancestors of
|
|
240 |
:param base: The ancestor of the target to stop including acestors
|
|
241 |
at.
|
|
242 |
:param fileobj: The file-like object to write to
|
|
243 |
"""
|
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
244 |
write_op = BundleWriteOperation(base, target, repository, fileobj) |
245 |
return write_op.do_write() |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
246 |
|
247 |
def read(self, file): |
|
2520.4.118
by Aaron Bentley
Add docs |
248 |
"""return a reader object for a given file"""
|
2520.4.72
by Aaron Bentley
Rename format to 4alpha |
249 |
bundle = BundleInfoV4(file, self) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
250 |
return bundle |
251 |
||
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
252 |
@staticmethod
|
253 |
def get_source_serializer(info): |
|
2520.4.118
by Aaron Bentley
Add docs |
254 |
"""Retrieve the serializer for a given info object"""
|
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
255 |
return serializer.format_registry.get(info['serializer']) |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
256 |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
257 |
|
258 |
class BundleWriteOperation(object): |
|
2520.4.118
by Aaron Bentley
Add docs |
259 |
"""Perform the operation of writing revisions to a bundle"""
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
260 |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
261 |
@classmethod
|
262 |
def from_old_args(cls, repository, revision_ids, forced_bases, fileobj): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
263 |
"""Create a BundleWriteOperation from old-style arguments"""
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
264 |
base, target = cls.get_base_target(revision_ids, forced_bases, |
265 |
repository) |
|
266 |
return BundleWriteOperation(base, target, repository, fileobj, |
|
267 |
revision_ids) |
|
268 |
||
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
269 |
def __init__(self, base, target, repository, fileobj, revision_ids=None): |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
270 |
self.base = base |
271 |
self.target = target |
|
272 |
self.repository = repository |
|
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
273 |
bundle = BundleWriter(fileobj) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
274 |
self.bundle = bundle |
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
275 |
if revision_ids is not None: |
276 |
self.revision_ids = revision_ids |
|
277 |
else: |
|
4154.1.1
by Ian Clatworthy
make send use graph.find_difference() instead of walking all of history twice |
278 |
graph = repository.get_graph() |
4154.1.3
by Ian Clatworthy
strip ghosts so test_bundle_with_ghosts works again |
279 |
revision_ids = graph.find_unique_ancestors(target, [base]) |
280 |
# Strip ghosts
|
|
281 |
parents = graph.get_parent_map(revision_ids) |
|
282 |
self.revision_ids = [r for r in revision_ids if r in parents] |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
283 |
self.revision_keys = set([(revid,) for revid in self.revision_ids]) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
284 |
|
285 |
def do_write(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
286 |
"""Write all data to the bundle"""
|
3794.4.1
by Aaron Bentley
Note the number of revisions being bundled (abentley) |
287 |
trace.note('Bundling %d revision(s).', len(self.revision_ids)) |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
288 |
self.repository.lock_read() |
289 |
try: |
|
290 |
self.bundle.begin() |
|
291 |
self.write_info() |
|
292 |
self.write_files() |
|
293 |
self.write_revisions() |
|
294 |
self.bundle.end() |
|
295 |
finally: |
|
296 |
self.repository.unlock() |
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
297 |
return self.revision_ids |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
298 |
|
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
299 |
def write_info(self): |
2520.4.118
by Aaron Bentley
Add docs |
300 |
"""Write format info"""
|
2520.4.113
by Aaron Bentley
Avoid peeking at Repository._serializer |
301 |
serializer_format = self.repository.get_serializer_format() |
2520.4.99
by Aaron Bentley
Test conversion across models |
302 |
supports_rich_root = {True: 1, False: 0}[ |
303 |
self.repository.supports_rich_root()] |
|
2520.4.113
by Aaron Bentley
Avoid peeking at Repository._serializer |
304 |
self.bundle.add_info_record(serializer=serializer_format, |
2520.4.99
by Aaron Bentley
Test conversion across models |
305 |
supports_rich_root=supports_rich_root) |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
306 |
|
2520.4.51
by Aaron Bentley
Split iteration through file revisions into a method, so we can vary it |
307 |
def write_files(self): |
2520.4.118
by Aaron Bentley
Add docs |
308 |
"""Write bundle records for all revisions of all files"""
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
309 |
text_keys = [] |
3350.6.7
by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where. |
310 |
altered_fileids = self.repository.fileids_altered_by_revision_ids( |
311 |
self.revision_ids) |
|
312 |
for file_id, revision_ids in altered_fileids.iteritems(): |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
313 |
for revision_id in revision_ids: |
314 |
text_keys.append((file_id, revision_id)) |
|
3350.6.10
by Martin Pool
VersionedFiles review cleanups |
315 |
self._add_mp_records_keys('file', self.repository.texts, text_keys) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
316 |
|
317 |
def write_revisions(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
318 |
"""Write bundle records for all revisions and signatures"""
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
319 |
inv_vf = self.repository.inventories |
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
320 |
topological_order = [key[-1] for key in multiparent.topo_iter_keys( |
321 |
inv_vf, self.revision_keys)] |
|
322 |
revision_order = topological_order |
|
2520.4.75
by Aaron Bentley
Fix traceback on empty bundles. |
323 |
if self.target is not None and self.target in self.revision_ids: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
324 |
# Make sure the target revision is always the last entry
|
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
325 |
revision_order = list(topological_order) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
326 |
revision_order.remove(self.target) |
327 |
revision_order.append(self.target) |
|
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
328 |
if self.repository._serializer.support_altered_by_hack: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
329 |
# Repositories that support_altered_by_hack means that
|
330 |
# inventories.make_mpdiffs() contains all the data about the tree
|
|
331 |
# shape. Formats without support_altered_by_hack require
|
|
332 |
# chk_bytes/etc, so we use a different code path.
|
|
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
333 |
self._add_mp_records_keys('inventory', inv_vf, |
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
334 |
[(revid,) for revid in topological_order]) |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
335 |
else: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
336 |
# Inventories should always be added in pure-topological order, so
|
337 |
# that we can apply the mpdiff for the child to the parent texts.
|
|
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
338 |
self._add_inventory_mpdiffs_from_serializer(topological_order) |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
339 |
self._add_revision_texts(revision_order) |
340 |
||
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
341 |
def _add_inventory_mpdiffs_from_serializer(self, revision_order): |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
342 |
"""Generate mpdiffs by serializing inventories.
|
343 |
||
344 |
The current repository only has part of the tree shape information in
|
|
345 |
the 'inventories' vf. So we use serializer.write_inventory_to_string to
|
|
346 |
get a 'full' representation of the tree shape, and then generate
|
|
347 |
mpdiffs on that data stream. This stream can then be reconstructed on
|
|
348 |
the other side.
|
|
349 |
"""
|
|
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
350 |
inventory_key_order = [(r,) for r in revision_order] |
351 |
parent_map = self.repository.inventories.get_parent_map( |
|
352 |
inventory_key_order) |
|
4543.2.5
by John Arbash Meinel
Fix issues with keys/ids and ghost handling. |
353 |
missing_keys = set(inventory_key_order).difference(parent_map) |
354 |
if missing_keys: |
|
355 |
raise errors.RevisionNotPresent(list(missing_keys)[0], |
|
356 |
self.repository.inventories) |
|
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
357 |
inv_to_str = self.repository._serializer.write_inventory_to_string |
358 |
# Make sure that we grab the parent texts first
|
|
359 |
just_parents = set() |
|
360 |
map(just_parents.update, parent_map.itervalues()) |
|
4543.2.5
by John Arbash Meinel
Fix issues with keys/ids and ghost handling. |
361 |
just_parents.difference_update(parent_map) |
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
362 |
# Ignore ghost parents
|
363 |
present_parents = self.repository.inventories.get_parent_map( |
|
364 |
just_parents) |
|
4543.2.5
by John Arbash Meinel
Fix issues with keys/ids and ghost handling. |
365 |
ghost_keys = just_parents.difference(present_parents) |
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
366 |
needed_inventories = list(present_parents) + inventory_key_order |
367 |
needed_inventories = [k[-1] for k in needed_inventories] |
|
368 |
all_lines = {} |
|
369 |
for inv in self.repository.iter_inventories(needed_inventories): |
|
370 |
revision_id = inv.revision_id |
|
371 |
key = (revision_id,) |
|
372 |
as_bytes = inv_to_str(inv) |
|
373 |
# The sha1 is validated as the xml/textual form, not as the
|
|
374 |
# form-in-the-repository
|
|
375 |
sha1 = osutils.sha_string(as_bytes) |
|
376 |
as_lines = osutils.split_lines(as_bytes) |
|
4543.2.5
by John Arbash Meinel
Fix issues with keys/ids and ghost handling. |
377 |
del as_bytes |
378 |
all_lines[key] = as_lines |
|
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
379 |
if key in just_parents: |
380 |
# We don't transmit those entries
|
|
381 |
continue
|
|
382 |
# Create an mpdiff for this text, and add it to the output
|
|
383 |
parent_keys = parent_map[key] |
|
4543.2.5
by John Arbash Meinel
Fix issues with keys/ids and ghost handling. |
384 |
# See the comment in VF.make_mpdiffs about how this effects
|
385 |
# ordering when there are ghosts present. I think we have a latent
|
|
386 |
# bug
|
|
387 |
parent_lines = [all_lines[p_key] for p_key in parent_keys |
|
388 |
if p_key not in ghost_keys] |
|
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
389 |
diff = multiparent.MultiParent.from_lines( |
390 |
as_lines, parent_lines) |
|
391 |
text = ''.join(diff.to_patch()) |
|
4543.2.5
by John Arbash Meinel
Fix issues with keys/ids and ghost handling. |
392 |
parent_ids = [k[-1] for k in parent_keys] |
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
393 |
self.bundle.add_multiparent_record(text, sha1, parent_ids, |
394 |
'inventory', revision_id, None) |
|
395 |
||
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
396 |
def _add_revision_texts(self, revision_order): |
3099.3.5
by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly. |
397 |
parent_map = self.repository.get_parent_map(revision_order) |
4202.3.1
by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together. |
398 |
revision_to_str = self.repository._serializer.write_revision_to_string |
399 |
revisions = self.repository.get_revisions(revision_order) |
|
400 |
for revision in revisions: |
|
401 |
revision_id = revision.revision_id |
|
3099.3.5
by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly. |
402 |
parents = parent_map.get(revision_id, None) |
4202.3.1
by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together. |
403 |
revision_text = revision_to_str(revision) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
404 |
self.bundle.add_fulltext_record(revision_text, parents, |
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
405 |
'revision', revision_id) |
2520.4.34
by Aaron Bentley
Add signature support |
406 |
try: |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
407 |
self.bundle.add_fulltext_record( |
408 |
self.repository.get_signature_text( |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
409 |
revision_id), parents, 'signature', revision_id) |
2520.4.34
by Aaron Bentley
Add signature support |
410 |
except errors.NoSuchRevision: |
411 |
pass
|
|
412 |
||
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
413 |
@staticmethod
|
414 |
def get_base_target(revision_ids, forced_bases, repository): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
415 |
"""Determine the base and target from old-style revision ids"""
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
416 |
if len(revision_ids) == 0: |
417 |
return None, None |
|
418 |
target = revision_ids[0] |
|
419 |
base = forced_bases.get(target) |
|
420 |
if base is None: |
|
421 |
parents = repository.get_revision(target).parent_ids |
|
422 |
if len(parents) == 0: |
|
423 |
base = _mod_revision.NULL_REVISION |
|
424 |
else: |
|
425 |
base = parents[0] |
|
426 |
return base, target |
|
427 |
||
3350.6.10
by Martin Pool
VersionedFiles review cleanups |
428 |
def _add_mp_records_keys(self, repo_kind, vf, keys): |
2520.4.118
by Aaron Bentley
Add docs |
429 |
"""Add multi-parent diff records to a bundle"""
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
430 |
ordered_keys = list(multiparent.topo_iter_keys(vf, keys)) |
431 |
mpdiffs = vf.make_mpdiffs(ordered_keys) |
|
432 |
sha1s = vf.get_sha1s(ordered_keys) |
|
433 |
parent_map = vf.get_parent_map(ordered_keys) |
|
3350.8.3
by Robert Collins
VF.get_sha1s needed changing to be stackable. |
434 |
for mpdiff, item_key, in zip(mpdiffs, ordered_keys): |
435 |
sha1 = sha1s[item_key] |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
436 |
parents = [key[-1] for key in parent_map[item_key]] |
2520.4.41
by Aaron Bentley
Accelerate mpdiff generation |
437 |
text = ''.join(mpdiff.to_patch()) |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
438 |
# Infer file id records as appropriate.
|
439 |
if len(item_key) == 2: |
|
440 |
file_id = item_key[0] |
|
441 |
else: |
|
442 |
file_id = None |
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
443 |
self.bundle.add_multiparent_record(text, sha1, parents, repo_kind, |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
444 |
item_key[-1], file_id) |
2520.4.6
by Aaron Bentley
Get installation started |
445 |
|
446 |
||
2520.4.72
by Aaron Bentley
Rename format to 4alpha |
447 |
class BundleInfoV4(object): |
2520.4.6
by Aaron Bentley
Get installation started |
448 |
|
2520.4.118
by Aaron Bentley
Add docs |
449 |
"""Provide (most of) the BundleInfo interface"""
|
2520.4.6
by Aaron Bentley
Get installation started |
450 |
def __init__(self, fileobj, serializer): |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
451 |
self._fileobj = fileobj |
452 |
self._serializer = serializer |
|
453 |
self.__real_revisions = None |
|
454 |
self.__revisions = None |
|
455 |
||
456 |
def install(self, repository): |
|
457 |
return self.install_revisions(repository) |
|
458 |
||
2520.4.148
by Aaron Bentley
Updates from review |
459 |
def install_revisions(self, repository, stream_input=True): |
460 |
"""Install this bundle's revisions into the specified repository
|
|
461 |
||
462 |
:param target_repo: The repository to install into
|
|
463 |
:param stream_input: If True, will stream input rather than reading it
|
|
464 |
all into memory at once. Reading it into memory all at once is
|
|
465 |
(currently) faster.
|
|
466 |
"""
|
|
2520.4.18
by Aaron Bentley
Generate mpdiffs for inventory |
467 |
repository.lock_write() |
468 |
try: |
|
2520.4.148
by Aaron Bentley
Updates from review |
469 |
ri = RevisionInstaller(self.get_bundle_reader(stream_input), |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
470 |
self._serializer, repository) |
2520.4.18
by Aaron Bentley
Generate mpdiffs for inventory |
471 |
return ri.install() |
472 |
finally: |
|
473 |
repository.unlock() |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
474 |
|
2520.4.109
by Aaron Bentley
start work on directive cherry-picking |
475 |
def get_merge_request(self, target_repo): |
476 |
"""Provide data for performing a merge
|
|
477 |
||
478 |
Returns suggested base, suggested target, and patch verification status
|
|
479 |
"""
|
|
480 |
return None, self.target, 'inapplicable' |
|
481 |
||
2520.4.148
by Aaron Bentley
Updates from review |
482 |
def get_bundle_reader(self, stream_input=True): |
483 |
"""Return a new BundleReader for the associated bundle
|
|
484 |
||
485 |
:param stream_input: If True, the BundleReader stream input rather than
|
|
486 |
reading it all into memory at once. Reading it into memory all at
|
|
487 |
once is (currently) faster.
|
|
488 |
"""
|
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
489 |
self._fileobj.seek(0) |
2520.4.148
by Aaron Bentley
Updates from review |
490 |
return BundleReader(self._fileobj, stream_input) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
491 |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
492 |
def _get_real_revisions(self): |
493 |
if self.__real_revisions is None: |
|
494 |
self.__real_revisions = [] |
|
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
495 |
bundle_reader = self.get_bundle_reader() |
2520.4.102
by Aaron Bentley
rename parents to metadata |
496 |
for bytes, metadata, repo_kind, revision_id, file_id in \ |
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
497 |
bundle_reader.iter_records(): |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
498 |
if repo_kind == 'info': |
499 |
serializer =\ |
|
2520.4.102
by Aaron Bentley
rename parents to metadata |
500 |
self._serializer.get_source_serializer(metadata) |
2520.4.22
by Aaron Bentley
Create ContainerReader |
501 |
if repo_kind == 'revision': |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
502 |
rev = serializer.read_revision_from_string(bytes) |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
503 |
self.__real_revisions.append(rev) |
504 |
return self.__real_revisions |
|
505 |
real_revisions = property(_get_real_revisions) |
|
506 |
||
507 |
def _get_revisions(self): |
|
508 |
if self.__revisions is None: |
|
509 |
self.__revisions = [] |
|
510 |
for revision in self.real_revisions: |
|
2520.4.33
by Aaron Bentley
remove test dependencies on serialization minutia |
511 |
self.__revisions.append( |
512 |
bundle_data.RevisionInfo.from_revision(revision)) |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
513 |
return self.__revisions |
514 |
||
515 |
revisions = property(_get_revisions) |
|
516 |
||
2520.4.29
by Aaron Bentley
Reactivate some testing, fix topo_iter |
517 |
def _get_target(self): |
518 |
return self.revisions[-1].revision_id |
|
519 |
||
520 |
target = property(_get_target) |
|
521 |
||
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
522 |
|
523 |
class RevisionInstaller(object): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
524 |
"""Installs revisions into a repository"""
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
525 |
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
526 |
def __init__(self, container, serializer, repository): |
527 |
self._container = container |
|
2520.4.6
by Aaron Bentley
Get installation started |
528 |
self._serializer = serializer |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
529 |
self._repository = repository |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
530 |
self._info = None |
2520.4.99
by Aaron Bentley
Test conversion across models |
531 |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
532 |
def install(self): |
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
533 |
"""Perform the installation.
|
3943.8.1
by Marius Kruger
remove all trailing whitespace from bzr source |
534 |
|
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
535 |
Must be called with the Repository locked.
|
536 |
"""
|
|
537 |
self._repository.start_write_group() |
|
538 |
try: |
|
2856.1.2
by Robert Collins
Review feedback. |
539 |
result = self._install_in_write_group() |
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
540 |
except: |
541 |
self._repository.abort_write_group() |
|
542 |
raise
|
|
543 |
self._repository.commit_write_group() |
|
544 |
return result |
|
545 |
||
2856.1.2
by Robert Collins
Review feedback. |
546 |
def _install_in_write_group(self): |
2520.4.6
by Aaron Bentley
Get installation started |
547 |
current_file = None |
548 |
current_versionedfile = None |
|
549 |
pending_file_records = [] |
|
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
550 |
inventory_vf = None |
551 |
pending_inventory_records = [] |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
552 |
added_inv = set() |
2520.4.29
by Aaron Bentley
Reactivate some testing, fix topo_iter |
553 |
target_revision = None |
2520.4.58
by Aaron Bentley
Propogate support for metadata to iter_revisions, add storage kind |
554 |
for bytes, metadata, repo_kind, revision_id, file_id in\ |
2520.4.22
by Aaron Bentley
Create ContainerReader |
555 |
self._container.iter_records(): |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
556 |
if repo_kind == 'info': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
557 |
if self._info is not None: |
558 |
raise AssertionError() |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
559 |
self._handle_info(metadata) |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
560 |
if (pending_file_records and |
561 |
(repo_kind, file_id) != ('file', current_file)): |
|
562 |
# Flush the data for a single file - prevents memory
|
|
563 |
# spiking due to buffering all files in memory.
|
|
564 |
self._install_mp_records_keys(self._repository.texts, |
|
565 |
pending_file_records) |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
566 |
current_file = None |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
567 |
del pending_file_records[:] |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
568 |
if len(pending_inventory_records) > 0 and repo_kind != 'inventory': |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
569 |
self._install_inventory_records(pending_inventory_records) |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
570 |
pending_inventory_records = [] |
571 |
if repo_kind == 'inventory': |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
572 |
pending_inventory_records.append(((revision_id,), metadata, bytes)) |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
573 |
if repo_kind == 'revision': |
574 |
target_revision = revision_id |
|
575 |
self._install_revision(revision_id, metadata, bytes) |
|
576 |
if repo_kind == 'signature': |
|
577 |
self._install_signature(revision_id, metadata, bytes) |
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
578 |
if repo_kind == 'file': |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
579 |
current_file = file_id |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
580 |
pending_file_records.append(((file_id, revision_id), metadata, bytes)) |
581 |
self._install_mp_records_keys(self._repository.texts, pending_file_records) |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
582 |
return target_revision |
2520.4.6
by Aaron Bentley
Get installation started |
583 |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
584 |
def _handle_info(self, info): |
585 |
"""Extract data from an info record"""
|
|
586 |
self._info = info |
|
587 |
self._source_serializer = self._serializer.get_source_serializer(info) |
|
588 |
if (info['supports_rich_root'] == 0 and |
|
589 |
self._repository.supports_rich_root()): |
|
590 |
self.update_root = True |
|
591 |
else: |
|
592 |
self.update_root = False |
|
593 |
||
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
594 |
def _install_mp_records(self, versionedfile, records): |
2520.4.61
by Aaron Bentley
Do bulk insertion of records |
595 |
if len(records) == 0: |
596 |
return
|
|
597 |
d_func = multiparent.MultiParent.from_patch |
|
598 |
vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in |
|
599 |
records if r not in versionedfile] |
|
600 |
versionedfile.add_mpdiffs(vf_records) |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
601 |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
602 |
def _install_mp_records_keys(self, versionedfile, records): |
603 |
d_func = multiparent.MultiParent.from_patch |
|
604 |
vf_records = [] |
|
605 |
for key, meta, text in records: |
|
3350.6.7
by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where. |
606 |
# Adapt to tuple interface: A length two key is a file_id,
|
607 |
# revision_id pair, a length 1 key is a
|
|
608 |
# revision/signature/inventory. We need to do this because
|
|
609 |
# the metadata extraction from the bundle has not yet been updated
|
|
610 |
# to use the consistent tuple interface itself.
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
611 |
if len(key) == 2: |
612 |
prefix = key[:1] |
|
613 |
else: |
|
614 |
prefix = () |
|
615 |
parents = [prefix + (parent,) for parent in meta['parents']] |
|
616 |
vf_records.append((key, parents, meta['sha1'], d_func(text))) |
|
617 |
versionedfile.add_mpdiffs(vf_records) |
|
618 |
||
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
619 |
def _get_parent_inventory_texts(self, inventory_text_cache, |
620 |
inventory_cache, parent_ids): |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
621 |
cached_parent_texts = {} |
622 |
remaining_parent_ids = [] |
|
623 |
for parent_id in parent_ids: |
|
624 |
p_text = inventory_text_cache.get(parent_id, None) |
|
625 |
if p_text is None: |
|
626 |
remaining_parent_ids.append(parent_id) |
|
627 |
else: |
|
628 |
cached_parent_texts[parent_id] = p_text |
|
629 |
ghosts = () |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
630 |
# TODO: Use inventory_cache to grab inventories we already have in
|
631 |
# memory
|
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
632 |
if remaining_parent_ids: |
633 |
# first determine what keys are actually present in the local
|
|
634 |
# inventories object (don't use revisions as they haven't been
|
|
635 |
# installed yet.)
|
|
636 |
parent_keys = [(r,) for r in remaining_parent_ids] |
|
637 |
present_parent_map = self._repository.inventories.get_parent_map( |
|
638 |
parent_keys) |
|
639 |
present_parent_ids = [] |
|
640 |
ghosts = set() |
|
641 |
for p_id in remaining_parent_ids: |
|
642 |
if (p_id,) in present_parent_map: |
|
643 |
present_parent_ids.append(p_id) |
|
644 |
else: |
|
645 |
ghosts.add(p_id) |
|
646 |
to_string = self._source_serializer.write_inventory_to_string |
|
647 |
for parent_inv in self._repository.iter_inventories( |
|
648 |
present_parent_ids): |
|
649 |
p_text = to_string(parent_inv) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
650 |
inventory_cache[parent_inv.revision_id] = parent_inv |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
651 |
cached_parent_texts[parent_inv.revision_id] = p_text |
652 |
inventory_text_cache[parent_inv.revision_id] = p_text |
|
653 |
||
654 |
parent_texts = [cached_parent_texts[parent_id] |
|
655 |
for parent_id in parent_ids |
|
656 |
if parent_id not in ghosts] |
|
657 |
return parent_texts |
|
658 |
||
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
659 |
def _install_inventory_records(self, records): |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
660 |
if (self._info['serializer'] == self._repository._serializer.format_num |
661 |
and self._repository._serializer.support_altered_by_hack): |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
662 |
return self._install_mp_records_keys(self._repository.inventories, |
663 |
records) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
664 |
# Use a 10MB text cache, since these are string xml inventories. Note
|
665 |
# that 10MB is fairly small for large projects (a single inventory can
|
|
666 |
# be >5MB). Another possibility is to cache 10-20 inventory texts
|
|
667 |
# instead
|
|
668 |
inventory_text_cache = lru_cache.LRUSizeCache(10*1024*1024) |
|
4543.2.21
by John Arbash Meinel
A few more tiny tweaks to comments, etc. |
669 |
# Also cache the in-memory representation. This allows us to create
|
670 |
# inventory deltas to apply rather than calling add_inventory from
|
|
671 |
# scratch each time.
|
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
672 |
inventory_cache = lru_cache.LRUCache(10) |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
673 |
pb = ui.ui_factory.nested_progress_bar() |
674 |
try: |
|
675 |
num_records = len(records) |
|
676 |
for idx, (key, metadata, bytes) in enumerate(records): |
|
677 |
pb.update('installing inventory', idx, num_records) |
|
678 |
revision_id = key[-1] |
|
679 |
parent_ids = metadata['parents'] |
|
680 |
# Note: This assumes the local ghosts are identical to the
|
|
681 |
# ghosts in the source, as the Bundle serialization
|
|
682 |
# format doesn't record ghosts.
|
|
683 |
p_texts = self._get_parent_inventory_texts(inventory_text_cache, |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
684 |
inventory_cache, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
685 |
parent_ids) |
686 |
# Why does to_lines() take strings as the source, it seems that
|
|
687 |
# it would have to cast to a list of lines, which we get back
|
|
688 |
# as lines and then cast back to a string.
|
|
689 |
target_lines = multiparent.MultiParent.from_patch(bytes |
|
690 |
).to_lines(p_texts) |
|
691 |
inv_text = ''.join(target_lines) |
|
692 |
del target_lines |
|
693 |
sha1 = osutils.sha_string(inv_text) |
|
694 |
if sha1 != metadata['sha1']: |
|
695 |
raise errors.BadBundle("Can't convert to target format") |
|
696 |
# Add this to the cache so we don't have to extract it again.
|
|
697 |
inventory_text_cache[revision_id] = inv_text |
|
698 |
target_inv = self._source_serializer.read_inventory_from_string( |
|
699 |
inv_text) |
|
700 |
self._handle_root(target_inv, parent_ids) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
701 |
parent_inv = None |
702 |
if parent_ids: |
|
703 |
parent_inv = inventory_cache.get(parent_ids[0], None) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
704 |
try: |
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
705 |
if parent_inv is None: |
706 |
self._repository.add_inventory(revision_id, target_inv, |
|
707 |
parent_ids) |
|
708 |
else: |
|
709 |
delta = target_inv._make_delta(parent_inv) |
|
710 |
self._repository.add_inventory_by_delta(parent_ids[0], |
|
711 |
delta, revision_id, parent_ids) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
712 |
except errors.UnsupportedInventoryKind: |
713 |
raise errors.IncompatibleRevision(repr(self._repository)) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
714 |
inventory_cache[revision_id] = target_inv |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
715 |
finally: |
716 |
pb.finished() |
|
2520.4.99
by Aaron Bentley
Test conversion across models |
717 |
|
718 |
def _handle_root(self, target_inv, parent_ids): |
|
719 |
revision_id = target_inv.revision_id |
|
720 |
if self.update_root: |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
721 |
text_key = (target_inv.root.file_id, revision_id) |
722 |
parent_keys = [(target_inv.root.file_id, parent) for |
|
723 |
parent in parent_ids] |
|
724 |
self._repository.texts.add_lines(text_key, parent_keys, []) |
|
2520.4.99
by Aaron Bentley
Test conversion across models |
725 |
elif not self._repository.supports_rich_root(): |
726 |
if target_inv.root.revision != revision_id: |
|
727 |
raise errors.IncompatibleRevision(repr(self._repository)) |
|
728 |
||
2520.4.59
by Aaron Bentley
Push metadata down the stack |
729 |
def _install_revision(self, revision_id, metadata, text): |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
730 |
if self._repository.has_revision(revision_id): |
731 |
return
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
732 |
revision = self._source_serializer.read_revision_from_string(text) |
733 |
self._repository.add_revision(revision.revision_id, revision) |
|
2520.4.34
by Aaron Bentley
Add signature support |
734 |
|
2520.4.59
by Aaron Bentley
Push metadata down the stack |
735 |
def _install_signature(self, revision_id, metadata, text): |
2520.4.100
by Aaron Bentley
Fix repeat signature installs |
736 |
transaction = self._repository.get_transaction() |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
737 |
if self._repository.has_signature_for_revision_id(revision_id): |
2520.4.100
by Aaron Bentley
Fix repeat signature installs |
738 |
return
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
739 |
self._repository.add_signature_text(revision_id, text) |