2
# -*- coding: UTF-8 -*-
1
# Copyright (C) 2005-2010 Canonical Ltd
4
3
# This program is free software; you can redistribute it and/or modify
5
4
# it under the terms of the GNU General Public License as published by
6
5
# the Free Software Foundation; either version 2 of the License, or
7
6
# (at your option) any later version.
9
8
# This program is distributed in the hope that it will be useful,
10
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
11
# GNU General Public License for more details.
14
13
# You should have received a copy of the GNU General Public License
15
14
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
17
"""XML externalization support."""
19
from __future__ import absolute_import
20
21
# "XML is like violence: if it doesn't solve your problem, you aren't
21
22
# using enough of it." -- various
23
# importing this module is fairly slow because it has to load several ElementTree bits
24
# importing this module is fairly slow because it has to load several
29
from bzrlib.serializer import Serializer
30
from bzrlib.trace import mutter
25
from cElementTree import ElementTree, SubElement, Element
34
# it's in this package in python2.5
35
from xml.etree.cElementTree import (ElementTree, SubElement, Element,
36
XMLTreeBuilder, fromstring, tostring)
37
import xml.etree as elementtree
38
# Also import ElementTree module so monkey-patching below always works
39
import xml.etree.ElementTree
41
from cElementTree import (ElementTree, SubElement, Element,
42
XMLTreeBuilder, fromstring, tostring)
43
import elementtree.ElementTree
44
ParseError = SyntaxError
26
45
except ImportError:
27
from elementtree.ElementTree import ElementTree, SubElement, Element
31
"""Write object o to file f as XML.
33
o must provide a to_element method.
35
ElementTree(o.to_element()).write(f, 'utf-8')
39
def unpack_xml(cls, f):
40
return cls.from_element(ElementTree().parse(f))
46
mutter('WARNING: using slower ElementTree; consider installing cElementTree'
47
" and make sure it's on your PYTHONPATH")
48
# this copy is shipped with bzr
49
from util.elementtree.ElementTree import (ElementTree, SubElement,
50
Element, XMLTreeBuilder,
52
import util.elementtree as elementtree
53
from xml.parsers.expat import ExpatError as ParseError
63
class XMLSerializer(Serializer):
64
"""Abstract XML object serialize/deserialize"""
66
squashes_xml_invalid_characters = True
68
def read_inventory_from_string(self, xml_string, revision_id=None,
69
entry_cache=None, return_from_cache=False):
70
"""Read xml_string into an inventory object.
72
:param xml_string: The xml to read.
73
:param revision_id: If not-None, the expected revision id of the
74
inventory. Some serialisers use this to set the results' root
75
revision. This should be supplied for deserialising all
76
from-repository inventories so that xml5 inventories that were
77
serialised without a revision identifier can be given the right
78
revision id (but not for working tree inventories where users can
79
edit the data without triggering checksum errors or anything).
80
:param entry_cache: An optional cache of InventoryEntry objects. If
81
supplied we will look up entries via (file_id, revision_id) which
82
should map to a valid InventoryEntry (File/Directory/etc) object.
83
:param return_from_cache: Return entries directly from the cache,
84
rather than copying them first. This is only safe if the caller
85
promises not to mutate the returned inventory entries, but it can
86
make some operations significantly faster.
89
return self._unpack_inventory(fromstring(xml_string), revision_id,
90
entry_cache=entry_cache,
91
return_from_cache=return_from_cache)
93
raise errors.UnexpectedInventoryFormat(e)
95
def read_inventory(self, f, revision_id=None):
98
return self._unpack_inventory(self._read_element(f),
102
except ParseError, e:
103
raise errors.UnexpectedInventoryFormat(e)
105
def write_revision(self, rev, f):
106
self._write_element(self._pack_revision(rev), f)
108
def write_revision_to_string(self, rev):
109
return tostring(self._pack_revision(rev)) + '\n'
111
def read_revision(self, f):
112
return self._unpack_revision(self._read_element(f))
114
def read_revision_from_string(self, xml_string):
115
return self._unpack_revision(fromstring(xml_string))
117
def _write_element(self, elt, f):
118
ElementTree(elt).write(f, 'utf-8')
121
def _read_element(self, f):
122
return ElementTree().parse(f)
125
def escape_invalid_chars(message):
126
"""Escape the XML-invalid characters in a commit message.
128
:param message: Commit message to escape
129
:return: tuple with escaped message and number of characters escaped
133
# Python strings can include characters that can't be
134
# represented in well-formed XML; escape characters that
135
# aren't listed in the XML specification
136
# (http://www.w3.org/TR/REC-xml/#NT-Char).
137
return re.subn(u'[^\x09\x0A\x0D\u0020-\uD7FF\uE000-\uFFFD]+',
138
lambda match: match.group(0).encode('unicode_escape'),
142
def get_utf8_or_ascii(a_str, _encode_utf8=cache_utf8.encode):
143
"""Return a cached version of the string.
145
cElementTree will return a plain string if the XML is plain ascii. It only
146
returns Unicode when it needs to. We want to work in utf-8 strings. So if
147
cElementTree returns a plain string, we can just return the cached version.
148
If it is Unicode, then we need to encode it.
150
:param a_str: An 8-bit string or Unicode as returned by
151
cElementTree.Element.get()
152
:return: A utf-8 encoded 8-bit string.
154
# This is fairly optimized because we know what cElementTree does, this is
155
# not meant as a generic function for all cases. Because it is possible for
156
# an 8-bit string to not be ascii or valid utf8.
157
if a_str.__class__ is unicode:
158
return _encode_utf8(a_str)
163
_utf8_re = lazy_regex.lazy_compile('[&<>\'\"]|[\x80-\xff]+')
164
_unicode_re = lazy_regex.lazy_compile(u'[&<>\'\"\u0080-\uffff]')
169
"'":"'", # FIXME: overkill
176
def _unicode_escape_replace(match, _map=_xml_escape_map):
177
"""Replace a string of non-ascii, non XML safe characters with their escape
179
This will escape both Standard XML escapes, like <>"', etc.
180
As well as escaping non ascii characters, because ElementTree did.
181
This helps us remain compatible to older versions of bzr. We may change
182
our policy in the future, though.
184
# jam 20060816 Benchmarks show that try/KeyError is faster if you
185
# expect the entity to rarely miss. There is about a 10% difference
186
# in overall time. But if you miss frequently, then if None is much
187
# faster. For our use case, we *rarely* have a revision id, file id
188
# or path name that is unicode. So use try/KeyError.
190
return _map[match.group()]
192
return "&#%d;" % ord(match.group())
195
def _utf8_escape_replace(match, _map=_xml_escape_map):
196
"""Escape utf8 characters into XML safe ones.
198
This uses 2 tricks. It is either escaping "standard" characters, like "&<>,
199
or it is handling characters with the high-bit set. For ascii characters,
200
we just lookup the replacement in the dictionary. For everything else, we
201
decode back into Unicode, and then use the XML escape code.
204
return _map[match.group()]
206
return ''.join('&#%d;' % ord(uni_chr)
207
for uni_chr in match.group().decode('utf8'))
212
def encode_and_escape(unicode_or_utf8_str, _map=_to_escaped_map):
213
"""Encode the string into utf8, and escape invalid XML characters"""
214
# We frequently get entities we have not seen before, so it is better
215
# to check if None, rather than try/KeyError
216
text = _map.get(unicode_or_utf8_str)
218
if unicode_or_utf8_str.__class__ is unicode:
219
# The alternative policy is to do a regular UTF8 encoding
220
# and then escape only XML meta characters.
221
# Performance is equivalent once you use cache_utf8. *However*
222
# this makes the serialized texts incompatible with old versions
223
# of bzr. So no net gain. (Perhaps the read code would handle utf8
224
# better than entity escapes, but cElementTree seems to do just fine
226
text = str(_unicode_re.sub(_unicode_escape_replace,
227
unicode_or_utf8_str)) + '"'
229
# Plain strings are considered to already be in utf-8 so we do a
230
# slightly different method for escaping.
231
text = _utf8_re.sub(_utf8_escape_replace,
232
unicode_or_utf8_str) + '"'
233
_map[unicode_or_utf8_str] = text
238
"""Clean out the unicode => escaped map"""
239
_to_escaped_map.clear()
242
def unpack_inventory_entry(elt, entry_cache=None, return_from_cache=False):
244
file_id = elt_get('file_id')
245
revision = elt_get('revision')
246
# Check and see if we have already unpacked this exact entry
247
# Some timings for "repo.revision_trees(last_100_revs)"
249
# unmodified 4.1s 40.8s
251
# using fifo 2.83s 29.1s
255
# no_copy 2.00s 20.5s
256
# no_c,dict 1.95s 18.0s
257
# Note that a cache of 10k nodes is more than sufficient to hold all of
258
# the inventory for the last 100 revs for bzr, but not for mysql (20k
259
# is enough for mysql, which saves the same 2s as using a dict)
261
# Breakdown of mysql using time.clock()
262
# 4.1s 2 calls to element.get for file_id, revision_id
263
# 4.5s cache_hit lookup
264
# 7.1s InventoryFile.copy()
265
# 2.4s InventoryDirectory.copy()
266
# 0.4s decoding unique entries
267
# 1.6s decoding entries after FIFO fills up
268
# 0.8s Adding nodes to FIFO (including flushes)
269
# 0.1s cache miss lookups
271
# 4.1s 2 calls to element.get for file_id, revision_id
272
# 9.9s cache_hit lookup
273
# 10.8s InventoryEntry.copy()
274
# 0.3s cache miss lookus
275
# 1.2s decoding entries
276
# 1.0s adding nodes to LRU
277
if entry_cache is not None and revision is not None:
278
key = (file_id, revision)
280
# We copy it, because some operations may mutate it
281
cached_ie = entry_cache[key]
285
# Only copying directory entries drops us 2.85s => 2.35s
286
if return_from_cache:
287
if cached_ie.kind == 'directory':
288
return cached_ie.copy()
290
return cached_ie.copy()
293
if not inventory.InventoryEntry.versionable_kind(kind):
294
raise AssertionError('unsupported entry kind %s' % kind)
296
file_id = get_utf8_or_ascii(file_id)
297
if revision is not None:
298
revision = get_utf8_or_ascii(revision)
299
parent_id = elt_get('parent_id')
300
if parent_id is not None:
301
parent_id = get_utf8_or_ascii(parent_id)
303
if kind == 'directory':
304
ie = inventory.InventoryDirectory(file_id,
308
ie = inventory.InventoryFile(file_id,
311
ie.text_sha1 = elt_get('text_sha1')
312
if elt_get('executable') == 'yes':
314
v = elt_get('text_size')
315
ie.text_size = v and int(v)
316
elif kind == 'symlink':
317
ie = inventory.InventoryLink(file_id,
320
ie.symlink_target = elt_get('symlink_target')
322
raise errors.UnsupportedInventoryKind(kind)
323
ie.revision = revision
324
if revision is not None and entry_cache is not None:
325
# We cache a copy() because callers like to mutate objects, and
326
# that would cause the item in cache to mutate as well.
327
# This has a small effect on many-inventory performance, because
328
# the majority fraction is spent in cache hits, not misses.
329
entry_cache[key] = ie.copy()
334
def unpack_inventory_flat(elt, format_num, unpack_entry,
335
entry_cache=None, return_from_cache=False):
336
"""Unpack a flat XML inventory.
338
:param elt: XML element for the inventory
339
:param format_num: Expected format number
340
:param unpack_entry: Function for unpacking inventory entries
341
:return: An inventory
342
:raise UnexpectedInventoryFormat: When unexpected elements or data is
345
if elt.tag != 'inventory':
346
raise errors.UnexpectedInventoryFormat('Root tag is %r' % elt.tag)
347
format = elt.get('format')
348
if format != format_num:
349
raise errors.UnexpectedInventoryFormat('Invalid format version %r'
351
revision_id = elt.get('revision_id')
352
if revision_id is not None:
353
revision_id = cache_utf8.encode(revision_id)
354
inv = inventory.Inventory(root_id=None, revision_id=revision_id)
356
ie = unpack_entry(e, entry_cache, return_from_cache)
361
def serialize_inventory_flat(inv, append, root_id, supported_kinds, working):
362
"""Serialize an inventory to a flat XML file.
364
:param inv: Inventory to serialize
365
:param append: Function for writing a line of output
366
:param working: If True skip history data - text_sha1, text_size,
367
reference_revision, symlink_target. self._check_revisions(inv)
369
entries = inv.iter_entries()
371
root_path, root_ie = entries.next()
372
for path, ie in entries:
373
if ie.parent_id != root_id:
374
parent_str = ' parent_id="'
375
parent_id = encode_and_escape(ie.parent_id)
379
if ie.kind == 'file':
381
executable = ' executable="yes"'
385
append('<file%s file_id="%s name="%s%s%s revision="%s '
386
'text_sha1="%s" text_size="%d" />\n' % (
387
executable, encode_and_escape(ie.file_id),
388
encode_and_escape(ie.name), parent_str, parent_id,
389
encode_and_escape(ie.revision), ie.text_sha1,
392
append('<file%s file_id="%s name="%s%s%s />\n' % (
393
executable, encode_and_escape(ie.file_id),
394
encode_and_escape(ie.name), parent_str, parent_id))
395
elif ie.kind == 'directory':
397
append('<directory file_id="%s name="%s%s%s revision="%s '
399
encode_and_escape(ie.file_id),
400
encode_and_escape(ie.name),
401
parent_str, parent_id,
402
encode_and_escape(ie.revision)))
404
append('<directory file_id="%s name="%s%s%s />\n' % (
405
encode_and_escape(ie.file_id),
406
encode_and_escape(ie.name),
407
parent_str, parent_id))
408
elif ie.kind == 'symlink':
410
append('<symlink file_id="%s name="%s%s%s revision="%s '
411
'symlink_target="%s />\n' % (
412
encode_and_escape(ie.file_id),
413
encode_and_escape(ie.name),
414
parent_str, parent_id,
415
encode_and_escape(ie.revision),
416
encode_and_escape(ie.symlink_target)))
418
append('<symlink file_id="%s name="%s%s%s />\n' % (
419
encode_and_escape(ie.file_id),
420
encode_and_escape(ie.name),
421
parent_str, parent_id))
422
elif ie.kind == 'tree-reference':
423
if ie.kind not in supported_kinds:
424
raise errors.UnsupportedInventoryKind(ie.kind)
426
append('<tree-reference file_id="%s name="%s%s%s '
427
'revision="%s reference_revision="%s />\n' % (
428
encode_and_escape(ie.file_id),
429
encode_and_escape(ie.name),
430
parent_str, parent_id,
431
encode_and_escape(ie.revision),
432
encode_and_escape(ie.reference_revision)))
434
append('<tree-reference file_id="%s name="%s%s%s />\n' % (
435
encode_and_escape(ie.file_id),
436
encode_and_escape(ie.name),
437
parent_str, parent_id))
439
raise errors.UnsupportedInventoryKind(ie.kind)
440
append('</inventory>\n')
added
removed
bzrlib/xml_serializer.py
