1
# Copyright (C) 2005, 2006 Canonical Ltd
1
# Copyright (C) 2005-2010 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
from bzrlib.xml_serializer import SubElement, Element, Serializer
19
from bzrlib.inventory import ROOT_ID, Inventory, InventoryEntry
20
import bzrlib.inventory as inventory
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24
revision as _mod_revision,
27
from bzrlib.xml_serializer import (
33
from bzrlib.inventory import InventoryEntry
21
34
from bzrlib.revision import Revision
22
35
from bzrlib.errors import BzrError
25
class Serializer_v5(Serializer):
26
"""Version 5 serializer
28
Packs objects into XML and vice versa.
42
"'":"'", # FIXME: overkill
49
def _ensure_utf8_re():
50
"""Make sure the _utf8_re and _unicode_re regexes have been compiled."""
51
global _utf8_re, _unicode_re
53
_utf8_re = re.compile('[&<>\'\"]|[\x80-\xff]+')
54
if _unicode_re is None:
55
_unicode_re = re.compile(u'[&<>\'\"\u0080-\uffff]')
58
def _unicode_escape_replace(match, _map=_xml_escape_map):
59
"""Replace a string of non-ascii, non XML safe characters with their escape
61
This will escape both Standard XML escapes, like <>"', etc.
62
As well as escaping non ascii characters, because ElementTree did.
63
This helps us remain compatible to older versions of bzr. We may change
64
our policy in the future, though.
66
# jam 20060816 Benchmarks show that try/KeyError is faster if you
67
# expect the entity to rarely miss. There is about a 10% difference
68
# in overall time. But if you miss frequently, then if None is much
69
# faster. For our use case, we *rarely* have a revision id, file id
70
# or path name that is unicode. So use try/KeyError.
72
return _map[match.group()]
74
return "&#%d;" % ord(match.group())
77
def _utf8_escape_replace(match, _map=_xml_escape_map):
78
"""Escape utf8 characters into XML safe ones.
80
This uses 2 tricks. It is either escaping "standard" characters, like "&<>,
81
or it is handling characters with the high-bit set. For ascii characters,
82
we just lookup the replacement in the dictionary. For everything else, we
83
decode back into Unicode, and then use the XML escape code.
86
return _map[match.group()]
88
return ''.join('&#%d;' % ord(uni_chr)
89
for uni_chr in match.group().decode('utf8'))
94
def _encode_and_escape(unicode_or_utf8_str, _map=_to_escaped_map):
95
"""Encode the string into utf8, and escape invalid XML characters"""
96
# We frequently get entities we have not seen before, so it is better
97
# to check if None, rather than try/KeyError
98
text = _map.get(unicode_or_utf8_str)
100
if unicode_or_utf8_str.__class__ is unicode:
101
# The alternative policy is to do a regular UTF8 encoding
102
# and then escape only XML meta characters.
103
# Performance is equivalent once you use cache_utf8. *However*
104
# this makes the serialized texts incompatible with old versions
105
# of bzr. So no net gain. (Perhaps the read code would handle utf8
106
# better than entity escapes, but cElementTree seems to do just fine
108
text = str(_unicode_re.sub(_unicode_escape_replace,
109
unicode_or_utf8_str)) + '"'
111
# Plain strings are considered to already be in utf-8 so we do a
112
# slightly different method for escaping.
113
text = _utf8_re.sub(_utf8_escape_replace,
114
unicode_or_utf8_str) + '"'
115
_map[unicode_or_utf8_str] = text
119
def _get_utf8_or_ascii(a_str,
120
_encode_utf8=cache_utf8.encode,
121
_get_cached_ascii=cache_utf8.get_cached_ascii):
122
"""Return a cached version of the string.
124
cElementTree will return a plain string if the XML is plain ascii. It only
125
returns Unicode when it needs to. We want to work in utf-8 strings. So if
126
cElementTree returns a plain string, we can just return the cached version.
127
If it is Unicode, then we need to encode it.
129
:param a_str: An 8-bit string or Unicode as returned by
130
cElementTree.Element.get()
131
:return: A utf-8 encoded 8-bit string.
133
# This is fairly optimized because we know what cElementTree does, this is
134
# not meant as a generic function for all cases. Because it is possible for
135
# an 8-bit string to not be ascii or valid utf8.
136
if a_str.__class__ is unicode:
137
return _encode_utf8(a_str)
143
"""Clean out the unicode => escaped map"""
144
_to_escaped_map.clear()
147
class Serializer_v8(XMLSerializer):
148
"""This serialiser adds rich roots.
150
Its revision format number matches its inventory number.
33
def _pack_inventory(self, inv):
34
"""Convert to XML Element"""
35
e = Element('inventory',
38
if inv.root.file_id not in (None, ROOT_ID):
39
e.set('file_id', inv.root.file_id)
156
support_altered_by_hack = True
157
# This format supports the altered-by hack that reads file ids directly out
158
# of the versionedfile, without doing XML parsing.
160
supported_kinds = set(['file', 'directory', 'symlink'])
162
revision_format_num = None
164
def _check_revisions(self, inv):
165
"""Extension point for subclasses to check during serialisation.
167
:param inv: An inventory about to be serialised, to be checked.
168
:raises: AssertionError if an error has occurred.
170
if inv.revision_id is None:
171
raise AssertionError("inv.revision_id is None")
172
if inv.root.revision is None:
173
raise AssertionError("inv.root.revision is None")
175
def _check_cache_size(self, inv_size, entry_cache):
176
"""Check that the entry_cache is large enough.
178
We want the cache to be ~2x the size of an inventory. The reason is
179
because we use a FIFO cache, and how Inventory records are likely to
180
change. In general, you have a small number of records which change
181
often, and a lot of records which do not change at all. So when the
182
cache gets full, you actually flush out a lot of the records you are
183
interested in, which means you need to recreate all of those records.
184
An LRU Cache would be better, but the overhead negates the cache
187
One way to look at it, only the size of the cache > len(inv) is your
188
'working' set. And in general, it shouldn't be a problem to hold 2
189
inventories in memory anyway.
191
:param inv_size: The number of entries in an inventory.
193
if entry_cache is None:
195
# 1.5 times might also be reasonable.
196
recommended_min_cache_size = inv_size * 1.5
197
if entry_cache.cache_size() < recommended_min_cache_size:
198
recommended_cache_size = inv_size * 2
199
trace.mutter('Resizing the inventory entry cache from %d to %d',
200
entry_cache.cache_size(), recommended_cache_size)
201
entry_cache.resize(recommended_cache_size)
203
def write_inventory_to_lines(self, inv):
204
"""Return a list of lines with the encoded inventory."""
205
return self.write_inventory(inv, None)
207
def write_inventory_to_string(self, inv, working=False):
208
"""Just call write_inventory with a StringIO and return the value.
210
:param working: If True skip history data - text_sha1, text_size,
211
reference_revision, symlink_target.
213
sio = cStringIO.StringIO()
214
self.write_inventory(inv, sio, working)
215
return sio.getvalue()
217
def write_inventory(self, inv, f, working=False):
218
"""Write inventory to a file.
220
:param inv: the inventory to write.
221
:param f: the file to write. (May be None if the lines are the desired
223
:param working: If True skip history data - text_sha1, text_size,
224
reference_revision, symlink_target.
225
:return: The inventory as a list of lines.
228
self._check_revisions(inv)
230
append = output.append
231
self._append_inventory_root(append, inv)
232
entries = inv.iter_entries()
234
root_path, root_ie = entries.next()
235
for path, ie in entries:
236
if ie.parent_id != self.root_id:
237
parent_str = ' parent_id="'
238
parent_id = _encode_and_escape(ie.parent_id)
242
if ie.kind == 'file':
244
executable = ' executable="yes"'
248
append('<file%s file_id="%s name="%s%s%s revision="%s '
249
'text_sha1="%s" text_size="%d" />\n' % (
250
executable, _encode_and_escape(ie.file_id),
251
_encode_and_escape(ie.name), parent_str, parent_id,
252
_encode_and_escape(ie.revision), ie.text_sha1,
255
append('<file%s file_id="%s name="%s%s%s />\n' % (
256
executable, _encode_and_escape(ie.file_id),
257
_encode_and_escape(ie.name), parent_str, parent_id))
258
elif ie.kind == 'directory':
260
append('<directory file_id="%s name="%s%s%s revision="%s '
262
_encode_and_escape(ie.file_id),
263
_encode_and_escape(ie.name),
264
parent_str, parent_id,
265
_encode_and_escape(ie.revision)))
267
append('<directory file_id="%s name="%s%s%s />\n' % (
268
_encode_and_escape(ie.file_id),
269
_encode_and_escape(ie.name),
270
parent_str, parent_id))
271
elif ie.kind == 'symlink':
273
append('<symlink file_id="%s name="%s%s%s revision="%s '
274
'symlink_target="%s />\n' % (
275
_encode_and_escape(ie.file_id),
276
_encode_and_escape(ie.name),
277
parent_str, parent_id,
278
_encode_and_escape(ie.revision),
279
_encode_and_escape(ie.symlink_target)))
281
append('<symlink file_id="%s name="%s%s%s />\n' % (
282
_encode_and_escape(ie.file_id),
283
_encode_and_escape(ie.name),
284
parent_str, parent_id))
285
elif ie.kind == 'tree-reference':
286
if ie.kind not in self.supported_kinds:
287
raise errors.UnsupportedInventoryKind(ie.kind)
289
append('<tree-reference file_id="%s name="%s%s%s '
290
'revision="%s reference_revision="%s />\n' % (
291
_encode_and_escape(ie.file_id),
292
_encode_and_escape(ie.name),
293
parent_str, parent_id,
294
_encode_and_escape(ie.revision),
295
_encode_and_escape(ie.reference_revision)))
297
append('<tree-reference file_id="%s name="%s%s%s />\n' % (
298
_encode_and_escape(ie.file_id),
299
_encode_and_escape(ie.name),
300
parent_str, parent_id))
302
raise errors.UnsupportedInventoryKind(ie.kind)
303
append('</inventory>\n')
306
# Just to keep the cache from growing without bounds
307
# but we may actually not want to do clear the cache
311
def _append_inventory_root(self, append, inv):
312
"""Append the inventory root to output."""
40
313
if inv.revision_id is not None:
41
e.set('revision_id', inv.revision_id)
42
for path, ie in inv.iter_entries():
43
e.append(self._pack_entry(ie))
46
def _pack_entry(self, ie):
47
"""Convert InventoryEntry to XML element"""
48
# TODO: should just be a plain assertion
49
if not InventoryEntry.versionable_kind(ie.kind):
50
raise AssertionError('unsupported entry kind %s' % ie.kind)
52
e.set('name', ie.name)
53
e.set('file_id', ie.file_id)
55
if ie.text_size != None:
56
e.set('text_size', '%d' % ie.text_size)
58
for f in ['text_sha1', 'revision', 'symlink_target']:
64
e.set('executable', 'yes')
66
# to be conservative, we don't externalize the root pointers
67
# for now, leaving them as null in the xml form. in a future
68
# version it will be implied by nested elements.
69
if ie.parent_id != ROOT_ID:
70
assert isinstance(ie.parent_id, basestring)
71
e.set('parent_id', ie.parent_id)
314
revid1 = ' revision_id="'
315
revid2 = _encode_and_escape(inv.revision_id)
319
append('<inventory format="%s"%s%s>\n' % (
320
self.format_num, revid1, revid2))
321
append('<directory file_id="%s name="%s revision="%s />\n' % (
322
_encode_and_escape(inv.root.file_id),
323
_encode_and_escape(inv.root.name),
324
_encode_and_escape(inv.root.revision)))
75
326
def _pack_revision(self, rev):
76
327
"""Revision object -> xml tree"""
328
# For the XML format, we need to write them as Unicode rather than as
329
# utf-8 strings. So that cElementTree can handle properly escaping
331
decode_utf8 = cache_utf8.decode
332
revision_id = rev.revision_id
333
if isinstance(revision_id, str):
334
revision_id = decode_utf8(revision_id)
335
format_num = self.format_num
336
if self.revision_format_num is not None:
337
format_num = self.revision_format_num
77
338
root = Element('revision',
78
339
committer = rev.committer,
79
timestamp = '%.9f' % rev.timestamp,
80
revision_id = rev.revision_id,
340
timestamp = '%.3f' % rev.timestamp,
341
revision_id = revision_id,
81
342
inventory_sha1 = rev.inventory_sha1,
345
if rev.timezone is not None:
85
346
root.set('timezone', str(rev.timezone))
87
348
msg = SubElement(root, 'message')
88
msg.text = rev.message
349
msg.text = escape_invalid_chars(rev.message)[0]
90
351
if rev.parent_ids:
91
352
pelts = SubElement(root, 'parents')
92
353
pelts.tail = pelts.text = '\n'
93
354
for parent_id in rev.parent_ids:
94
assert isinstance(parent_id, basestring)
355
_mod_revision.check_not_reserved_id(parent_id)
95
356
p = SubElement(pelts, 'revision_ref')
358
if isinstance(parent_id, str):
359
parent_id = decode_utf8(parent_id)
97
360
p.set('revision_id', parent_id)
98
361
if rev.properties:
99
362
self._pack_revision_properties(rev, root)
103
365
def _pack_revision_properties(self, rev, under_element):
104
366
top_elt = SubElement(under_element, 'properties')
105
367
for prop_name, prop_value in sorted(rev.properties.items()):
106
assert isinstance(prop_name, basestring)
107
assert isinstance(prop_value, basestring)
108
368
prop_elt = SubElement(top_elt, 'property')
109
369
prop_elt.set('name', prop_name)
110
370
prop_elt.text = prop_value
111
371
prop_elt.tail = '\n'
112
372
top_elt.tail = '\n'
115
def _unpack_inventory(self, elt):
116
"""Construct from XML Element
118
assert elt.tag == 'inventory'
119
root_id = elt.get('file_id') or ROOT_ID
374
def _unpack_inventory(self, elt, revision_id=None, entry_cache=None,
375
return_from_cache=False):
376
"""Construct from XML Element"""
377
if elt.tag != 'inventory':
378
raise errors.UnexpectedInventoryFormat('Root tag is %r' % elt.tag)
120
379
format = elt.get('format')
121
if format is not None:
123
raise BzrError("invalid format version %r on inventory"
380
if format != self.format_num:
381
raise errors.UnexpectedInventoryFormat('Invalid format version %r'
125
383
revision_id = elt.get('revision_id')
126
inv = Inventory(root_id, revision_id=revision_id)
384
if revision_id is not None:
385
revision_id = cache_utf8.encode(revision_id)
386
inv = inventory.Inventory(root_id=None, revision_id=revision_id)
128
ie = self._unpack_entry(e)
129
if ie.parent_id == ROOT_ID:
130
ie.parent_id = root_id
388
ie = self._unpack_entry(e, entry_cache=entry_cache,
389
return_from_cache=return_from_cache)
391
self._check_cache_size(len(inv), entry_cache)
135
def _unpack_entry(self, elt):
394
def _unpack_entry(self, elt, entry_cache=None, return_from_cache=False):
396
file_id = elt_get('file_id')
397
revision = elt_get('revision')
398
# Check and see if we have already unpacked this exact entry
399
# Some timings for "repo.revision_trees(last_100_revs)"
401
# unmodified 4.1s 40.8s
403
# using fifo 2.83s 29.1s
407
# no_copy 2.00s 20.5s
408
# no_c,dict 1.95s 18.0s
409
# Note that a cache of 10k nodes is more than sufficient to hold all of
410
# the inventory for the last 100 revs for bzr, but not for mysql (20k
411
# is enough for mysql, which saves the same 2s as using a dict)
413
# Breakdown of mysql using time.clock()
414
# 4.1s 2 calls to element.get for file_id, revision_id
415
# 4.5s cache_hit lookup
416
# 7.1s InventoryFile.copy()
417
# 2.4s InventoryDirectory.copy()
418
# 0.4s decoding unique entries
419
# 1.6s decoding entries after FIFO fills up
420
# 0.8s Adding nodes to FIFO (including flushes)
421
# 0.1s cache miss lookups
423
# 4.1s 2 calls to element.get for file_id, revision_id
424
# 9.9s cache_hit lookup
425
# 10.8s InventoryEntry.copy()
426
# 0.3s cache miss lookus
427
# 1.2s decoding entries
428
# 1.0s adding nodes to LRU
429
if entry_cache is not None and revision is not None:
430
key = (file_id, revision)
432
# We copy it, because some operations may mutate it
433
cached_ie = entry_cache[key]
437
# Only copying directory entries drops us 2.85s => 2.35s
438
if return_from_cache:
439
if cached_ie.kind == 'directory':
440
return cached_ie.copy()
442
return cached_ie.copy()
137
445
if not InventoryEntry.versionable_kind(kind):
138
446
raise AssertionError('unsupported entry kind %s' % kind)
140
parent_id = elt.get('parent_id')
141
if parent_id == None:
448
get_cached = _get_utf8_or_ascii
450
file_id = get_cached(file_id)
451
if revision is not None:
452
revision = get_cached(revision)
453
parent_id = elt_get('parent_id')
454
if parent_id is not None:
455
parent_id = get_cached(parent_id)
144
457
if kind == 'directory':
145
ie = inventory.InventoryDirectory(elt.get('file_id'),
458
ie = inventory.InventoryDirectory(file_id,
148
461
elif kind == 'file':
149
ie = inventory.InventoryFile(elt.get('file_id'),
462
ie = inventory.InventoryFile(file_id,
152
ie.text_sha1 = elt.get('text_sha1')
153
if elt.get('executable') == 'yes':
465
ie.text_sha1 = elt_get('text_sha1')
466
if elt_get('executable') == 'yes':
154
467
ie.executable = True
155
v = elt.get('text_size')
468
v = elt_get('text_size')
156
469
ie.text_size = v and int(v)
157
470
elif kind == 'symlink':
158
ie = inventory.InventoryLink(elt.get('file_id'),
471
ie = inventory.InventoryLink(file_id,
161
ie.symlink_target = elt.get('symlink_target')
474
ie.symlink_target = elt_get('symlink_target')
163
raise BzrError("unknown kind %r" % kind)
164
ie.revision = elt.get('revision')
476
raise errors.UnsupportedInventoryKind(kind)
477
ie.revision = revision
478
if revision is not None and entry_cache is not None:
479
# We cache a copy() because callers like to mutate objects, and
480
# that would cause the item in cache to mutate as well.
481
# This has a small effect on many-inventory performance, because
482
# the majority fraction is spent in cache hits, not misses.
483
entry_cache[key] = ie.copy()
169
487
def _unpack_revision(self, elt):
170
488
"""XML Element -> Revision object"""
171
assert elt.tag == 'revision'
172
489
format = elt.get('format')
490
format_num = self.format_num
491
if self.revision_format_num is not None:
492
format_num = self.revision_format_num
173
493
if format is not None:
175
raise BzrError("invalid format version %r on inventory"
494
if format != format_num:
495
raise BzrError("invalid format version %r on revision"
497
get_cached = _get_utf8_or_ascii
177
498
rev = Revision(committer = elt.get('committer'),
178
499
timestamp = float(elt.get('timestamp')),
179
revision_id = elt.get('revision_id'),
500
revision_id = get_cached(elt.get('revision_id')),
180
501
inventory_sha1 = elt.get('inventory_sha1')
182
503
parents = elt.find('parents') or []
183
504
for p in parents:
184
assert p.tag == 'revision_ref', \
185
"bad parent node tag %r" % p.tag
186
rev.parent_ids.append(p.get('revision_id'))
505
rev.parent_ids.append(get_cached(p.get('revision_id')))
187
506
self._unpack_revision_properties(elt, rev)
188
507
v = elt.get('timezone')
189
rev.timezone = v and int(v)
511
rev.timezone = int(v)
190
512
rev.message = elt.findtext('message') # text of <message>
194
515
def _unpack_revision_properties(self, elt, rev):
195
516
"""Unpack properties onto a revision."""
196
517
props_elt = elt.find('properties')
197
assert len(rev.properties) == 0
198
518
if not props_elt:
200
520
for prop_elt in props_elt:
201
assert prop_elt.tag == 'property', \
202
"bad tag under properties list: %r" % prop_elt.tag
521
if prop_elt.tag != 'property':
522
raise AssertionError(
523
"bad tag under properties list: %r" % prop_elt.tag)
203
524
name = prop_elt.get('name')
204
525
value = prop_elt.text
205
assert name not in rev.properties, \
206
"repeated property %r" % name
526
# If a property had an empty value ('') cElementTree reads
527
# that back as None, convert it back to '', so that all
528
# properties have string values
531
if name in rev.properties:
532
raise AssertionError("repeated property %r" % name)
207
533
rev.properties[name] = value
210
serializer_v5 = Serializer_v5()
536
serializer_v8 = Serializer_v8()
added
removed
bzrlib/xml8.py
