1
# Copyright (C) 2006 Canonical
1
# Copyright (C) 2006 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
20
20
"""Some functions to enable caching the conversion between unicode to utf8"""
25
_utf8_encode = codecs.getencoder("utf-8")
26
_utf8_decode = codecs.getdecoder("utf-8")
27
def _utf8_decode_with_None(bytestring, _utf8_decode=_utf8_decode):
28
"""wrap _utf8_decode to support None->None for optional strings.
30
Also, only return the Unicode portion, since we don't care about the second
33
if bytestring is None:
36
return _utf8_decode(bytestring)[0]
23
38
# Map revisions from and to utf8 encoding
24
39
# Whenever we do an encode/decode operation, we save the result, so that
30
45
def encode(unicode_str,
31
46
_uni_to_utf8=_unicode_to_utf8_map,
32
_utf8_to_uni=_utf8_to_unicode_map):
47
_utf8_to_uni=_utf8_to_unicode_map,
48
_utf8_encode=_utf8_encode):
33
49
"""Take this unicode revision id, and get a unicode version"""
34
50
# If the key is in the cache try/KeyError is 50% faster than
35
51
# val = dict.get(key), if val is None:
43
59
return _uni_to_utf8[unicode_str]
45
_uni_to_utf8[unicode_str] = utf8_str = unicode_str.encode('utf-8')
61
_uni_to_utf8[unicode_str] = utf8_str = _utf8_encode(unicode_str)[0]
46
62
_utf8_to_uni[utf8_str] = unicode_str
50
66
def decode(utf8_str,
51
67
_uni_to_utf8=_unicode_to_utf8_map,
52
_utf8_to_uni=_utf8_to_unicode_map):
68
_utf8_to_uni=_utf8_to_unicode_map,
69
_utf8_decode=_utf8_decode):
53
70
"""Take a utf8 revision id, and decode it, but cache the result"""
55
72
return _utf8_to_uni[utf8_str]
57
_utf8_to_uni[utf8_str] = unicode_str = utf8_str.decode('utf-8')
74
unicode_str = _utf8_decode(utf8_str)[0]
75
_utf8_to_uni[utf8_str] = unicode_str
58
76
_uni_to_utf8[unicode_str] = utf8_str
71
89
return decode(encode(unicode_str))
92
def get_cached_utf8(utf8_str):
93
"""Return a cached version of the utf-8 string.
95
Get a cached version of this string (similar to intern()).
96
At present, this will be decoded to ensure it is a utf-8 string. In the
97
future this might change to simply caching the string.
99
return encode(decode(utf8_str))
102
def get_cached_ascii(ascii_str,
103
_uni_to_utf8=_unicode_to_utf8_map,
104
_utf8_to_uni=_utf8_to_unicode_map):
105
"""This is a string which is identical in utf-8 and unicode."""
106
# We don't need to do any encoding, but we want _utf8_to_uni to return a
107
# real Unicode string. Unicode and plain strings of this type will have the
108
# same hash, so we can just use it as the key in _uni_to_utf8, but we need
109
# the return value to be different in _utf8_to_uni
110
ascii_str = _uni_to_utf8.setdefault(ascii_str, ascii_str)
111
_utf8_to_uni.setdefault(ascii_str, unicode(ascii_str))
74
115
def clear_encoding_cache():
75
116
"""Clear the encoding and decoding caches"""
76
global _unicode_to_utf8_map, _utf8_to_unicode_map
77
117
_unicode_to_utf8_map.clear()
78
118
_utf8_to_unicode_map.clear()
added
removed
bzrlib/cache_utf8.py
