Mercurial > foo_out_sdl

comparison foosdk/sdk/pfc/unicode-normalize.cpp @ 1:20d02a178406 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
*: check in everything else yay
author Paper <paper@tflc.us>
date Mon, 05 Jan 2026 02:15:46 -0500
parents
children
comparison
equal deleted inserted replaced
0:e9bb126753e7 1:20d02a178406
1 #include "pfc-lite.h"
2 #include "unicode-normalize.h"
3 #include "string_base.h"
4 #include <map>
5
6
7
8 static constexpr uint16_t modifiers[] =
9 {0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0306, 0x0307, 0x0308, 0x0309, 0x030A, 0x030B, 0x030C, 0x030F, 0x0311, 0x0313, 0x0314, 0x031B, 0x0323, 0x0324, 0x0325, 0x0326, 0x0327, 0x0328, 0x032D, 0x032E, 0x0330, 0x0331, 0x0338, 0x0342, 0x0345, 0x05B4, 0x05B7, 0x05B8, 0x05B9, 0x05BC, 0x05BF, 0x05C1, 0x05C2, 0x0653, 0x0654, 0x0655, 0x093C, 0x09BC, 0x09BE, 0x09D7, 0x0A3C, 0x0B3C, 0x0B3E, 0x0B56, 0x0B57, 0x0BBE, 0x0BD7, 0x0C56, 0x0CC2, 0x0CD5, 0x0CD6, 0x0D3E, 0x0D57, 0x0DCA, 0x0DCF, 0x0DDF, 0x0F72, 0x0F74, 0x0F80, 0x0FB5, 0x0FB7, 0x102E, 0x1161, 0x1162, 0x1163, 0x1164, 0x1165, 0x1166, 0x1167, 0x1168, 0x1169, 0x116A, 0x116B, 0x116C, 0x116D, 0x116E, 0x116F, 0x1170, 0x1171, 0x1172, 0x1173, 0x1174, 0x1175, 0x11A8, 0x11A9, 0x11AA, 0x11AB, 0x11AC, 0x11AD, 0x11AE, 0x11AF, 0x11B0, 0x11B1, 0x11B2, 0x11B3, 0x11B4, 0x11B5, 0x11B6, 0x11B7, 0x11B8, 0x11B9, 0x11BA, 0x11BB, 0x11BC, 0x11BD, 0x11BE, 0x11BF, 0x11C0, 0x11C1, 0x11C2, 0x1B35, 0x3099, 0x309A};
10
11 static bool is_modifier( unsigned c ) {
12 if (c < modifiers[0]) return false; // common case, bail early
13 unsigned lo = 0, hi = (unsigned) std::size(modifiers);
14 do {
15 unsigned mid = (lo+hi)/2;
16 unsigned hit = modifiers[mid];
17 if ( c < hit ) hi = mid;
18 else if ( c > hit ) lo = mid + 1;
19 else return true;
20 } while(lo < hi);
21 return false;
22 }
23
24 static constexpr struct {
25 uint16_t form1;
26 uint16_t form2[2];
27 } normtable[] = {
28 {0x00C0, {0x0041,0x0300}},
29 {0x00C1, {0x0041,0x0301}},
30 {0x00C2, {0x0041,0x0302}},
31 {0x00C3, {0x0041,0x0303}},
32 {0x00C4, {0x0041,0x0308}},
33 {0x00C5, {0x0041,0x030A}},
34 {0x00C7, {0x0043,0x0327}},
35 {0x00C8, {0x0045,0x0300}},
36 {0x00C9, {0x0045,0x0301}},
37 {0x00CA, {0x0045,0x0302}},
38 {0x00CB, {0x0045,0x0308}},
39 {0x00CC, {0x0049,0x0300}},
40 {0x00CD, {0x0049,0x0301}},
41 {0x00CE, {0x0049,0x0302}},
42 {0x00CF, {0x0049,0x0308}},
43 {0x00D1, {0x004E,0x0303}},
44 {0x00D2, {0x004F,0x0300}},
45 {0x00D3, {0x004F,0x0301}},
46 {0x00D4, {0x004F,0x0302}},
47 {0x00D5, {0x004F,0x0303}},
48 {0x00D6, {0x004F,0x0308}},
49 {0x00D9, {0x0055,0x0300}},
50 {0x00DA, {0x0055,0x0301}},
51 {0x00DB, {0x0055,0x0302}},
52 {0x00DC, {0x0055,0x0308}},
53 {0x00DD, {0x0059,0x0301}},
54 {0x00E0, {0x0061,0x0300}},
55 {0x00E1, {0x0061,0x0301}},
56 {0x00E2, {0x0061,0x0302}},
57 {0x00E3, {0x0061,0x0303}},
58 {0x00E4, {0x0061,0x0308}},
59 {0x00E5, {0x0061,0x030A}},
60 {0x00E7, {0x0063,0x0327}},
61 {0x00E8, {0x0065,0x0300}},
62 {0x00E9, {0x0065,0x0301}},
63 {0x00EA, {0x0065,0x0302}},
64 {0x00EB, {0x0065,0x0308}},
65 {0x00EC, {0x0069,0x0300}},
66 {0x00ED, {0x0069,0x0301}},
67 {0x00EE, {0x0069,0x0302}},
68 {0x00EF, {0x0069,0x0308}},
69 {0x00F1, {0x006E,0x0303}},
70 {0x00F2, {0x006F,0x0300}},
71 {0x00F3, {0x006F,0x0301}},
72 {0x00F4, {0x006F,0x0302}},
73 {0x00F5, {0x006F,0x0303}},
74 {0x00F6, {0x006F,0x0308}},
75 {0x00F9, {0x0075,0x0300}},
76 {0x00FA, {0x0075,0x0301}},
77 {0x00FB, {0x0075,0x0302}},
78 {0x00FC, {0x0075,0x0308}},
79 {0x00FD, {0x0079,0x0301}},
80 {0x00FF, {0x0079,0x0308}},
81 {0x0100, {0x0041,0x0304}},
82 {0x0101, {0x0061,0x0304}},
83 {0x0102, {0x0041,0x0306}},
84 {0x0103, {0x0061,0x0306}},
85 {0x0104, {0x0041,0x0328}},
86 {0x0105, {0x0061,0x0328}},
87 {0x0106, {0x0043,0x0301}},
88 {0x0107, {0x0063,0x0301}},
89 {0x0108, {0x0043,0x0302}},
90 {0x0109, {0x0063,0x0302}},
91 {0x010A, {0x0043,0x0307}},
92 {0x010B, {0x0063,0x0307}},
93 {0x010C, {0x0043,0x030C}},
94 {0x010D, {0x0063,0x030C}},
95 {0x010E, {0x0044,0x030C}},
96 {0x010F, {0x0064,0x030C}},
97 {0x0112, {0x0045,0x0304}},
98 {0x0113, {0x0065,0x0304}},
99 {0x0114, {0x0045,0x0306}},
100 {0x0115, {0x0065,0x0306}},
101 {0x0116, {0x0045,0x0307}},
102 {0x0117, {0x0065,0x0307}},
103 {0x0118, {0x0045,0x0328}},
104 {0x0119, {0x0065,0x0328}},
105 {0x011A, {0x0045,0x030C}},
106 {0x011B, {0x0065,0x030C}},
107 {0x011C, {0x0047,0x0302}},
108 {0x011D, {0x0067,0x0302}},
109 {0x011E, {0x0047,0x0306}},
110 {0x011F, {0x0067,0x0306}},
111 {0x0120, {0x0047,0x0307}},
112 {0x0121, {0x0067,0x0307}},
113 {0x0122, {0x0047,0x0327}},
114 {0x0123, {0x0067,0x0327}},
115 {0x0124, {0x0048,0x0302}},
116 {0x0125, {0x0068,0x0302}},
117 {0x0128, {0x0049,0x0303}},
118 {0x0129, {0x0069,0x0303}},
119 {0x012A, {0x0049,0x0304}},
120 {0x012B, {0x0069,0x0304}},
121 {0x012C, {0x0049,0x0306}},
122 {0x012D, {0x0069,0x0306}},
123 {0x012E, {0x0049,0x0328}},
124 {0x012F, {0x0069,0x0328}},
125 {0x0130, {0x0049,0x0307}},
126 {0x0134, {0x004A,0x0302}},
127 {0x0135, {0x006A,0x0302}},
128 {0x0136, {0x004B,0x0327}},
129 {0x0137, {0x006B,0x0327}},
130 {0x0139, {0x004C,0x0301}},
131 {0x013A, {0x006C,0x0301}},
132 {0x013B, {0x004C,0x0327}},
133 {0x013C, {0x006C,0x0327}},
134 {0x013D, {0x004C,0x030C}},
135 {0x013E, {0x006C,0x030C}},
136 {0x0143, {0x004E,0x0301}},
137 {0x0144, {0x006E,0x0301}},
138 {0x0145, {0x004E,0x0327}},
139 {0x0146, {0x006E,0x0327}},
140 {0x0147, {0x004E,0x030C}},
141 {0x0148, {0x006E,0x030C}},
142 {0x014C, {0x004F,0x0304}},
143 {0x014D, {0x006F,0x0304}},
144 {0x014E, {0x004F,0x0306}},
145 {0x014F, {0x006F,0x0306}},
146 {0x0150, {0x004F,0x030B}},
147 {0x0151, {0x006F,0x030B}},
148 {0x0154, {0x0052,0x0301}},
149 {0x0155, {0x0072,0x0301}},
150 {0x0156, {0x0052,0x0327}},
151 {0x0157, {0x0072,0x0327}},
152 {0x0158, {0x0052,0x030C}},
153 {0x0159, {0x0072,0x030C}},
154 {0x015A, {0x0053,0x0301}},
155 {0x015B, {0x0073,0x0301}},
156 {0x015C, {0x0053,0x0302}},
157 {0x015D, {0x0073,0x0302}},
158 {0x015E, {0x0053,0x0327}},
159 {0x015F, {0x0073,0x0327}},
160 {0x0160, {0x0053,0x030C}},
161 {0x0161, {0x0073,0x030C}},
162 {0x0162, {0x0054,0x0327}},
163 {0x0163, {0x0074,0x0327}},
164 {0x0164, {0x0054,0x030C}},
165 {0x0165, {0x0074,0x030C}},
166 {0x0168, {0x0055,0x0303}},
167 {0x0169, {0x0075,0x0303}},
168 {0x016A, {0x0055,0x0304}},
169 {0x016B, {0x0075,0x0304}},
170 {0x016C, {0x0055,0x0306}},
171 {0x016D, {0x0075,0x0306}},
172 {0x016E, {0x0055,0x030A}},
173 {0x016F, {0x0075,0x030A}},
174 {0x0170, {0x0055,0x030B}},
175 {0x0171, {0x0075,0x030B}},
176 {0x0172, {0x0055,0x0328}},
177 {0x0173, {0x0075,0x0328}},
178 {0x0174, {0x0057,0x0302}},
179 {0x0175, {0x0077,0x0302}},
180 {0x0176, {0x0059,0x0302}},
181 {0x0177, {0x0079,0x0302}},
182 {0x0178, {0x0059,0x0308}},
183 {0x0179, {0x005A,0x0301}},
184 {0x017A, {0x007A,0x0301}},
185 {0x017B, {0x005A,0x0307}},
186 {0x017C, {0x007A,0x0307}},
187 {0x017D, {0x005A,0x030C}},
188 {0x017E, {0x007A,0x030C}},
189 {0x01A0, {0x004F,0x031B}},
190 {0x01A1, {0x006F,0x031B}},
191 {0x01AF, {0x0055,0x031B}},
192 {0x01B0, {0x0075,0x031B}},
193 {0x01CD, {0x0041,0x030C}},
194 {0x01CE, {0x0061,0x030C}},
195 {0x01CF, {0x0049,0x030C}},
196 {0x01D0, {0x0069,0x030C}},
197 {0x01D1, {0x004F,0x030C}},
198 {0x01D2, {0x006F,0x030C}},
199 {0x01D3, {0x0055,0x030C}},
200 {0x01D4, {0x0075,0x030C}},
201 {0x01E2, {0x00C6,0x0304}},
202 {0x01E3, {0x00E6,0x0304}},
203 {0x01E6, {0x0047,0x030C}},
204 {0x01E7, {0x0067,0x030C}},
205 {0x01E8, {0x004B,0x030C}},
206 {0x01E9, {0x006B,0x030C}},
207 {0x01EA, {0x004F,0x0328}},
208 {0x01EB, {0x006F,0x0328}},
209 {0x01EE, {0x01B7,0x030C}},
210 {0x01EF, {0x0292,0x030C}},
211 {0x01F0, {0x006A,0x030C}},
212 {0x01F4, {0x0047,0x0301}},
213 {0x01F5, {0x0067,0x0301}},
214 {0x01F8, {0x004E,0x0300}},
215 {0x01F9, {0x006E,0x0300}},
216 {0x01FC, {0x00C6,0x0301}},
217 {0x01FD, {0x00E6,0x0301}},
218 {0x01FE, {0x00D8,0x0301}},
219 {0x01FF, {0x00F8,0x0301}},
220 {0x0200, {0x0041,0x030F}},
221 {0x0201, {0x0061,0x030F}},
222 {0x0202, {0x0041,0x0311}},
223 {0x0203, {0x0061,0x0311}},
224 {0x0204, {0x0045,0x030F}},
225 {0x0205, {0x0065,0x030F}},
226 {0x0206, {0x0045,0x0311}},
227 {0x0207, {0x0065,0x0311}},
228 {0x0208, {0x0049,0x030F}},
229 {0x0209, {0x0069,0x030F}},
230 {0x020A, {0x0049,0x0311}},
231 {0x020B, {0x0069,0x0311}},
232 {0x020C, {0x004F,0x030F}},
233 {0x020D, {0x006F,0x030F}},
234 {0x020E, {0x004F,0x0311}},
235 {0x020F, {0x006F,0x0311}},
236 {0x0210, {0x0052,0x030F}},
237 {0x0211, {0x0072,0x030F}},
238 {0x0212, {0x0052,0x0311}},
239 {0x0213, {0x0072,0x0311}},
240 {0x0214, {0x0055,0x030F}},
241 {0x0215, {0x0075,0x030F}},
242 {0x0216, {0x0055,0x0311}},
243 {0x0217, {0x0075,0x0311}},
244 {0x0218, {0x0053,0x0326}},
245 {0x0219, {0x0073,0x0326}},
246 {0x021A, {0x0054,0x0326}},
247 {0x021B, {0x0074,0x0326}},
248 {0x021E, {0x0048,0x030C}},
249 {0x021F, {0x0068,0x030C}},
250 {0x0226, {0x0041,0x0307}},
251 {0x0227, {0x0061,0x0307}},
252 {0x0228, {0x0045,0x0327}},
253 {0x0229, {0x0065,0x0327}},
254 {0x022E, {0x004F,0x0307}},
255 {0x022F, {0x006F,0x0307}},
256 {0x0232, {0x0059,0x0304}},
257 {0x0233, {0x0079,0x0304}},
258 {0x0344, {0x0308,0x0301}},
259 {0x0385, {0x00A8,0x0301}},
260 {0x0386, {0x0391,0x0301}},
261 {0x0388, {0x0395,0x0301}},
262 {0x0389, {0x0397,0x0301}},
263 {0x038A, {0x0399,0x0301}},
264 {0x038C, {0x039F,0x0301}},
265 {0x038E, {0x03A5,0x0301}},
266 {0x038F, {0x03A9,0x0301}},
267 {0x03AA, {0x0399,0x0308}},
268 {0x03AB, {0x03A5,0x0308}},
269 {0x03AC, {0x03B1,0x0301}},
270 {0x03AD, {0x03B5,0x0301}},
271 {0x03AE, {0x03B7,0x0301}},
272 {0x03AF, {0x03B9,0x0301}},
273 {0x03CA, {0x03B9,0x0308}},
274 {0x03CB, {0x03C5,0x0308}},
275 {0x03CC, {0x03BF,0x0301}},
276 {0x03CD, {0x03C5,0x0301}},
277 {0x03CE, {0x03C9,0x0301}},
278 {0x03D3, {0x03D2,0x0301}},
279 {0x03D4, {0x03D2,0x0308}},
280 {0x0400, {0x0415,0x0300}},
281 {0x0401, {0x0415,0x0308}},
282 {0x0403, {0x0413,0x0301}},
283 {0x0407, {0x0406,0x0308}},
284 {0x040C, {0x041A,0x0301}},
285 {0x040D, {0x0418,0x0300}},
286 {0x040E, {0x0423,0x0306}},
287 {0x0419, {0x0418,0x0306}},
288 {0x0439, {0x0438,0x0306}},
289 {0x0450, {0x0435,0x0300}},
290 {0x0451, {0x0435,0x0308}},
291 {0x0453, {0x0433,0x0301}},
292 {0x0457, {0x0456,0x0308}},
293 {0x045C, {0x043A,0x0301}},
294 {0x045D, {0x0438,0x0300}},
295 {0x045E, {0x0443,0x0306}},
296 {0x0476, {0x0474,0x030F}},
297 {0x0477, {0x0475,0x030F}},
298 {0x04C1, {0x0416,0x0306}},
299 {0x04C2, {0x0436,0x0306}},
300 {0x04D0, {0x0410,0x0306}},
301 {0x04D1, {0x0430,0x0306}},
302 {0x04D2, {0x0410,0x0308}},
303 {0x04D3, {0x0430,0x0308}},
304 {0x04D6, {0x0415,0x0306}},
305 {0x04D7, {0x0435,0x0306}},
306 {0x04DA, {0x04D8,0x0308}},
307 {0x04DB, {0x04D9,0x0308}},
308 {0x04DC, {0x0416,0x0308}},
309 {0x04DD, {0x0436,0x0308}},
310 {0x04DE, {0x0417,0x0308}},
311 {0x04DF, {0x0437,0x0308}},
312 {0x04E2, {0x0418,0x0304}},
313 {0x04E3, {0x0438,0x0304}},
314 {0x04E4, {0x0418,0x0308}},
315 {0x04E5, {0x0438,0x0308}},
316 {0x04E6, {0x041E,0x0308}},
317 {0x04E7, {0x043E,0x0308}},
318 {0x04EA, {0x04E8,0x0308}},
319 {0x04EB, {0x04E9,0x0308}},
320 {0x04EC, {0x042D,0x0308}},
321 {0x04ED, {0x044D,0x0308}},
322 {0x04EE, {0x0423,0x0304}},
323 {0x04EF, {0x0443,0x0304}},
324 {0x04F0, {0x0423,0x0308}},
325 {0x04F1, {0x0443,0x0308}},
326 {0x04F2, {0x0423,0x030B}},
327 {0x04F3, {0x0443,0x030B}},
328 {0x04F4, {0x0427,0x0308}},
329 {0x04F5, {0x0447,0x0308}},
330 {0x04F8, {0x042B,0x0308}},
331 {0x04F9, {0x044B,0x0308}},
332 };
333
334
335 static const uint16_t * match( uint16_t c ) {
336 size_t lo = 0, hi = std::size(normtable);
337 do {
338 size_t mid = (lo+hi)/2;
339 auto & rec = normtable[mid];
340 if ( rec.form1 < c ) lo = mid + 1;
341 else if ( rec.form1 > c ) hi = mid;
342 else return rec.form2;
343 } while( lo < hi );
344 return nullptr;
345 }
346 namespace {
347 class shortener {
348 public:
349 shortener() {
350 for (auto& walk : normtable) {
351 m_data[walk.form2[0]][walk.form2[1]] = walk.form1;
352 }
353 }
354
355 std::map< uint16_t, std::map<uint16_t, uint16_t> > m_data;
356 };
357 }
358
359 namespace pfc {
360 pfc::string8 unicodeNormalizeD_Lite(const char* in) {
361 pfc::string8 ret; ret.prealloc(1024);
362 for (;; ) {
363 unsigned c;
364 auto d = utf8_decode_char(in, c);
365 if (d == 0) break;
366 in += d;
367 const uint16_t* m = nullptr;
368 if (c < 0x10000) {
369 m = match((uint16_t)c);
370 }
371 if (m != nullptr) {
372 ret.add_char(m[0]);
373 ret.add_char(m[1]);
374 } else {
375 ret.add_char(c);
376 }
377 }
378
379 return ret;
380 }
381 pfc::string8 unicodeNormalizeC_Lite(const char* in) {
382 static shortener g_shortener; auto& data = g_shortener.m_data;
383 pfc::string8 ret; ret.prealloc(strlen(in));
384 for (;; ) {
385 unsigned c;
386 auto d = pfc::utf8_decode_char(in, c);
387 if (d == 0) break;
388 in += d;
389 if ( c < 0x10000 ) {
390 auto iter1 = data.find((uint16_t)c);
391 if (iter1 != data.end()) {
392 unsigned next;
393 auto d2 = pfc::utf8_decode_char(in, next);
394 if (d2 != 0) {
395 auto& data2 = iter1->second;
396 auto iter2 = data2.find(next);
397 if (iter2 != data2.end()) {
398 in += d2;
399 ret.add_char(iter2->second); continue;
400 }
401 }
402 }
403 }
404 ret.add_char(c);
405 }
406 return ret;
407 }
408 bool stringContainsFormD(const char* in) {
409 for (;; ) {
410 unsigned c;
411 auto d = pfc::utf8_decode_char(in, c);
412 if (d == 0) return false;
413 if (is_modifier(c)) return true;
414 in += d;
415 }
416 }
417 }
418 #if ! defined(__APPLE__) && !defined(_WIN32)
419 namespace pfc {
420 pfc::string8 unicodeNormalizeD(const char* in) {
421 return unicodeNormalizeD_Lite(in);
422 }
423 pfc::string8 unicodeNormalizeC(const char* in) {
424 return unicodeNormalizeC_Lite(in);
425 }
426 }
427
428 #endif // #if ! defined(__APPLE__) && !defined(_WIN32)
429