jcs.org
Serenity Operating System
1/*
2 * Copyright (c) 2021, Tim Flynn <trflynn89@serenityos.org>
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6
7#include <LibTest/TestCase.h>
8
9#include <AK/Array.h>
10#include <AK/String.h>
11#include <AK/StringView.h>
12#include <AK/Types.h>
13#include <AK/Utf16View.h>
14
15TEST_CASE(decode_ascii)
16{
17 auto string = MUST(AK::utf8_to_utf16("Hello World!11"sv));
18 Utf16View view { string };
19
20 size_t valid_code_units = 0;
21 EXPECT(view.validate(valid_code_units));
22 EXPECT_EQ(valid_code_units, view.length_in_code_units());
23
24 auto expected = Array { (u32)72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 49, 49 };
25 EXPECT_EQ(expected.size(), view.length_in_code_points());
26
27 size_t i = 0;
28 for (u32 code_point : view) {
29 EXPECT_EQ(code_point, expected[i++]);
30 }
31 EXPECT_EQ(i, expected.size());
32}
33
34TEST_CASE(decode_utf8)
35{
36 auto string = MUST(AK::utf8_to_utf16("Привет, мир! 😀 γειά σου κόσμος こんにちは世界"sv));
37 Utf16View view { string };
38
39 size_t valid_code_units = 0;
40 EXPECT(view.validate(valid_code_units));
41 EXPECT_EQ(valid_code_units, view.length_in_code_units());
42
43 auto expected = Array { (u32)1055, 1088, 1080, 1074, 1077, 1090, 44, 32, 1084, 1080, 1088, 33, 32, 128512, 32, 947, 949, 953, 940, 32, 963, 959, 965, 32, 954, 972, 963, 956, 959, 962, 32, 12371, 12435, 12395, 12385, 12399, 19990, 30028 };
44 EXPECT_EQ(expected.size(), view.length_in_code_points());
45
46 size_t i = 0;
47 for (u32 code_point : view) {
48 EXPECT_EQ(code_point, expected[i++]);
49 }
50 EXPECT_EQ(i, expected.size());
51}
52
53TEST_CASE(encode_utf8)
54{
55 {
56 auto utf8_string = MUST("Привет, мир! 😀 γειά σου κόσμος こんにちは世界"_string);
57 auto string = MUST(AK::utf8_to_utf16(utf8_string));
58 Utf16View view { string };
59 EXPECT_EQ(MUST(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes)), utf8_string);
60 EXPECT_EQ(MUST(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No)), utf8_string);
61 }
62 {
63 auto encoded = Array { (u16)0xd83d };
64 Utf16View view { encoded };
65 EXPECT_EQ(MUST(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes)), "\xed\xa0\xbd"sv);
66 EXPECT_EQ(MUST(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No)), "\ufffd"sv);
67 }
68}
69
70TEST_CASE(decode_utf16)
71{
72 // Same string as the decode_utf8 test.
73 auto encoded = Array { (u16)0x041f, 0x0440, 0x0438, 0x0432, 0x0435, 0x0442, 0x002c, 0x0020, 0x043c, 0x0438, 0x0440, 0x0021, 0x0020, 0xd83d, 0xde00, 0x0020, 0x03b3, 0x03b5, 0x03b9, 0x03ac, 0x0020, 0x03c3, 0x03bf, 0x03c5, 0x0020, 0x03ba, 0x03cc, 0x03c3, 0x03bc, 0x03bf, 0x03c2, 0x0020, 0x3053, 0x3093, 0x306b, 0x3061, 0x306f, 0x4e16, 0x754c };
74
75 Utf16View view { encoded };
76 EXPECT_EQ(encoded.size(), view.length_in_code_units());
77
78 size_t valid_code_units = 0;
79 EXPECT(view.validate(valid_code_units));
80 EXPECT_EQ(valid_code_units, view.length_in_code_units());
81
82 auto expected = Array { (u32)1055, 1088, 1080, 1074, 1077, 1090, 44, 32, 1084, 1080, 1088, 33, 32, 128512, 32, 947, 949, 953, 940, 32, 963, 959, 965, 32, 954, 972, 963, 956, 959, 962, 32, 12371, 12435, 12395, 12385, 12399, 19990, 30028 };
83 EXPECT_EQ(expected.size(), view.length_in_code_points());
84
85 size_t i = 0;
86 for (u32 code_point : view) {
87 EXPECT_EQ(code_point, expected[i++]);
88 }
89 EXPECT_EQ(i, expected.size());
90}
91
92TEST_CASE(iterate_utf16)
93{
94 auto string = MUST(AK::utf8_to_utf16("Привет 😀"sv));
95 Utf16View view { string };
96 auto iterator = view.begin();
97
98 EXPECT(*iterator == 1055);
99 EXPECT(iterator.length_in_code_units() == 1);
100
101 EXPECT(++iterator != view.end());
102 EXPECT(*iterator == 1088);
103 EXPECT(iterator.length_in_code_units() == 1);
104
105 EXPECT(++iterator != view.end());
106 EXPECT(*iterator == 1080);
107 EXPECT(iterator.length_in_code_units() == 1);
108
109 EXPECT(++iterator != view.end());
110 EXPECT(*iterator == 1074);
111 EXPECT(iterator.length_in_code_units() == 1);
112
113 EXPECT(++iterator != view.end());
114 EXPECT(*iterator == 1077);
115 EXPECT(iterator.length_in_code_units() == 1);
116
117 EXPECT(++iterator != view.end());
118 EXPECT(*iterator == 1090);
119 EXPECT(iterator.length_in_code_units() == 1);
120
121 EXPECT(++iterator != view.end());
122 EXPECT(*iterator == 32);
123 EXPECT(iterator.length_in_code_units() == 1);
124
125 EXPECT(++iterator != view.end());
126 EXPECT(*iterator == 128512);
127 EXPECT(iterator.length_in_code_units() == 2);
128
129 EXPECT(++iterator == view.end());
130 EXPECT_CRASH("Dereferencing Utf16CodePointIterator which is at its end.", [&iterator] {
131 *iterator;
132 return Test::Crash::Failure::DidNotCrash;
133 });
134
135 EXPECT_CRASH("Incrementing Utf16CodePointIterator which is at its end.", [&iterator] {
136 ++iterator;
137 return Test::Crash::Failure::DidNotCrash;
138 });
139}
140
141TEST_CASE(validate_invalid_utf16)
142{
143 size_t valid_code_units = 0;
144 {
145 // Lonely high surrogate.
146 auto invalid = Array { (u16)0xd800 };
147 EXPECT(!Utf16View(invalid).validate(valid_code_units));
148 EXPECT(valid_code_units == 0);
149
150 invalid = Array { (u16)0xdbff };
151 EXPECT(!Utf16View(invalid).validate(valid_code_units));
152 EXPECT(valid_code_units == 0);
153 }
154 {
155 // Lonely low surrogate.
156 auto invalid = Array { (u16)0xdc00 };
157 EXPECT(!Utf16View(invalid).validate(valid_code_units));
158 EXPECT(valid_code_units == 0);
159
160 invalid = Array { (u16)0xdfff };
161 EXPECT(!Utf16View(invalid).validate(valid_code_units));
162 EXPECT(valid_code_units == 0);
163 }
164 {
165 // High surrogate followed by non-surrogate.
166 auto invalid = Array { (u16)0xd800, 0 };
167 EXPECT(!Utf16View(invalid).validate(valid_code_units));
168 EXPECT(valid_code_units == 0);
169
170 invalid = Array { (u16)0xd800, 0xe000 };
171 EXPECT(!Utf16View(invalid).validate(valid_code_units));
172 EXPECT(valid_code_units == 0);
173 }
174 {
175 // High surrogate followed by high surrogate.
176 auto invalid = Array { (u16)0xd800, 0xd800 };
177 EXPECT(!Utf16View(invalid).validate(valid_code_units));
178 EXPECT(valid_code_units == 0);
179
180 invalid = Array { (u16)0xd800, 0xdbff };
181 EXPECT(!Utf16View(invalid).validate(valid_code_units));
182 EXPECT(valid_code_units == 0);
183 }
184 {
185 // Valid UTF-16 followed by invalid code units.
186 auto invalid = Array { (u16)0x41, 0x41, 0xd800 };
187 EXPECT(!Utf16View(invalid).validate(valid_code_units));
188 EXPECT(valid_code_units == 2);
189
190 invalid = Array { (u16)0x41, 0x41, 0xd800 };
191 EXPECT(!Utf16View(invalid).validate(valid_code_units));
192 EXPECT(valid_code_units == 2);
193 }
194}
195
196TEST_CASE(decode_invalid_utf16)
197{
198 {
199 // Lonely high surrogate.
200 auto invalid = Array { (u16)0x41, 0x42, 0xd800 };
201
202 Utf16View view { invalid };
203 EXPECT_EQ(invalid.size(), view.length_in_code_units());
204
205 auto expected = Array { (u32)0x41, 0x42, 0xfffd };
206 EXPECT_EQ(expected.size(), view.length_in_code_points());
207
208 size_t i = 0;
209 for (u32 code_point : view) {
210 EXPECT_EQ(code_point, expected[i++]);
211 }
212 EXPECT_EQ(i, expected.size());
213 }
214 {
215 // Lonely low surrogate.
216 auto invalid = Array { (u16)0x41, 0x42, 0xdc00 };
217
218 Utf16View view { invalid };
219 EXPECT_EQ(invalid.size(), view.length_in_code_units());
220
221 auto expected = Array { (u32)0x41, 0x42, 0xfffd };
222 EXPECT_EQ(expected.size(), view.length_in_code_points());
223
224 size_t i = 0;
225 for (u32 code_point : view) {
226 EXPECT_EQ(code_point, expected[i++]);
227 }
228 EXPECT_EQ(i, expected.size());
229 }
230 {
231 // High surrogate followed by non-surrogate.
232 auto invalid = Array { (u16)0x41, 0x42, 0xd800, 0 };
233
234 Utf16View view { invalid };
235 EXPECT_EQ(invalid.size(), view.length_in_code_units());
236
237 auto expected = Array { (u32)0x41, 0x42, 0xfffd, 0 };
238 EXPECT_EQ(expected.size(), view.length_in_code_points());
239
240 size_t i = 0;
241 for (u32 code_point : view) {
242 EXPECT_EQ(code_point, expected[i++]);
243 }
244 EXPECT_EQ(i, expected.size());
245 }
246 {
247 // High surrogate followed by high surrogate.
248 auto invalid = Array { (u16)0x41, 0x42, 0xd800, 0xd800 };
249
250 Utf16View view { invalid };
251 EXPECT_EQ(invalid.size(), view.length_in_code_units());
252
253 auto expected = Array { (u32)0x41, 0x42, 0xfffd, 0xfffd };
254 EXPECT_EQ(expected.size(), view.length_in_code_points());
255
256 size_t i = 0;
257 for (u32 code_point : view) {
258 EXPECT_EQ(code_point, expected[i++]);
259 }
260 EXPECT_EQ(i, expected.size());
261 }
262}
263
264TEST_CASE(substring_view)
265{
266 auto string = MUST(AK::utf8_to_utf16("Привет 😀"sv));
267 {
268 Utf16View view { string };
269 view = view.substring_view(7, 2);
270
271 EXPECT(view.length_in_code_units() == 2);
272 EXPECT_EQ(MUST(view.to_utf8()), "😀"sv);
273 }
274 {
275 Utf16View view { string };
276 view = view.substring_view(7, 1);
277
278 EXPECT(view.length_in_code_units() == 1);
279 EXPECT_EQ(MUST(view.to_utf8(Utf16View::AllowInvalidCodeUnits::Yes)), "\xed\xa0\xbd"sv);
280 EXPECT_EQ(MUST(view.to_utf8(Utf16View::AllowInvalidCodeUnits::No)), "\ufffd"sv);
281 }
282}