Skip to content

Commit 4acc273

Browse files
watildedanielleadams
watilde
authored and
danielleadams
committed
test: update wpt encoding
Refs: web-platform-tests/wpt#26385 PR-URL: #36659 Reviewed-By: Michaël Zasso <[email protected]> Reviewed-By: Rich Trott <[email protected]>
1 parent 986d5ac commit 4acc273

File tree

7 files changed

+374
-2
lines changed

7 files changed

+374
-2
lines changed

‎test/fixtures/wpt/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ See [test/wpt](../../wpt/README.md) for information on how these tests are run.
1111
Last update:
1212

1313
- console: https://github.com/web-platform-tests/wpt/tree/3b1f72e99a/console
14-
- encoding: https://github.com/web-platform-tests/wpt/tree/1821fb5f77/encoding
14+
- encoding: https://github.com/web-platform-tests/wpt/tree/3c9820d1cc/encoding
1515
- url: https://github.com/web-platform-tests/wpt/tree/1783c9bccf/url
1616
- resources: https://github.com/web-platform-tests/wpt/tree/001e50de41/resources
1717
- interfaces: https://github.com/web-platform-tests/wpt/tree/8719553b2d/interfaces

‎test/fixtures/wpt/encoding/legacy-mb-schinese/gb18030/gb18030-decoder.html

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<!doctype html>
2+
<script src=/resources/testharness.js></script>
3+
<script src=/resources/testharnessreport.js></script>
4+
<script src=resources/ranges.js></script>
5+
<script>
6+
const decode = (input, output, desc) => {
7+
test(function() {
8+
for (encoding of ["gb18030", "gbk"])
9+
assert_equals(new TextDecoder(encoding).decode(new Uint8Array(input)), output)
10+
}, "gb18030 decoder: " + desc)
11+
}
12+
13+
decode([115], "s", "ASCII");
14+
decode([0x80], "\u20AC", "euro");
15+
decode([0xFF], "\uFFFD", "initial byte out of accepted ranges");
16+
decode([0x81], "\uFFFD", "end of queue, gb18030 first not 0");
17+
decode([0x81, 0x28], "\ufffd(", "two bytes 0x81 0x28");
18+
decode([0x81, 0x40], "\u4E02", "two bytes 0x81 0x40");
19+
decode([0x81, 0x7E], "\u4E8A", "two bytes 0x81 0x7e");
20+
decode([0x81, 0x7F], "\ufffd\u007f", "two bytes 0x81 0x7f");
21+
decode([0x81, 0x80], "\u4E90", "two bytes 0x81 0x80");
22+
decode([0x81, 0xFE], "\u4FA2", "two bytes 0x81 0xFE");
23+
decode([0x81, 0xFF], "\ufffd", "two bytes 0x81 0xFF");
24+
decode([0xFE, 0x40], "\uFA0C", "two bytes 0xFE 0x40");
25+
decode([0xFE, 0xFE], "\uE4C5", "two bytes 0xFE 0xFE");
26+
decode([0xFE, 0xFF], "\ufffd", "two bytes 0xFE 0xFF");
27+
decode([0x81, 0x30], "\ufffd", "two bytes 0x81 0x30");
28+
decode([0x81, 0x30, 0xFE], "\ufffd", "three bytes 0x81 0x30 0xFE");
29+
decode([0x81, 0x30, 0xFF], "\ufffd0\ufffd", "three bytes 0x81 0x30 0xFF");
30+
decode([0x81, 0x30, 0xFE, 0x29], "\ufffd0\ufffd)", "four bytes 0x81 0x30 0xFE 0x29");
31+
decode([0xFE, 0x39, 0xFE, 0x39], "\ufffd", "four bytes 0xFE 0x39 0xFE 0x39");
32+
decode([0x81, 0x35, 0xF4, 0x36], "\u1E3E", "pointer 7458");
33+
decode([0x81, 0x35, 0xF4, 0x37], "\ue7c7", "pointer 7457");
34+
decode([0x81, 0x35, 0xF4, 0x38], "\u1E40", "pointer 7459");
35+
decode([0x84, 0x31, 0xA4, 0x39], "\uffff", "pointer 39419");
36+
decode([0x84, 0x31, 0xA5, 0x30], "\ufffd", "pointer 39420");
37+
decode([0x8F, 0x39, 0xFE, 0x39], "\ufffd", "pointer 189999");
38+
decode([0x90, 0x30, 0x81, 0x30], "\u{10000}", "pointer 189000");
39+
decode([0xE3, 0x32, 0x9A, 0x35], "\u{10FFFF}", "pointer 1237575");
40+
decode([0xE3, 0x32, 0x9A, 0x36], "\ufffd", "pointer 1237576");
41+
decode([0x83, 0x36, 0xC8, 0x30], "\uE7C8", "legacy ICU special case 1");
42+
decode([0xA1, 0xAD], "\u2026", "legacy ICU special case 2");
43+
decode([0xA1, 0xAB], "\uFF5E", "legacy ICU special case 3");
44+
45+
let i = 0;
46+
for (const range of ranges) {
47+
const pointer = range[0];
48+
decode([
49+
Math.floor(pointer / 12600) + 0x81,
50+
Math.floor((pointer % 12600) / 1260) + 0x30,
51+
Math.floor((pointer % 1260) / 10) + 0x81,
52+
pointer % 10 + 0x30
53+
], range[1], "range " + i++);
54+
}
55+
</script>

‎test/fixtures/wpt/encoding/legacy-mb-schinese/gb18030/gb18030-encoder.html

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
<!doctype html>
2+
<meta charset=gb18030>
3+
<script src=/resources/testharness.js></script>
4+
<script src=/resources/testharnessreport.js></script>
5+
<script src=resources/ranges.js></script>
6+
<script>
7+
const encode = (input, output, desc) => {
8+
test(function() {
9+
const a = document.createElement("a"); // <a> uses document encoding for URL's query
10+
a.href = "https://example.com/?" + input;
11+
assert_equals(a.search.substr(1), output); // remove leading "?"
12+
}, "gb18030 encoder: " + desc);
13+
}
14+
15+
encode("s", "s", "very basic");
16+
encode("\u20AC", "%A2%E3", "Euro");
17+
encode("\u4E02", "%81@", "character");
18+
encode("\uE4C6", "%A1@", "PUA");
19+
encode("\uE4C5", "%FE%FE", "PUA #2");
20+
encode("\uE5E5", "%26%2358853%3B", "PUA #3");
21+
encode("\ud83d\udca9", "%949%DA3", "poo");
22+
encode("\uE7C7", "%815%F47", "Ranges pointer special case");
23+
encode("\uE7C8", "%836%C80", "legacy ICU special case 1");
24+
encode("\u2026", "%A1%AD", "legacy ICU special case 2");
25+
encode("\uFF5E", "%A1%AB", "legacy ICU special case 3");
26+
27+
const upperCaseNibble = x => {
28+
return Math.floor(x).toString(16).toUpperCase();
29+
}
30+
31+
const encodePointer = pointer => {
32+
const firstByte = Math.floor(pointer / 12600) + 0x81;
33+
const thirdByte = Math.floor((pointer % 1260) / 10) + 0x81;
34+
return "%"
35+
+ upperCaseNibble(firstByte / 16)
36+
+ upperCaseNibble(firstByte % 16)
37+
+ String.fromCharCode(Math.floor((pointer % 12600) / 1260) + 0x30)
38+
+ "%"
39+
+ upperCaseNibble(thirdByte / 16)
40+
+ upperCaseNibble(thirdByte % 16)
41+
+ String.fromCharCode(pointer % 10 + 0x30);
42+
}
43+
44+
let i = 0;
45+
for (const range of ranges) {
46+
encode(range[1], encodePointer(range[0]), "range " + i++);
47+
}
48+
</script>

‎test/fixtures/wpt/encoding/legacy-mb-schinese/gb18030/resources/ranges.js

+210
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
// Based on https://encoding.spec.whatwg.org/index-gb18030-ranges.txt
2+
const ranges = [
3+
[0, "\u0080"],
4+
[36, "\u00A5"],
5+
[38, "\u00A9"],
6+
[45, "\u00B2"],
7+
[50, "\u00B8"],
8+
[81, "\u00D8"],
9+
[89, "\u00E2"],
10+
[95, "\u00EB"],
11+
[96, "\u00EE"],
12+
[100, "\u00F4"],
13+
[103, "\u00F8"],
14+
[104, "\u00FB"],
15+
[105, "\u00FD"],
16+
[109, "\u0102"],
17+
[126, "\u0114"],
18+
[133, "\u011C"],
19+
[148, "\u012C"],
20+
[172, "\u0145"],
21+
[175, "\u0149"],
22+
[179, "\u014E"],
23+
[208, "\u016C"],
24+
[306, "\u01CF"],
25+
[307, "\u01D1"],
26+
[308, "\u01D3"],
27+
[309, "\u01D5"],
28+
[310, "\u01D7"],
29+
[311, "\u01D9"],
30+
[312, "\u01DB"],
31+
[313, "\u01DD"],
32+
[341, "\u01FA"],
33+
[428, "\u0252"],
34+
[443, "\u0262"],
35+
[544, "\u02C8"],
36+
[545, "\u02CC"],
37+
[558, "\u02DA"],
38+
[741, "\u03A2"],
39+
[742, "\u03AA"],
40+
[749, "\u03C2"],
41+
[750, "\u03CA"],
42+
[805, "\u0402"],
43+
[819, "\u0450"],
44+
[820, "\u0452"],
45+
[7922, "\u2011"],
46+
[7924, "\u2017"],
47+
[7925, "\u201A"],
48+
[7927, "\u201E"],
49+
[7934, "\u2027"],
50+
[7943, "\u2031"],
51+
[7944, "\u2034"],
52+
[7945, "\u2036"],
53+
[7950, "\u203C"],
54+
[8062, "\u20AD"],
55+
[8148, "\u2104"],
56+
[8149, "\u2106"],
57+
[8152, "\u210A"],
58+
[8164, "\u2117"],
59+
[8174, "\u2122"],
60+
[8236, "\u216C"],
61+
[8240, "\u217A"],
62+
[8262, "\u2194"],
63+
[8264, "\u219A"],
64+
[8374, "\u2209"],
65+
[8380, "\u2210"],
66+
[8381, "\u2212"],
67+
[8384, "\u2216"],
68+
[8388, "\u221B"],
69+
[8390, "\u2221"],
70+
[8392, "\u2224"],
71+
[8393, "\u2226"],
72+
[8394, "\u222C"],
73+
[8396, "\u222F"],
74+
[8401, "\u2238"],
75+
[8406, "\u223E"],
76+
[8416, "\u2249"],
77+
[8419, "\u224D"],
78+
[8424, "\u2253"],
79+
[8437, "\u2262"],
80+
[8439, "\u2268"],
81+
[8445, "\u2270"],
82+
[8482, "\u2296"],
83+
[8485, "\u229A"],
84+
[8496, "\u22A6"],
85+
[8521, "\u22C0"],
86+
[8603, "\u2313"],
87+
[8936, "\u246A"],
88+
[8946, "\u249C"],
89+
[9046, "\u254C"],
90+
[9050, "\u2574"],
91+
[9063, "\u2590"],
92+
[9066, "\u2596"],
93+
[9076, "\u25A2"],
94+
[9092, "\u25B4"],
95+
[9100, "\u25BE"],
96+
[9108, "\u25C8"],
97+
[9111, "\u25CC"],
98+
[9113, "\u25D0"],
99+
[9131, "\u25E6"],
100+
[9162, "\u2607"],
101+
[9164, "\u260A"],
102+
[9218, "\u2641"],
103+
[9219, "\u2643"],
104+
[11329, "\u2E82"],
105+
[11331, "\u2E85"],
106+
[11334, "\u2E89"],
107+
[11336, "\u2E8D"],
108+
[11346, "\u2E98"],
109+
[11361, "\u2EA8"],
110+
[11363, "\u2EAB"],
111+
[11366, "\u2EAF"],
112+
[11370, "\u2EB4"],
113+
[11372, "\u2EB8"],
114+
[11375, "\u2EBC"],
115+
[11389, "\u2ECB"],
116+
[11682, "\u2FFC"],
117+
[11686, "\u3004"],
118+
[11687, "\u3018"],
119+
[11692, "\u301F"],
120+
[11694, "\u302A"],
121+
[11714, "\u303F"],
122+
[11716, "\u3094"],
123+
[11723, "\u309F"],
124+
[11725, "\u30F7"],
125+
[11730, "\u30FF"],
126+
[11736, "\u312A"],
127+
[11982, "\u322A"],
128+
[11989, "\u3232"],
129+
[12102, "\u32A4"],
130+
[12336, "\u3390"],
131+
[12348, "\u339F"],
132+
[12350, "\u33A2"],
133+
[12384, "\u33C5"],
134+
[12393, "\u33CF"],
135+
[12395, "\u33D3"],
136+
[12397, "\u33D6"],
137+
[12510, "\u3448"],
138+
[12553, "\u3474"],
139+
[12851, "\u359F"],
140+
[12962, "\u360F"],
141+
[12973, "\u361B"],
142+
[13738, "\u3919"],
143+
[13823, "\u396F"],
144+
[13919, "\u39D1"],
145+
[13933, "\u39E0"],
146+
[14080, "\u3A74"],
147+
[14298, "\u3B4F"],
148+
[14585, "\u3C6F"],
149+
[14698, "\u3CE1"],
150+
[15583, "\u4057"],
151+
[15847, "\u4160"],
152+
[16318, "\u4338"],
153+
[16434, "\u43AD"],
154+
[16438, "\u43B2"],
155+
[16481, "\u43DE"],
156+
[16729, "\u44D7"],
157+
[17102, "\u464D"],
158+
[17122, "\u4662"],
159+
[17315, "\u4724"],
160+
[17320, "\u472A"],
161+
[17402, "\u477D"],
162+
[17418, "\u478E"],
163+
[17859, "\u4948"],
164+
[17909, "\u497B"],
165+
[17911, "\u497E"],
166+
[17915, "\u4984"],
167+
[17916, "\u4987"],
168+
[17936, "\u499C"],
169+
[17939, "\u49A0"],
170+
[17961, "\u49B8"],
171+
[18664, "\u4C78"],
172+
[18703, "\u4CA4"],
173+
[18814, "\u4D1A"],
174+
[18962, "\u4DAF"],
175+
[19043, "\u9FA6"],
176+
[33469, "\uE76C"],
177+
[33470, "\uE7C8"],
178+
[33471, "\uE7E7"],
179+
[33484, "\uE815"],
180+
[33485, "\uE819"],
181+
[33490, "\uE81F"],
182+
[33497, "\uE827"],
183+
[33501, "\uE82D"],
184+
[33505, "\uE833"],
185+
[33513, "\uE83C"],
186+
[33520, "\uE844"],
187+
[33536, "\uE856"],
188+
[33550, "\uE865"],
189+
[37845, "\uF92D"],
190+
[37921, "\uF97A"],
191+
[37948, "\uF996"],
192+
[38029, "\uF9E8"],
193+
[38038, "\uF9F2"],
194+
[38064, "\uFA10"],
195+
[38065, "\uFA12"],
196+
[38066, "\uFA15"],
197+
[38069, "\uFA19"],
198+
[38075, "\uFA22"],
199+
[38076, "\uFA25"],
200+
[38078, "\uFA2A"],
201+
[39108, "\uFE32"],
202+
[39109, "\uFE45"],
203+
[39113, "\uFE53"],
204+
[39114, "\uFE58"],
205+
[39115, "\uFE67"],
206+
[39116, "\uFE6C"],
207+
[39265, "\uFF5F"],
208+
[39394, "\uFFE6"],
209+
[189000, "\u{10000}"]
210+
];

‎test/fixtures/wpt/encoding/legacy-mb-schinese/gbk/gbk-decoder.html

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
<!doctype html>
2+
<script src=/resources/testharness.js></script>
3+
<script src=/resources/testharnessreport.js></script>
4+
<script>
5+
const gbkPointers = [
6+
6432, 7533, 7536, 7672, 7673, 7674, 7675, 7676, 7677, 7678, 7679, 7680, 7681, 7682, 7683, 7684,
7+
23766, 23770, 23771, 23772, 23773, 23774, 23776, 23777, 23778, 23779, 23780, 23781, 23782, 23784, 23785, 23786,
8+
23787, 23790, 23791, 23792, 23793, 23796, 23797, 23798, 23799, 23800, 23801, 23802, 23803, 23805, 23806, 23807,
9+
23808, 23809, 23810, 23811, 23813, 23814, 23815, 23816, 23817, 23818, 23819, 23820, 23821, 23822, 23823, 23824,
10+
23825, 23826, 23827, 23828, 23831, 23832, 23833, 23834, 23835, 23836, 23837, 23838, 23839, 23840, 23841, 23842,
11+
23843, 23844
12+
];
13+
const codePoints = [
14+
0x20ac, 0x1e3f, 0x01f9, 0x303e, 0x2ff0, 0x2ff1, 0x2ff2, 0x2ff3, 0x2ff4, 0x2ff5, 0x2ff6, 0x2ff7, 0x2ff8, 0x2ff9, 0x2ffa, 0x2ffb,
15+
0x2e81, 0x2e84, 0x3473, 0x3447, 0x2e88, 0x2e8b, 0x359e, 0x361a, 0x360e, 0x2e8c, 0x2e97, 0x396e, 0x3918, 0x39cf, 0x39df, 0x3a73,
16+
0x39d0, 0x3b4e, 0x3c6e, 0x3ce0, 0x2ea7, 0x2eaa, 0x4056, 0x415f, 0x2eae, 0x4337, 0x2eb3, 0x2eb6, 0x2eb7, 0x43b1, 0x43ac, 0x2ebb,
17+
0x43dd, 0x44d6, 0x4661, 0x464c, 0x4723, 0x4729, 0x477c, 0x478d, 0x2eca, 0x4947, 0x497a, 0x497d, 0x4982, 0x4983, 0x4985, 0x4986,
18+
0x499f, 0x499b, 0x49b7, 0x49b6, 0x4ca3, 0x4c9f, 0x4ca0, 0x4ca1, 0x4c77, 0x4ca2, 0x4d13, 0x4d14, 0x4d15, 0x4d16, 0x4d17, 0x4d18,
19+
0x4d19, 0x4dae
20+
];
21+
22+
for (let i = 0; i < gbkPointers.length; i++) {
23+
const pointer = gbkPointers[i];
24+
test(function() {
25+
const lead = pointer / 190 + 0x81;
26+
const trail = pointer % 190;
27+
const offset = trail < 0x3F ? 0x40 : 0x41;
28+
const encoded = [lead, trail + offset];
29+
const decoded = new TextDecoder("GBK").decode(new Uint8Array(encoded)).charCodeAt(0);
30+
assert_equals(decoded, codePoints[i]);
31+
}, "gbk pointer: " + pointer)
32+
}
33+
</script>

‎test/fixtures/wpt/encoding/legacy-mb-schinese/gbk/gbk-encoder.html

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<!doctype html>
2+
<meta charset=gbk> <!-- if the server overrides this, it is stupid, as this is a testsuite -->
3+
<script src=/resources/testharness.js></script>
4+
<script src=/resources/testharnessreport.js></script>
5+
<script>
6+
function encode(input, output, desc) {
7+
test(function() {
8+
const a = document.createElement("a") // <a> uses document encoding for URL's query
9+
a.href = "https://example.com/?" + input;
10+
assert_equals(a.search.substr(1), output); // remove leading "?"
11+
}, "gbk encoder: " + desc);
12+
}
13+
14+
encode("s", "s", "very basic");
15+
encode("\u20AC", "%80", "Euro");
16+
encode("\u4E02", "%81@", "character");
17+
encode("\uE4C6", "%A1@", "PUA");
18+
encode("\uE4C5", "%FE%FE", "PUA #2");
19+
encode("\ud83d\udca9", "%26%23128169%3B", "poo");
20+
encode("\uE7C8", "%26%2359336%3B", "legacy ICU special case 1");
21+
encode("\u2026", "%A1%AD", "legacy ICU special case 2");
22+
encode("\uFF5E", "%A1%AB", "legacy ICU special case 3");
23+
encode("\u00A5", "%26%23165%3B", "legacy WebKit case 1");
24+
encode("\u22EF", "%26%238943%3B", "legacy WebKit case 2");
25+
encode("\u301C", "%26%2312316%3B", "legacy WebKit case 3");
26+
</script>

‎test/fixtures/wpt/versions.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"path": "console"
55
},
66
"encoding": {
7-
"commit": "1821fb5f77723b5361058c6a8ed0b71f9d2d6b8d",
7+
"commit": "3c9820d1cc5d9d2627c26ef1268b6d54a35adf22",
88
"path": "encoding"
99
},
1010
"url": {

0 commit comments

Comments
 (0)