v2 / vlib / x / json2 / scanner_test.v
374 lines · 336 sloc · 7.35 KB · e9a39321b12d20089f2b5642b5543717b425938a
Raw
1module json2
2
3fn test_str() {
4 mut sc := Scanner{
5 text: '"test"'.bytes()
6 }
7 tok := sc.scan()
8 assert tok.kind == .str
9 assert tok.lit.len == 4
10 assert tok.lit.bytestr() == 'test'
11}
12
13fn test_str_valid_unicode_escape() {
14 mut sc := Scanner{
15 text: r'"\u0048"'.bytes()
16 }
17 tok := sc.scan()
18 assert tok.kind == .str
19 assert tok.lit.len == 1
20 assert tok.lit.bytestr() == 'H'
21}
22
23fn test_str_valid_unicode_escape_2() {
24 mut sc := Scanner{
25 text: r'"\u2714"'.bytes()
26 }
27 tok := sc.scan()
28 assert tok.kind == .str
29 assert tok.lit.len == 3
30 assert tok.lit.bytestr() == '✔'
31}
32
33fn test_str_valid_escaped_backslashes_before_string_end() {
34 mut sc := Scanner{
35 text: r'"some_value \n [ ] { } ( ) , ; ? * = ! \\@ \\"'.bytes()
36 }
37 tok := sc.scan()
38 assert tok.kind == .str
39 assert tok.lit.bytestr() == 'some_value \n [ ] { } ( ) , ; ? * = ! \\@ \\'
40}
41
42fn test_str_invalid_escape() {
43 mut sc := Scanner{
44 text: r'"\z"'.bytes()
45 }
46 tok := sc.scan()
47 assert tok.kind == .error
48 assert tok.lit.bytestr() == 'invalid backslash escape'
49}
50
51fn test_str_invalid_must_be_escape() {
52 for ch in important_escapable_chars {
53 mut sc := Scanner{
54 text: [u8(`"`), `t`, ch, `"`]
55 }
56 tok := sc.scan()
57 assert tok.kind == .error
58 assert tok.lit.bytestr() == 'character must be escaped with a backslash, replace with: \\${valid_unicode_escapes[important_escapable_chars.index(ch)]}'
59 }
60}
61
62fn test_str_control_must_be_escape() {
63 for ch := u8(0); ch < 0x20; ch++ {
64 if ch in important_escapable_chars {
65 continue
66 }
67 mut sc := Scanner{
68 text: [u8(`"`), `t`, ch, `"`]
69 }
70 tok := sc.scan()
71 assert tok.kind == .error
72 assert tok.lit.bytestr() == 'character must be escaped with a unicode escape, replace with: \\u${ch:04x}'
73 }
74}
75
76fn test_str_invalid_unicode_escape() {
77 mut sc := Scanner{
78 text: r'"\u010G"'.bytes()
79 }
80 tok := sc.scan()
81 assert tok.kind == .error
82 assert tok.lit.bytestr() == '`G` is not a hex digit'
83}
84
85fn test_str_invalid_unicode_escape_len() {
86 mut sc := Scanner{
87 text: r'"\u001"'.bytes()
88 }
89 tok := sc.scan()
90 assert tok.kind == .error
91 assert tok.lit.bytestr() == 'unicode escape must have 4 hex digits'
92}
93
94fn test_str_invalid_uppercase_u() {
95 mut sc := Scanner{
96 text: r'"\U0000"'.bytes()
97 }
98 tok := sc.scan()
99 assert tok.kind == .error
100 assert tok.lit.bytestr() == 'unicode endpoints must be in lowercase `u`'
101}
102
103fn test_str_missing_closing_bracket() {
104 mut sc := Scanner{
105 text: '"incomplete'.bytes()
106 }
107 tok := sc.scan()
108 assert tok.kind == .error
109 assert tok.lit.bytestr() == 'missing double quotes in string closing'
110}
111
112fn test_int() {
113 mut sc := Scanner{
114 text: '10'.bytes()
115 }
116 tok := sc.scan()
117 assert tok.kind == .int
118 assert tok.lit.len == 2
119 assert tok.lit.bytestr() == '10'
120}
121
122fn test_int_negative() {
123 mut sc := Scanner{
124 text: '-10'.bytes()
125 }
126 tok := sc.scan()
127 assert tok.kind == .int
128 assert tok.lit.len == 3
129 assert tok.lit.bytestr() == '-10'
130}
131
132fn test_float() {
133 mut sc := Scanner{
134 text: '123.400'.bytes()
135 }
136 tok := sc.scan()
137 assert tok.kind == .float
138 assert tok.lit.len == 7
139 assert tok.lit.bytestr() == '123.400'
140}
141
142fn test_float_negative() {
143 mut sc := Scanner{
144 text: '-123.400'.bytes()
145 }
146 tok := sc.scan()
147 assert tok.kind == .float
148 assert tok.lit.len == 8
149 assert tok.lit.bytestr() == '-123.400'
150}
151
152fn test_int_exp() {
153 mut sc := Scanner{
154 text: '1E22'.bytes()
155 }
156 tok := sc.scan()
157 assert tok.kind == .int
158 assert tok.lit.len == 4
159 assert tok.lit.bytestr() == '1E22'
160}
161
162fn test_int_exp_negative() {
163 mut sc := Scanner{
164 text: '1E-2'.bytes()
165 }
166 tok := sc.scan()
167 assert tok.kind == .int
168 assert tok.lit.len == 4
169 assert tok.lit.bytestr() == '1E-2'
170}
171
172fn test_int_exp_positive() {
173 mut sc := Scanner{
174 text: '1E+2'.bytes()
175 }
176 tok := sc.scan()
177 assert tok.kind == .int
178 assert tok.lit.len == 4
179 assert tok.lit.bytestr() == '1E+2'
180}
181
182fn test_float_exp() {
183 mut sc := Scanner{
184 text: '123.456e78'.bytes()
185 }
186 tok := sc.scan()
187 assert tok.kind == .float
188 assert tok.lit.len == 10
189 assert tok.lit.bytestr() == '123.456e78'
190}
191
192fn test_float_exp_negative() {
193 mut sc := Scanner{
194 text: '20.56e-5'.bytes()
195 }
196 tok := sc.scan()
197 assert tok.kind == .float
198 assert tok.lit.len == 8
199 assert tok.lit.bytestr() == '20.56e-5'
200}
201
202fn test_float_exp_positive() {
203 mut sc := Scanner{
204 text: '20.56e+5'.bytes()
205 }
206 tok := sc.scan()
207 assert tok.kind == .float
208 assert tok.lit.len == 8
209 assert tok.lit.bytestr() == '20.56e+5'
210}
211
212fn test_number_with_space() {
213 mut sc := Scanner{
214 text: ' 4'.bytes()
215 }
216 tok := sc.scan()
217 assert tok.kind == .int
218 assert tok.lit.len == 1
219 assert tok.lit.bytestr() == '4'
220}
221
222fn test_number_invalid_leading_zero() {
223 mut sc := Scanner{
224 text: '0010'.bytes()
225 }
226 tok := sc.scan()
227 assert tok.kind == .error
228 assert tok.lit.bytestr() == 'leading zeroes in a number are not allowed'
229}
230
231fn test_number_invalid_leading_zero_negative() {
232 mut sc := Scanner{
233 text: '-0010'.bytes()
234 }
235 tok := sc.scan()
236 assert tok.kind == .error
237 assert tok.lit.bytestr() == 'leading zeroes in a number are not allowed'
238}
239
240fn test_number_invalid_start_char() {
241 mut sc := Scanner{
242 text: '+1'.bytes()
243 }
244 tok := sc.scan()
245 assert tok.kind == .error
246 assert tok.lit.bytestr() == 'invalid token `+`'
247}
248
249fn test_number_invalid_char() {
250 mut sc := Scanner{
251 text: '122x'.bytes()
252 }
253 sc.scan()
254 tok := sc.scan()
255 assert tok.kind == .error
256 assert tok.lit.bytestr() == 'invalid token `x`'
257}
258
259fn test_number_invalid_char_float() {
260 mut sc := Scanner{
261 text: '122x.1'.bytes()
262 }
263 sc.scan()
264 tok := sc.scan()
265 assert tok.kind == .error
266 assert tok.lit.bytestr() == 'invalid token `x`'
267}
268
269fn test_number_invalid_multiple_dot() {
270 mut sc := Scanner{
271 text: '122.108.10'.bytes()
272 }
273 sc.scan()
274 tok := sc.scan()
275 assert tok.kind == .error
276 assert tok.lit.bytestr() == 'invalid token `.`'
277}
278
279fn test_number_invalid_exp() {
280 mut sc := Scanner{
281 text: '0.3e'.bytes()
282 }
283 tok := sc.scan()
284 assert tok.kind == .error
285 assert tok.lit.bytestr() == 'invalid exponent'
286}
287
288fn test_number_invalid_exp_with_sign() {
289 mut sc := Scanner{
290 text: '0.3e+'.bytes()
291 }
292 tok := sc.scan()
293 assert tok.kind == .error
294 assert tok.lit.bytestr() == 'invalid exponent'
295}
296
297fn test_number_invalid_zero_exp() {
298 mut sc := Scanner{
299 text: '0e'.bytes()
300 }
301 tok := sc.scan()
302 assert tok.kind == .error
303 assert tok.lit.bytestr() == 'invalid exponent'
304}
305
306fn test_number_invalid_dot_exp() {
307 mut sc := Scanner{
308 text: '0.e'.bytes()
309 }
310 tok := sc.scan()
311 assert tok.kind == .error
312 assert tok.lit.bytestr() == 'invalid float'
313}
314
315fn test_number_invalid_double_exp() {
316 mut sc := Scanner{
317 text: '2eE'.bytes()
318 }
319 sc.scan()
320 tok := sc.scan()
321 assert tok.kind == .error
322 assert tok.lit.bytestr() == 'invalid token `E`'
323}
324
325fn test_null() {
326 mut sc := Scanner{
327 text: 'null'.bytes()
328 }
329 tok := sc.scan()
330 assert tok.kind == .null
331 assert tok.lit.len == 4
332 assert tok.lit.bytestr() == 'null'
333}
334
335fn test_bool_true() {
336 mut sc := Scanner{
337 text: 'true'.bytes()
338 }
339 tok := sc.scan()
340 assert tok.kind == .bool
341 assert tok.lit.len == 4
342 assert tok.lit.bytestr() == 'true'
343}
344
345fn test_bool_false() {
346 mut sc := Scanner{
347 text: 'false'.bytes()
348 }
349 tok := sc.scan()
350 assert tok.kind == .bool
351 assert tok.lit.len == 5
352 assert tok.lit.bytestr() == 'false'
353}
354
355fn test_json_with_whitespace_start() {
356 mut sc := Scanner{
357 text: ' \n \n\t {'.bytes()
358 }
359 tok := sc.scan()
360 eprintln(tok)
361 assert tok.kind == .lcbr
362 assert tok.lit.len == 0
363}
364
365fn test_json_with_whitespace_end() {
366 mut sc := Scanner{
367 text: '} \n\t'.bytes()
368 }
369 tok := sc.scan()
370 assert tok.kind == .rcbr
371 tok2 := sc.scan()
372 eprintln(tok2)
373 assert tok2.kind == .eof
374}
375