| 1 | module json2 |
| 2 | |
| 3 | fn test_str() { |
| 4 | mut sc := Scanner{ |
| 5 | text: '"test"'.bytes() |
| 6 | } |
| 7 | tok := sc.scan() |
| 8 | assert tok.kind == .str |
| 9 | assert tok.lit.len == 4 |
| 10 | assert tok.lit.bytestr() == 'test' |
| 11 | } |
| 12 | |
| 13 | fn test_str_valid_unicode_escape() { |
| 14 | mut sc := Scanner{ |
| 15 | text: r'"\u0048"'.bytes() |
| 16 | } |
| 17 | tok := sc.scan() |
| 18 | assert tok.kind == .str |
| 19 | assert tok.lit.len == 1 |
| 20 | assert tok.lit.bytestr() == 'H' |
| 21 | } |
| 22 | |
| 23 | fn test_str_valid_unicode_escape_2() { |
| 24 | mut sc := Scanner{ |
| 25 | text: r'"\u2714"'.bytes() |
| 26 | } |
| 27 | tok := sc.scan() |
| 28 | assert tok.kind == .str |
| 29 | assert tok.lit.len == 3 |
| 30 | assert tok.lit.bytestr() == '✔' |
| 31 | } |
| 32 | |
| 33 | fn test_str_valid_escaped_backslashes_before_string_end() { |
| 34 | mut sc := Scanner{ |
| 35 | text: r'"some_value \n [ ] { } ( ) , ; ? * = ! \\@ \\"'.bytes() |
| 36 | } |
| 37 | tok := sc.scan() |
| 38 | assert tok.kind == .str |
| 39 | assert tok.lit.bytestr() == 'some_value \n [ ] { } ( ) , ; ? * = ! \\@ \\' |
| 40 | } |
| 41 | |
| 42 | fn test_str_invalid_escape() { |
| 43 | mut sc := Scanner{ |
| 44 | text: r'"\z"'.bytes() |
| 45 | } |
| 46 | tok := sc.scan() |
| 47 | assert tok.kind == .error |
| 48 | assert tok.lit.bytestr() == 'invalid backslash escape' |
| 49 | } |
| 50 | |
| 51 | fn test_str_invalid_must_be_escape() { |
| 52 | for ch in important_escapable_chars { |
| 53 | mut sc := Scanner{ |
| 54 | text: [u8(`"`), `t`, ch, `"`] |
| 55 | } |
| 56 | tok := sc.scan() |
| 57 | assert tok.kind == .error |
| 58 | assert tok.lit.bytestr() == 'character must be escaped with a backslash, replace with: \\${valid_unicode_escapes[important_escapable_chars.index(ch)]}' |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | fn test_str_control_must_be_escape() { |
| 63 | for ch := u8(0); ch < 0x20; ch++ { |
| 64 | if ch in important_escapable_chars { |
| 65 | continue |
| 66 | } |
| 67 | mut sc := Scanner{ |
| 68 | text: [u8(`"`), `t`, ch, `"`] |
| 69 | } |
| 70 | tok := sc.scan() |
| 71 | assert tok.kind == .error |
| 72 | assert tok.lit.bytestr() == 'character must be escaped with a unicode escape, replace with: \\u${ch:04x}' |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | fn test_str_invalid_unicode_escape() { |
| 77 | mut sc := Scanner{ |
| 78 | text: r'"\u010G"'.bytes() |
| 79 | } |
| 80 | tok := sc.scan() |
| 81 | assert tok.kind == .error |
| 82 | assert tok.lit.bytestr() == '`G` is not a hex digit' |
| 83 | } |
| 84 | |
| 85 | fn test_str_invalid_unicode_escape_len() { |
| 86 | mut sc := Scanner{ |
| 87 | text: r'"\u001"'.bytes() |
| 88 | } |
| 89 | tok := sc.scan() |
| 90 | assert tok.kind == .error |
| 91 | assert tok.lit.bytestr() == 'unicode escape must have 4 hex digits' |
| 92 | } |
| 93 | |
| 94 | fn test_str_invalid_uppercase_u() { |
| 95 | mut sc := Scanner{ |
| 96 | text: r'"\U0000"'.bytes() |
| 97 | } |
| 98 | tok := sc.scan() |
| 99 | assert tok.kind == .error |
| 100 | assert tok.lit.bytestr() == 'unicode endpoints must be in lowercase `u`' |
| 101 | } |
| 102 | |
| 103 | fn test_str_missing_closing_bracket() { |
| 104 | mut sc := Scanner{ |
| 105 | text: '"incomplete'.bytes() |
| 106 | } |
| 107 | tok := sc.scan() |
| 108 | assert tok.kind == .error |
| 109 | assert tok.lit.bytestr() == 'missing double quotes in string closing' |
| 110 | } |
| 111 | |
| 112 | fn test_int() { |
| 113 | mut sc := Scanner{ |
| 114 | text: '10'.bytes() |
| 115 | } |
| 116 | tok := sc.scan() |
| 117 | assert tok.kind == .int |
| 118 | assert tok.lit.len == 2 |
| 119 | assert tok.lit.bytestr() == '10' |
| 120 | } |
| 121 | |
| 122 | fn test_int_negative() { |
| 123 | mut sc := Scanner{ |
| 124 | text: '-10'.bytes() |
| 125 | } |
| 126 | tok := sc.scan() |
| 127 | assert tok.kind == .int |
| 128 | assert tok.lit.len == 3 |
| 129 | assert tok.lit.bytestr() == '-10' |
| 130 | } |
| 131 | |
| 132 | fn test_float() { |
| 133 | mut sc := Scanner{ |
| 134 | text: '123.400'.bytes() |
| 135 | } |
| 136 | tok := sc.scan() |
| 137 | assert tok.kind == .float |
| 138 | assert tok.lit.len == 7 |
| 139 | assert tok.lit.bytestr() == '123.400' |
| 140 | } |
| 141 | |
| 142 | fn test_float_negative() { |
| 143 | mut sc := Scanner{ |
| 144 | text: '-123.400'.bytes() |
| 145 | } |
| 146 | tok := sc.scan() |
| 147 | assert tok.kind == .float |
| 148 | assert tok.lit.len == 8 |
| 149 | assert tok.lit.bytestr() == '-123.400' |
| 150 | } |
| 151 | |
| 152 | fn test_int_exp() { |
| 153 | mut sc := Scanner{ |
| 154 | text: '1E22'.bytes() |
| 155 | } |
| 156 | tok := sc.scan() |
| 157 | assert tok.kind == .int |
| 158 | assert tok.lit.len == 4 |
| 159 | assert tok.lit.bytestr() == '1E22' |
| 160 | } |
| 161 | |
| 162 | fn test_int_exp_negative() { |
| 163 | mut sc := Scanner{ |
| 164 | text: '1E-2'.bytes() |
| 165 | } |
| 166 | tok := sc.scan() |
| 167 | assert tok.kind == .int |
| 168 | assert tok.lit.len == 4 |
| 169 | assert tok.lit.bytestr() == '1E-2' |
| 170 | } |
| 171 | |
| 172 | fn test_int_exp_positive() { |
| 173 | mut sc := Scanner{ |
| 174 | text: '1E+2'.bytes() |
| 175 | } |
| 176 | tok := sc.scan() |
| 177 | assert tok.kind == .int |
| 178 | assert tok.lit.len == 4 |
| 179 | assert tok.lit.bytestr() == '1E+2' |
| 180 | } |
| 181 | |
| 182 | fn test_float_exp() { |
| 183 | mut sc := Scanner{ |
| 184 | text: '123.456e78'.bytes() |
| 185 | } |
| 186 | tok := sc.scan() |
| 187 | assert tok.kind == .float |
| 188 | assert tok.lit.len == 10 |
| 189 | assert tok.lit.bytestr() == '123.456e78' |
| 190 | } |
| 191 | |
| 192 | fn test_float_exp_negative() { |
| 193 | mut sc := Scanner{ |
| 194 | text: '20.56e-5'.bytes() |
| 195 | } |
| 196 | tok := sc.scan() |
| 197 | assert tok.kind == .float |
| 198 | assert tok.lit.len == 8 |
| 199 | assert tok.lit.bytestr() == '20.56e-5' |
| 200 | } |
| 201 | |
| 202 | fn test_float_exp_positive() { |
| 203 | mut sc := Scanner{ |
| 204 | text: '20.56e+5'.bytes() |
| 205 | } |
| 206 | tok := sc.scan() |
| 207 | assert tok.kind == .float |
| 208 | assert tok.lit.len == 8 |
| 209 | assert tok.lit.bytestr() == '20.56e+5' |
| 210 | } |
| 211 | |
| 212 | fn test_number_with_space() { |
| 213 | mut sc := Scanner{ |
| 214 | text: ' 4'.bytes() |
| 215 | } |
| 216 | tok := sc.scan() |
| 217 | assert tok.kind == .int |
| 218 | assert tok.lit.len == 1 |
| 219 | assert tok.lit.bytestr() == '4' |
| 220 | } |
| 221 | |
| 222 | fn test_number_invalid_leading_zero() { |
| 223 | mut sc := Scanner{ |
| 224 | text: '0010'.bytes() |
| 225 | } |
| 226 | tok := sc.scan() |
| 227 | assert tok.kind == .error |
| 228 | assert tok.lit.bytestr() == 'leading zeroes in a number are not allowed' |
| 229 | } |
| 230 | |
| 231 | fn test_number_invalid_leading_zero_negative() { |
| 232 | mut sc := Scanner{ |
| 233 | text: '-0010'.bytes() |
| 234 | } |
| 235 | tok := sc.scan() |
| 236 | assert tok.kind == .error |
| 237 | assert tok.lit.bytestr() == 'leading zeroes in a number are not allowed' |
| 238 | } |
| 239 | |
| 240 | fn test_number_invalid_start_char() { |
| 241 | mut sc := Scanner{ |
| 242 | text: '+1'.bytes() |
| 243 | } |
| 244 | tok := sc.scan() |
| 245 | assert tok.kind == .error |
| 246 | assert tok.lit.bytestr() == 'invalid token `+`' |
| 247 | } |
| 248 | |
| 249 | fn test_number_invalid_char() { |
| 250 | mut sc := Scanner{ |
| 251 | text: '122x'.bytes() |
| 252 | } |
| 253 | sc.scan() |
| 254 | tok := sc.scan() |
| 255 | assert tok.kind == .error |
| 256 | assert tok.lit.bytestr() == 'invalid token `x`' |
| 257 | } |
| 258 | |
| 259 | fn test_number_invalid_char_float() { |
| 260 | mut sc := Scanner{ |
| 261 | text: '122x.1'.bytes() |
| 262 | } |
| 263 | sc.scan() |
| 264 | tok := sc.scan() |
| 265 | assert tok.kind == .error |
| 266 | assert tok.lit.bytestr() == 'invalid token `x`' |
| 267 | } |
| 268 | |
| 269 | fn test_number_invalid_multiple_dot() { |
| 270 | mut sc := Scanner{ |
| 271 | text: '122.108.10'.bytes() |
| 272 | } |
| 273 | sc.scan() |
| 274 | tok := sc.scan() |
| 275 | assert tok.kind == .error |
| 276 | assert tok.lit.bytestr() == 'invalid token `.`' |
| 277 | } |
| 278 | |
| 279 | fn test_number_invalid_exp() { |
| 280 | mut sc := Scanner{ |
| 281 | text: '0.3e'.bytes() |
| 282 | } |
| 283 | tok := sc.scan() |
| 284 | assert tok.kind == .error |
| 285 | assert tok.lit.bytestr() == 'invalid exponent' |
| 286 | } |
| 287 | |
| 288 | fn test_number_invalid_exp_with_sign() { |
| 289 | mut sc := Scanner{ |
| 290 | text: '0.3e+'.bytes() |
| 291 | } |
| 292 | tok := sc.scan() |
| 293 | assert tok.kind == .error |
| 294 | assert tok.lit.bytestr() == 'invalid exponent' |
| 295 | } |
| 296 | |
| 297 | fn test_number_invalid_zero_exp() { |
| 298 | mut sc := Scanner{ |
| 299 | text: '0e'.bytes() |
| 300 | } |
| 301 | tok := sc.scan() |
| 302 | assert tok.kind == .error |
| 303 | assert tok.lit.bytestr() == 'invalid exponent' |
| 304 | } |
| 305 | |
| 306 | fn test_number_invalid_dot_exp() { |
| 307 | mut sc := Scanner{ |
| 308 | text: '0.e'.bytes() |
| 309 | } |
| 310 | tok := sc.scan() |
| 311 | assert tok.kind == .error |
| 312 | assert tok.lit.bytestr() == 'invalid float' |
| 313 | } |
| 314 | |
| 315 | fn test_number_invalid_double_exp() { |
| 316 | mut sc := Scanner{ |
| 317 | text: '2eE'.bytes() |
| 318 | } |
| 319 | sc.scan() |
| 320 | tok := sc.scan() |
| 321 | assert tok.kind == .error |
| 322 | assert tok.lit.bytestr() == 'invalid token `E`' |
| 323 | } |
| 324 | |
| 325 | fn test_null() { |
| 326 | mut sc := Scanner{ |
| 327 | text: 'null'.bytes() |
| 328 | } |
| 329 | tok := sc.scan() |
| 330 | assert tok.kind == .null |
| 331 | assert tok.lit.len == 4 |
| 332 | assert tok.lit.bytestr() == 'null' |
| 333 | } |
| 334 | |
| 335 | fn test_bool_true() { |
| 336 | mut sc := Scanner{ |
| 337 | text: 'true'.bytes() |
| 338 | } |
| 339 | tok := sc.scan() |
| 340 | assert tok.kind == .bool |
| 341 | assert tok.lit.len == 4 |
| 342 | assert tok.lit.bytestr() == 'true' |
| 343 | } |
| 344 | |
| 345 | fn test_bool_false() { |
| 346 | mut sc := Scanner{ |
| 347 | text: 'false'.bytes() |
| 348 | } |
| 349 | tok := sc.scan() |
| 350 | assert tok.kind == .bool |
| 351 | assert tok.lit.len == 5 |
| 352 | assert tok.lit.bytestr() == 'false' |
| 353 | } |
| 354 | |
| 355 | fn test_json_with_whitespace_start() { |
| 356 | mut sc := Scanner{ |
| 357 | text: ' \n \n\t {'.bytes() |
| 358 | } |
| 359 | tok := sc.scan() |
| 360 | eprintln(tok) |
| 361 | assert tok.kind == .lcbr |
| 362 | assert tok.lit.len == 0 |
| 363 | } |
| 364 | |
| 365 | fn test_json_with_whitespace_end() { |
| 366 | mut sc := Scanner{ |
| 367 | text: '} \n\t'.bytes() |
| 368 | } |
| 369 | tok := sc.scan() |
| 370 | assert tok.kind == .rcbr |
| 371 | tok2 := sc.scan() |
| 372 | eprintln(tok2) |
| 373 | assert tok2.kind == .eof |
| 374 | } |
| 375 | |