// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module scanner

import toml.input
import toml.token
import toml.util

pub const digit_extras = [`_`, `.`, `x`, `o`, `b`, `e`, `E`]
pub const end_of_text = u32(~0)

// Scanner contains the necessary fields for the state of the scan process.
// the task the scanner does is also referred to as "lexing" or "tokenizing".
// The Scanner methods are based on much of the work in `vlib/strings/textscanner`.
pub struct Scanner {
pub:
    config Config
    text   string // the input TOML text
mut:
    col        int // current column number (x coordinate)
    line_nr    int = 1 // current line number (y coordinate)
    pos        int // current flat/index position in the `text` field
    header_len int // Length, how many bytes of header was found
    // Quirks
    is_left_of_assign bool = true // indicates if the scanner is on the *left* side of an assignment
}

// State is a read-only copy of the scanner's internal state.
// See also `Scanner.state()`.
pub struct State {
pub:
    col     int // current column number (x coordinate)
    line_nr int = 1 // current line number (y coordinate)
    pos     int // current flat/index position in the `text` field
}

// Config is used to configure a Scanner instance.
// Only one of the fields `text` and `file_path` is allowed to be set at time of configuration.
pub struct Config {
pub:
    input               input.Config
    tokenize_formatting bool = true // if true, generate tokens for `\n`, ` `, `\t`, `\r` etc.
}

// new_scanner returns a new *heap* allocated `Scanner` instance, based on the file in config.input.file_path,
// or based on the text in config.input.text .
pub fn new_scanner(config Config) !&Scanner {
    mut s := &Scanner{
        config: config
        text:   config.input.read_input()!
    }
    return s
}

// new_simple returns a new *stack* allocated `Scanner` instance.
pub fn new_simple(config Config) !Scanner {
    return Scanner{
        config: config
        text:   config.input.read_input()!
    }
}

// new_simple_text returns a new *stack* allocated `Scanner` instance
// ready for parsing TOML in `text`.
pub fn new_simple_text(text string) !Scanner {
    in_config := input.Config{
        text: text
    }
    config := Config{
        input: in_config
    }
    return Scanner{
        config: config
        text:   config.input.read_input()!
    }
}

// new_simple_file returns a new *stack* allocated `Scanner` instance
// ready for parsing TOML in file read from `path`.
pub fn new_simple_file(path string) !Scanner {
    in_config := input.Config{
        file_path: path
    }
    config := Config{
        input: in_config
    }
    return Scanner{
        config: config
        text:   config.input.read_input()!
    }
}

// scan returns the next token from the input.
@[direct_array_access]
pub fn (mut s Scanner) scan() !token.Token {
    s.validate_and_skip_headers()!

    for {
        c := s.next()
        byte_c := u8(c)
        if c == end_of_text {
            s.inc_line_number()
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'reached EOF')
            return s.new_token(.eof, '', 1)
        }

        ascii := byte_c.ascii_str()
        util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'current char "${ascii}"')

        if byte_c == u8(0x0) {
            s.reset()
            return error(@MOD + '.' + @STRUCT + '.' + @FN +
                ' NULL control character `${c.hex()}` is not allowed at (${s.line_nr},${s.col}) "${ascii}" near ...${s.excerpt(s.pos, 5)}...')
        }

        is_sign := c == `+` || c == `-`

        // (+/-)nan & (+/-)inf
        peek_1 := s.peek(1)
        peek_2 := s.peek(2)
        is_nan := c == `n` && s.at() == `a` && peek_1 == `n`
        is_inf := !is_nan && c == `i` && s.at() == `n` && peek_1 == `f`
        is_signed_nan := is_sign && s.at() == `n` && peek_1 == `a` && peek_2 == `n`
        is_signed_inf := !is_signed_nan && is_sign && s.at() == `i` && peek_1 == `n`
            && peek_2 == `f`
        if !s.is_left_of_assign && (is_nan || is_inf || is_signed_nan || is_signed_inf) {
            num := s.extract_nan_or_inf_number()!
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                'identified a special number "${num}" (${num.len})')
            return s.new_token(.number, num, num.len)
        }

        is_signed_number := is_sign && u8(s.at()).is_digit() && !u8(s.peek(-1)).is_digit()
        is_digit := byte_c.is_digit()
        if is_digit || is_signed_number {
            num := s.extract_number()!
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                'identified a number "${num}" (${num.len})')
            return s.new_token(.number, num, num.len)
        }

        if util.is_key_char(byte_c) {
            key := s.extract_key()
            if u8(s.peek(1)) != `=` && (key == 'true' || key == 'false') {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified a boolean "${key}" (${key.len})')
                return s.new_token(.boolean, key, key.len)
            }
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                'identified a bare key "${key}" (${key.len})')
            return s.new_token(.bare, key, key.len)
        }

        match rune(c) {
            ` `, `\t`, `\n`, `\r` {
                if c == `\n` {
                    s.inc_line_number()
                    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                        'incremented line nr to ${s.line_nr}')
                } else if c == `\r` {
                    // CR should always be followed by a `\n`
                    if s.at() != `\n` {
                        return error(@MOD + '.' + @STRUCT + '.' + @FN +
                            ' missing newline/linefeed character after "\\c" carriage return at (${s.line_nr},${s.col}) "${ascii}" near ...${s.excerpt(s.pos, 5)}...')
                    }
                }
                // Date-Time in RFC 3339 is allowed to have a space between the date and time in supplement to the 'T'
                // so we allow space characters to slip through to the parser if the space is between two digits...
                // util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, '"'+u8(s.peek(-1)).ascii_str()+'" < "${ascii}" > "'+u8(s.at()).ascii_str()+'"')
                if c == ` ` && u8(s.peek(-1)).is_digit() && u8(s.at()).is_digit() {
                    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                        'identified, what could be, a space between a RFC 3339 date and time ("${ascii}") (${ascii.len})')
                    return s.new_token(token.Kind.whitespace, ascii, ascii.len)
                }
                if s.config.tokenize_formatting {
                    mut kind := token.Kind.whitespace
                    if c == `\t` {
                        kind = token.Kind.tab
                    } else if c == `\r` {
                        kind = token.Kind.cr
                    } else if c == `\n` {
                        kind = token.Kind.nl
                    }
                    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                        'identified formatting character ("${ascii}") (${ascii.len})')
                    return s.new_token(kind, ascii, ascii.len)
                } else {
                    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                        'skipping " ", "\\t" or "\\n" ("${ascii}") (${ascii.len})')
                }
                continue
            }
            `-` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified minus "${ascii}" (${ascii.len})')
                return s.new_token(.minus, ascii, ascii.len)
            }
            `_` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified underscore "${ascii}" (${ascii.len})')
                return s.new_token(.underscore, ascii, ascii.len)
            }
            `+` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified plus "${ascii}" (${ascii.len})')
                return s.new_token(.plus, ascii, ascii.len)
            }
            `=` {
                s.is_left_of_assign = false
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified assignment "${ascii}" (${ascii.len})')
                return s.new_token(.assign, ascii, ascii.len)
            }
            `"`, `'` { // ... some string "/'
                ident_string := s.extract_string()!
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified quoted string `${ident_string}`')
                return s.new_token(.quoted, ident_string, ident_string.len)
            }
            `#` {
                hash := s.ignore_line()!
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified comment hash "${hash}" (${hash.len})')
                return s.new_token(.hash, hash, hash.len + 1)
            }
            `{` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified left curly bracket "${ascii}" (${ascii.len})')
                return s.new_token(.lcbr, ascii, ascii.len)
            }
            `}` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified right curly bracket "${ascii}" (${ascii.len})')
                return s.new_token(.rcbr, ascii, ascii.len)
            }
            `[` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified left square bracket "${ascii}" (${ascii.len})')
                return s.new_token(.lsbr, ascii, ascii.len)
            }
            `]` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified right square bracket "${ascii}" (${ascii.len})')
                return s.new_token(.rsbr, ascii, ascii.len)
            }
            `:` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified colon "${ascii}" (${ascii.len})')
                return s.new_token(.colon, ascii, ascii.len)
            }
            `,` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified comma "${ascii}" (${ascii.len})')
                return s.new_token(.comma, ascii, ascii.len)
            }
            `.` {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'identified period "${ascii}" (${ascii.len})')
                return s.new_token(.period, ascii, ascii.len)
            }
            else {
                return error(@MOD + '.' + @STRUCT + '.' + @FN +
                    ' could not scan character `${ascii}` / ${c} at ${s.pos} (${s.line_nr},${s.col}) near ...${s.excerpt(s.pos, 5)}...')
            }
        }
    }
    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'unknown character code at ${s.pos} (${s.line_nr},${s.col}) near ...${s.excerpt(s.pos,
        5)}...')
    return s.new_token(.unknown, '', 0)
}

// free frees all allocated resources.
@[unsafe]
pub fn (mut s Scanner) free() {
    unsafe {
        s.text.free()
    }
}

// remaining returns how many characters remain in the text input.
@[inline]
pub fn (s &Scanner) remaining() int {
    return s.text.len - s.pos
}

// next returns the next character code from the input text.
// next returns `end_of_text` if it can't reach the next character.
@[direct_array_access; inline]
pub fn (mut s Scanner) next() u32 {
    if s.pos < s.text.len {
        opos := s.pos
        s.pos++
        s.col++
        c := s.text[opos]
        return c
    }
    return end_of_text
}

// skip skips one character ahead.
@[inline]
pub fn (mut s Scanner) skip() {
    if s.pos + 1 < s.text.len {
        s.pos++
        s.col++
    }
}

// skip_n skips ahead `n` characters.
// If the skip goes out of bounds from the length of `Scanner.text`,
// the scanner position will be sat to the last character possible.
@[inline]
pub fn (mut s Scanner) skip_n(n int) {
    s.pos += n
    if s.pos > s.text.len {
        s.pos = s.text.len
    }
    s.col = s.pos
}

// at returns the *current* character code from the input text.
// at returns `end_of_text` if it can't get the current character.
// unlike `next()`, `at()` does not change the state of the scanner.
@[direct_array_access; inline]
pub fn (s &Scanner) at() u32 {
    if s.pos < s.text.len {
        return s.text[s.pos]
    }
    return end_of_text
}

// at_crlf returns `true` if the scanner is at a `\r` character
// and the next character is a `\n`.
fn (s &Scanner) at_crlf() bool {
    return s.at() == `\r` && s.peek(1) == `\n`
}

// peek returns the character code from the input text at position + `n`.
// peek returns `end_of_text` if it can't peek `n` characters ahead.
@[direct_array_access; inline]
pub fn (s &Scanner) peek(n int) u32 {
    if s.pos + n < s.text.len {
        // Allow peeking back - needed for spaces between date and time in RFC 3339 format :/
        if n - 1 < 0 && s.pos + n - 1 >= 0 {
            // util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'LOOKING BAAAA-AACK - OOOVER MY SHOOOOULDEEEER "${s.text[s.pos + n-1]}"')
            return s.text[s.pos + n - 1]
        }
        return s.text[s.pos + n]
    }
    return end_of_text
}

// reset resets the internal state of the scanner.
pub fn (mut s Scanner) reset() {
    s.pos = 0
    s.col = 0
    s.line_nr = 1
    s.header_len = 0
}

// new_token returns a new `token.Token`.
@[inline]
fn (mut s Scanner) new_token(kind token.Kind, lit string, len int) token.Token {
    // println('new_token(${lit})')
    mut col := s.col - len + 1
    if s.line_nr == 1 {
        col -= s.header_len
    }
    return token.Token{
        kind:    kind
        lit:     lit
        col:     if col < 1 { 1 } else { col }
        line_nr: s.line_nr + 1
        pos:     s.pos - s.header_len - len + 1
        len:     len
    }
}

// ignore_line forwards the scanner to the end of the current line.
@[direct_array_access; inline]
fn (mut s Scanner) ignore_line() !string {
    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, ' ignoring until EOL...')
    start := s.pos
    for c := s.at(); c != end_of_text && c != `\n`; c = s.at() {
        util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'skipping "${u8(c).ascii_str()} / ${c}"')
        if s.at_crlf() {
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'letting `\\r\\n` slip through')
            break
        }
        s.next()
    }
    return s.text[start..s.pos]
}

// inc_line_number increases the internal line number.
@[inline]
fn (mut s Scanner) inc_line_number() {
    s.col = 0
    s.line_nr++
    s.is_left_of_assign = true
}

// extract_key parses and returns a TOML key as a string.
@[direct_array_access; inline]
fn (mut s Scanner) extract_key() string {
    s.pos--
    s.col--
    start := s.pos
    for s.pos < s.text.len {
        c := u8(s.at())
        if !(util.is_key_char(c) || c.is_digit() || c in [`_`, `-`]) {
            break
        }
        s.pos++
        s.col++
    }
    key := s.text[start..s.pos]
    return key
}

// extract_string collects and returns a string containing
// any bytes recognized as a TOML string.
// TOML strings are everything found between two double or single quotation marks (`"`/`'`).
@[direct_array_access; inline]
fn (mut s Scanner) extract_string() !string {
    // extract_string is called when the scanner has already reached
    // a byte that is the start of a string so we rewind it to start at the correct
    s.pos--
    s.col--
    quote := u8(s.at())
    start := s.pos
    mut lit := quote.ascii_str()

    is_multiline := s.text[s.pos + 1] == quote && s.text[s.pos + 2] == quote
    // Check for escaped multiline quote
    if is_multiline {
        mls := s.extract_multiline_string()!
        return mls
    }

    for {
        s.pos++
        s.col++

        if s.pos >= s.text.len {
            return error(@MOD + '.' + @STRUCT + '.' + @FN +
                ' unfinished single-line string literal `${quote.ascii_str()}` started at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
        }

        c := u8(s.at())
        util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
            'c: `${c.ascii_str()}` / ${c} (quote type: ${quote}/${quote.ascii_str()})')

        // Check for escaped chars
        if c == u8(92) {
            esc, skip := s.handle_escapes(quote, is_multiline)
            lit += esc
            if skip > 0 {
                s.pos += skip
                s.col += skip
                continue
            }
        }
        // Check for control characters (allow TAB)
        if util.is_illegal_ascii_control_character(c) {
            return error(@MOD + '.' + @STRUCT + '.' + @FN +
                ' control character `${c.hex()}` is not allowed at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
        }

        if c == quote {
            s.pos++
            s.col++
            return lit + quote.ascii_str()
        }

        lit += c.ascii_str()

        // Don't eat multiple lines in single-line mode
        if lit.contains('\n') {
            return error(@MOD + '.' + @STRUCT + '.' + @FN +
                ' unfinished single-line string literal `${quote.ascii_str()}` started at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
        }
    }
    return lit
}

// extract_multiline_string collects and returns a string containing
// any bytes recognized as a TOML string.
// TOML strings are everything found between two double or single quotation marks (`"`/`'`).
@[direct_array_access; inline]
fn (mut s Scanner) extract_multiline_string() !string {
    // extract_multiline_string is called from extract_string so we know the 3 first
    // characters is the quotes
    quote := u8(s.at())
    start := s.pos
    mut lit := quote.ascii_str() + quote.ascii_str() + quote.ascii_str()

    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'multi-line `${quote.ascii_str()}${s.text[
        s.pos + 1].ascii_str()}${s.text[s.pos + 2].ascii_str()}` string started at pos ${start} (${s.line_nr},${s.col}) (quote type: ${quote.ascii_str()} / ${quote})')

    s.pos += 2
    s.col += 2

    for {
        s.pos++
        s.col++

        if s.pos >= s.text.len {
            return error(@MOD + '.' + @STRUCT + '.' + @FN +
                ' unfinished multi-line string literal (${quote.ascii_str()}${quote.ascii_str()}${quote.ascii_str()}) started at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
        }

        c := u8(s.at())
        util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
            'c: `${c.ascii_str()}` / ${c} (quote type: ${quote}/${quote.ascii_str()})')

        if c == `\r` && s.peek(1) == `\n` {
            continue
        }
        if c == `\n` {
            s.inc_line_number()
            lit += c.ascii_str()
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'c: `\\n` / ${c}')
            continue
        }
        // Check for escaped chars
        if c == u8(92) {
            esc, skip := s.handle_escapes(quote, true)
            lit += esc
            if skip > 0 {
                s.pos += skip
                s.col += skip
                continue
            }
        }
        // Check for control characters (allow TAB)
        if util.is_illegal_ascii_control_character(c) {
            return error(@MOD + '.' + @STRUCT + '.' + @FN +
                ' control character `${c.hex()}` is not allowed at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
        }

        if c == quote {
            if s.peek(1) == quote && s.peek(2) == quote {
                if s.peek(3) == end_of_text {
                    s.pos += 3
                    s.col += 3
                    lit += quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
                    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                        'returning at ${c.ascii_str()} `${lit}`')
                    return lit
                } else if s.peek(3) != quote {
                    // lit += c.ascii_str()
                    // lit += quote.ascii_str()
                    s.pos += 3
                    s.col += 3
                    lit += quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
                    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                        'returning at ${c.ascii_str()} `${lit}`')
                    return lit
                }
            }
        }
        lit += c.ascii_str()
    }
    return lit
}

// handle_escapes returns any escape character sequence.
// For escape sequence validation see `Checker.check_quoted_escapes`.
fn (mut s Scanner) handle_escapes(quote u8, is_multiline bool) (string, int) {
    c := u8(s.at())
    mut lit := c.ascii_str()
    is_literal_string := quote == `'`
    if !is_literal_string {
        if s.peek(1) == `u` && u8(s.peek(2)).is_hex_digit() && u8(s.peek(3)).is_hex_digit()
            && u8(s.peek(4)).is_hex_digit() && u8(s.peek(5)).is_hex_digit() {
            lit += s.text[s.pos + 1..s.pos + 6] //.ascii_str()
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'gulp escaped unicode `${lit}`')
            return lit, 5
        } else if s.peek(1) == quote {
            if (!is_multiline && s.peek(2) == `\n`)
                || (is_multiline && s.peek(2) == quote && s.peek(3) == quote && s.peek(4) == `\n`) {
                util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                    'ignore special case escaped `${lit}` at end of string')
                return '', 0
            }
            lit += quote.ascii_str()
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'gulp escaped `${lit}`')
            return lit, 1
        }
    }
    if is_literal_string {
        if s.peek(1) == quote {
            util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
                'ignore escape `${lit}${u8(s.peek(1)).ascii_str()}` in literal string')
            return '', 0
        }
    }

    lit += u8(s.peek(1)).ascii_str()
    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'gulp escaped `${lit}`')
    return lit, 1
}

// extract_number collects and returns a string containing
// any bytes recognized as a TOML number except for "(+/-)nan" and "(+/-)inf".
// TOML numbers can include digits 0-9 and `_`.
@[direct_array_access; inline]
fn (mut s Scanner) extract_number() !string {
    // extract_number is called when the scanner has already reached
    // a byte that is a number or +/- - so we rewind it to start at the correct
    // position to get the complete number. Even if it's only one digit
    s.pos--
    s.col--
    start := s.pos

    mut c := s.at()
    is_digit := u8(c).is_digit()
    if !(is_digit || c in [`+`, `-`]) {
        return error(@MOD + '.' + @STRUCT + '.' + @FN +
            ' ${u8(c).ascii_str()} is not a number at ${s.excerpt(s.pos, 10)}')
    }
    s.pos++
    s.col++
    for s.pos < s.text.len {
        c = s.at()
        // Adjust scanner position to floating point numbers
        mut float_precision := 0
        if c == `.` {
            mut i := 1
            for c_ := u8(s.peek(i)); c_ != end_of_text && c_ != `\n`; c_ = u8(s.peek(i)) {
                if !c_.is_digit() && c_ != `,` {
                    float_precision = 0
                    break
                }
                float_precision++
                i++
            }
        }
        s.pos += float_precision
        s.col += float_precision
        // Handle signed exponent notation. I.e.: 3e2, 3E2, 3e-2, 3E+2, 3e0, 3.1e2, 3.1E2, -1E-1
        if c in [`e`, `E`] && s.peek(1) in [`+`, `-`] && u8(s.peek(2)).is_digit() {
            s.pos += 2
            s.col += 2
        }
        c = s.at()
        if !(u8(c).is_hex_digit() || c in digit_extras) || (c == `.` && s.is_left_of_assign) {
            break
        }
        s.pos++
        s.col++
    }
    key := s.text[start..s.pos]
    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
        'identified number "${key}" in range [${start} .. ${s.pos}]')
    return key
}

// extract_nan_or_inf_number collects and returns a string containing
// any bytes recognized as infinity or not-a-number TOML numbers.
@[direct_array_access; inline]
fn (mut s Scanner) extract_nan_or_inf_number() !string {
    // extract_nan_or_inf_number is called when the scanner has already identified that
    // +/- or 'nan'/'inf' bytes is up but we rewind it to start at the correct position
    s.pos--
    s.col--
    start := s.pos

    mut c := s.at()
    if c !in [`+`, `-`, `n`, `i`] {
        return error(@MOD + '.' + @STRUCT + '.' + @FN +
            ' ${u8(c).ascii_str()} is not a number at ${s.excerpt(s.pos, 10)}')
    }
    s.pos++
    s.col++
    for s.pos < s.text.len {
        c = s.at()
        if c !in [`n`, `a`, `i`, `f`] {
            break
        }
        s.pos++
        s.col++
    }
    key := s.text[start..s.pos]
    util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
        'identified special number "${key}" in range [${start} .. ${s.pos}]')
    return key
}

// excerpt returns a string excerpt of the input text centered
// at `pos`. The `margin` argument defines how many chacters
// on each side of `pos` is returned
pub fn (s &Scanner) excerpt(pos int, margin int) string {
    start := if pos > 0 && pos >= margin { pos - margin } else { 0 }
    end := if pos + margin < s.text.len { pos + margin } else { s.text.len }
    return s.text[start..end].replace('\n', r'\n')
}

// state returns a read-only view of the scanner's internal state.
pub fn (s &Scanner) state() State {
    return State{
        col:     s.col
        line_nr: s.line_nr
        pos:     s.pos
    }
}

fn (mut s Scanner) validate_and_skip_headers() ! {
    // UTF-16 / UTF-32 headers (BE/LE)
    s.check_utf16_or_32_bom()!

    // NICE-TO-HAVE-TODO Check other types of (UTF-?) headers and yield an error. TOML is UTF-8 only.

    // Skip optional UTF-8 header, if any.
    if s.at() == 0xEF && s.peek(1) == 0xBB && s.peek(2) == 0xBF {
        util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'skipping UTF-8 byte order mark (BOM)')
        s.header_len = 3
        s.skip_n(s.header_len)
    }

    // Check after we've skipped UTF-8 BOM
    s.check_utf16_or_32_bom()!
}

fn (mut s Scanner) check_utf16_or_32_bom() ! {
    if (s.at() == 0xFF && s.peek(1) == 0xFE && s.peek(2) == 0x00 && s.peek(3) == 0x00)
        || (s.at() == 0x00 && s.peek(1) == 0x00 && s.peek(2) == 0xFE && s.peek(3) == 0xFF) {
        s.header_len = 4
        s.skip_n(s.header_len)
        return error(@MOD + '.' + @STRUCT + '.' + @FN +
            ' UTF-32 is not a valid TOML encoding at ${s.pos} (${s.line_nr},${s.col}) near ...${s.excerpt(s.pos, 5)}...')
    }
    if (s.at() == 0xFE && s.peek(1) == 0xFF) || (s.at() == 0xFF && s.peek(1) == 0xFE) {
        s.header_len = 2
        s.skip_n(s.header_len)
        return error(@MOD + '.' + @STRUCT + '.' + @FN +
            ' UTF-16 is not a valid TOML encoding at ${s.pos} (${s.line_nr},${s.col}) near ...${s.excerpt(s.pos, 5)}...')
    }
}