// zstd(https://github.com/facebook/zstd) is a fast real-time compression algorithm developed by Facebook.
// zstd compression/decompression
module zstd

import os
import encoding.binary

#flag -I @VMODROOT/thirdparty/zstd
#include "zstd.c"    // msvc can't compile multiple source files, so included

const frame_header_size_max = 18
const content_size_unknown = u64(-1)
const content_size_error = u64(-2)

const buf_in_size = 1024 * 1024
const buf_out_size = 1024 * 1024

fn C.ZSTD_versionNumber() u32
fn C.ZSTD_versionString() charptr

fn C.ZSTD_compress(voidptr, usize, voidptr, usize, i32) usize
fn C.ZSTD_decompress(voidptr, usize, voidptr, usize) usize
fn C.ZSTD_getFrameContentSize(voidptr, usize) u64
fn C.ZSTD_findFrameCompressedSize(voidptr, usize) usize
fn C.ZSTD_compressBound(usize) usize
fn C.ZSTD_isError(usize) u32
fn C.ZSTD_getErrorName(usize) charptr
fn C.ZSTD_minCLevel() i32
fn C.ZSTD_maxCLevel() i32
fn C.ZSTD_defaultCLevel() i32
fn C.ZSTD_createCCtx() &C.ZSTD_CCtx_s
fn C.ZSTD_freeCCtx(voidptr) usize
fn C.ZSTD_compressCCtx(voidptr, voidptr, usize, voidptr, usize, i32) usize
fn C.ZSTD_createDCtx() &C.ZSTD_DCtx_s
fn C.ZSTD_freeDCtx(voidptr) usize
fn C.ZSTD_decompressDCtx(voidptr, voidptr, usize, voidptr, usize) usize

// note : new strategies _might_ be added in the future. Only the order (from fast to strong) is guaranteed
pub enum Strategy {
    default  = 0
    fast     = 1
    dfast    = 2
    greedy   = 3
    lazy     = 4
    lazy2    = 5
    btlazy2  = 6
    btopt    = 7
    btultra  = 8
    btultra2 = 9
}

pub enum CParameter {
    // compression parameters
    // Note: When compressing with a ZSTD_CDict these parameters are superseded
    // by the parameters used to construct the ZSTD_CDict.
    // See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict).
    //
    // Set compression parameters according to pre-defined cLevel table.
    // Note that exact compression parameters are dynamically determined,
    // depending on both compression level and srcSize (when known).
    // Default level is ZSTD_CLEVEL_DEFAULT==3.
    // Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
    // Note 1 : it's possible to pass a negative compression level.
    // Note 2 : setting a level does not automatically set all other compression parameters
    // to default. Setting this will however eventually dynamically impact the compression
    // parameters which have not been manually set. The manually set
    // ones will 'stick'.
    compression_level = 100
    // Advanced compression parameters :
    // It's possible to pin down compression parameters to some specific values.
    // In which case, these values are no longer dynamically selected by the compressor
    //
    // Maximum allowed back-reference distance, expressed as power of 2.
    // This will set a memory budget for streaming decompression,
    // with larger values requiring more memory
    // and typically compressing more.
    // Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
    // Special: value 0 means "use default windowLog".
    // Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
    // requires explicitly allowing such size at streaming decompression stage.
    window_log = 101
    // Size of the initial probe table, as a power of 2.
    // Resulting memory usage is (1 << (hashLog+2)).
    // Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
    // Larger tables improve compression ratio of strategies <= dFast,
    // and improve speed of strategies > dFast.
    // Special: value 0 means "use default hashLog".
    hash_log = 102
    // Size of the multi-probe search table, as a power of 2.
    // Resulting memory usage is (1 << (chainLog+2)).
    // Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
    // Larger tables result in better and slower compression.
    // This parameter is useless for "fast" strategy.
    // It's still useful when using "dfast" strategy,
    // in which case it defines a secondary probe table.
    // Special: value 0 means "use default chainLog".
    chain_log = 103
    // Number of search attempts, as a power of 2.
    // More attempts result in better and slower compression.
    // This parameter is useless for "fast" and "dFast" strategies.
    // Special: value 0 means "use default searchLog".
    search_log = 104
    // Minimum size of searched matches.
    // Note that Zstandard can still find matches of smaller size,
    // it just tweaks its search algorithm to look for this size and larger.
    // Larger values increase compression and decompression speed, but decrease ratio.
    // Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
    // Note that currently, for all strategies < btopt, effective minimum is 4.
    // , for all strategies > fast, effective maximum is 6.
    // Special: value 0 means "use default minMatchLength".
    min_match = 105
    // Impact of this field depends on strategy.
    // For strategies btopt, btultra & btultra2:
    // Length of Match considered "good enough" to stop search.
    // Larger values make compression stronger, and slower.
    // For strategy fast:
    // Distance between match sampling.
    // Larger values make compression faster, and weaker.
    // Special: value 0 means "use default targetLength".
    target_length = 106
    // See ZSTD_strategy enum definition.
    // The higher the value of selected strategy, the more complex it is,
    // resulting in stronger and slower compression.
    // Special: value 0 means "use default strategy".
    strategy = 107
    // v1.5.6+
    // Attempts to fit compressed block size into approximately targetCBlockSize.
    // Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
    // Note that it's not a guarantee, just a convergence target (default:0).
    // No target when targetCBlockSize == 0.
    // This is helpful in low bandwidth streaming environments to improve end-to-end latency,
    // when a client can make use of partial documents (a prominent example being Chrome).
    // Note: this parameter is stable since v1.5.6.
    // It was present as an experimental parameter in earlier versions,
    // but it's not recommended using it with earlier library versions
    // due to massive performance regressions.
    target_c_block_size = 130
    // LDM mode parameters
    // Enable long distance matching.
    // This parameter is designed to improve compression ratio
    // for large inputs, by finding large matches at long distance.
    // It increases memory usage and window size.
    // Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
    // except when expressly set to a different value.
    // Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
    // compression strategy >= ZSTD_btopt (== compression level 16+)
    enable_long_distance_matching = 160
    // Size of the table for long distance matching, as a power of 2.
    // Larger values increase memory usage and compression ratio,
    // but decrease compression speed.
    // Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
    // default: windowlog - 7.
    // Special: value 0 means "automatically determine hashlog".
    ldm_hash_log = 161
    // Minimum match size for long distance matcher.
    // Larger/too small values usually decrease compression ratio.
    // Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
    // Special: value 0 means "use default value" (default: 64).
    ldm_min_match = 162
    // log size of each bucket in the ldm hash table for collision resolution.
    // Larger values improve collision resolution but decrease compression speed.
    // The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
    // Special: value 0 means "use default value" (default: 3).
    ldm_bucket_size_log = 163
    // Frequency of inserting/looking up entries into the LDM hash table.
    // Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
    // Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
    // Larger values improve compression speed.
    // Deviating far from default value will likely result in a compression ratio decrease.
    // Special: value 0 means "automatically determine hashRateLog".
    ldm_hash_rate_log = 164
    // frame parameters
    // Content size will be written into frame header _whenever known_ (default:1)
    // Content size must be known at the beginning of compression.
    // This is automatically the case when using ZSTD_compress2(),
    // For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize()
    content_size_flag = 200
    // A 32-bits checksum of content is written at end of frame (default:0)
    checksum_flag = 201
    // When applicable, dictionary's ID is written into frame header (default:1)
    dict_id_flag = 202
    // multi-threading parameters
    // These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
    // Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
    // In a situation where it's unknown if the linked library supports multi-threading or not,
    // setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
    //
    // Select how many threads will be spawned to compress in parallel.
    // When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
    // ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
    // while compression is performed in parallel, within worker thread(s).
    // (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
    // in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
    // More workers improve speed, but also increase memory usage.
    // Default value is `0`, aka "single-threaded mode" : no worker is spawned,
    // compression is performed inside Caller's thread, and all invocations are blocking
    nb_workers = 400
    // Size of a compression job. This value is enforced only when nbWorkers >= 1.
    // Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
    // 0 means default, which is dynamically determined based on compression parameters.
    // Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
    // The minimum size is automatically and transparently enforced.
    job_size = 401
    // Control the overlap size, as a fraction of window size.
    // The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
    // It helps preserve compression ratio, while each job is compressed in parallel.
    // This value is enforced only when nbWorkers >= 1.
    // Larger values increase compression ratio, but decrease speed.
    // Possible values range from 0 to 9 :
    // - 0 means "default" : value will be determined by the library, depending on strategy
    // - 1 means "no overlap"
    // - 9 means "full overlap", using a full window size.
    // Each intermediate rank increases/decreases load size by a factor 2 :
    // 9: full window;  8: w/2;  7: w/4;  6: w/8;  5:w/16;  4: w/32;  3:w/64;  2:w/128;  1:no overlap;  0:default
    // default value varies between 6 and 9, depending on strategy
    overlap_log = 402
    // note : additional experimental parameters are also available
    // within the experimental section of the API.
    // At the time of this writing, they include :
    // zstd_c_rsyncable
    // zstd_c_format
    // zstd_c_force_max_window
    // zstd_c_force_attach_dict
    // zstd_c_literal_compression_mode
    // zstd_c_target_c_block_size
    // zstd_c_src_size_hint
    // zstd_c_enable_dedicated_dict_search
    // zstd_c_stable_in_buffer
    // zstd_c_stable_out_buffer
    // zstd_c_block_delimiters
    // zstd_c_validate_sequences
    // zstd_c_use_block_splitter
    // zstd_c_use_row_match_finder
    // zstd_c_prefetch_c_dict_tables
    // zstd_c_enable_seq_producer_fallback
    // zstd_c_max_block_size
    // Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
    // note : never ever use experimentalParam? names directly;
    //        also, the enums values themselves are unstable and can still change.
    //
    experimental_param1 = 500
    experimental_param2 = 10
    experimental_param3 = 1000
    experimental_param4 = 1001
    experimental_param5 = 1002
    // experimental_param6  = 1003 is now ZSTD_c_targetCBlockSize
    experimental_param7  = 1004
    experimental_param8  = 1005
    experimental_param9  = 1006
    experimental_param10 = 1007
    experimental_param11 = 1008
    experimental_param12 = 1009
    experimental_param13 = 1010
    experimental_param14 = 1011
    experimental_param15 = 1012
    experimental_param16 = 1013
    experimental_param17 = 1014
    experimental_param18 = 1015
    experimental_param19 = 1016
    experimental_param20 = 1017
}

pub struct Bounds {
pub:
    error       usize
    lower_bound int
    upper_bound int
}

fn C.ZSTD_cParam_getBounds(CParameter) Bounds
fn C.ZSTD_CCtx_setParameter(voidptr, CParameter, i32) usize
fn C.ZSTD_CCtx_setPledgedSrcSize(voidptr, u64) usize

pub enum ResetDirective {
    session_only           = 1
    parameters             = 2
    session_and_parameters = 3
}

fn C.ZSTD_CCtx_reset(voidptr, ResetDirective) usize
fn C.ZSTD_compress2(voidptr, voidptr, usize, voidptr, usize) usize

pub enum DParameter {
    // Select a size limit (in power of 2) beyond which
    // the streaming API will refuse to allocate memory buffer
    // in order to protect the host from unreasonable memory requirements.
    // This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
    // By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
    // Special: value 0 means "use default maximum windowLog".
    window_log_max = 100
    // note : additional experimental parameters are also available
    // within the experimental section of the API.
    // At the time of this writing, they include :
    // ZSTD_d_format
    // zstd_d_stable_out_buffer
    // zstd_d_force_ignore_checksum
    // zstd_d_ref_multipled_dicts
    // zstd_d_disable_huffman_assembly
    // Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
    // note : never ever use experimentalParam? names directly
    experimental_param1 = 1000
    experimental_param2 = 1001
    experimental_param3 = 1002
    experimental_param4 = 1003
    experimental_param5 = 1004
    experimental_param6 = 1005
}

fn C.ZSTD_dParam_getBounds(DParameter) Bounds
fn C.ZSTD_DCtx_setParameter(voidptr, DParameter, i32) usize
fn C.ZSTD_DCtx_reset(voidptr, ResetDirective) usize

// streaming compression
pub struct InBuffer {
pub mut:
    src  voidptr
    size usize
    pos  usize
}

pub struct OutBuffer {
pub mut:
    dst  voidptr
    size usize
    pos  usize
}

fn C.ZSTD_createCStream() voidptr
fn C.ZSTD_freeCStream(voidptr) usize

pub enum EndDirective {
    // collect more data, encoder decides when to output compressed result, for optimal compression ratio
    continue = 0
    // flush any data provided so far,
    // it creates (at least) one new block, that can be decoded immediately on reception;
    // frame will continue: any future data can still reference previously compressed data, improving compression.
    // note : multithreaded compression will block to flush as much output as possible.
    flush = 1
    // flush any remaining data _and_ close current frame.
    // note that frame is only closed after compressed data is fully flushed (return value == 0).
    // After that point, any additional data starts a new frame.
    // note : each frame is independent (does not reference any content from previous frame).
    // note : multithreaded compression will block to flush as much output as possible.
    end = 2
}

fn C.ZSTD_compressStream2(voidptr, &OutBuffer, &InBuffer, EndDirective) usize
fn C.ZSTD_CStreamInSize() usize
fn C.ZSTD_CStreamOutSize() usize
fn C.ZSTD_initCStream(voidptr, i32) usize
fn C.ZSTD_compressStream(voidptr, &OutBuffer, &InBuffer) usize
fn C.ZSTD_flushStream(voidptr, &OutBuffer) usize
fn C.ZSTD_endStream(voidptr, &OutBuffer) usize

// streaming decompression
fn C.ZSTD_createDStream() voidptr
fn C.ZSTD_freeDStream(voidptr) usize
fn C.ZSTD_initDStream(voidptr) usize
fn C.ZSTD_decompressStream(voidptr, &OutBuffer, &InBuffer) usize
fn C.ZSTD_DStreamInSize() usize
fn C.ZSTD_DStreamOutSize() usize

// version_number returns runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE).
pub fn version_number() u32 {
    return C.ZSTD_versionNumber()
}

// version_string returns runtime library version, like "1.5.7".
pub fn version_string() string {
    return unsafe { tos_clone(C.ZSTD_versionString()) }
}

// is_error tells if a `usize` function result is an error code.
pub fn is_error(code usize) bool {
    return C.ZSTD_isError(code) == 1
}

// get_error_name provides readable string from an error code.
pub fn get_error_name(code usize) string {
    return unsafe { tos_clone(&u8(C.ZSTD_getErrorName(code))) }
}

// check_zstd checks the zstd error code, and return a error string.
pub fn check_error(code usize) ! {
    if is_error(code) {
        return error(get_error_name(code))
    }
}

// min_c_level returns minimum negative compression level allowed.
pub fn min_c_level() int {
    return C.ZSTD_minCLevel()
}

// max_c_level returns maximum compression level available.
pub fn max_c_level() int {
    return C.ZSTD_maxCLevel()
}

// default_c_level returns default compression level.
pub fn default_c_level() int {
    return C.ZSTD_defaultCLevel()
}

@[params]
pub struct CompressParams {
pub:
    // 1~22
    compression_level int = default_c_level()
    // how many threads will be spawned to compress in parallel
    nb_threads    int      = 1
    checksum_flag bool     = true
    strategy      Strategy = .default
}

// compresses an array of bytes using zstd and returns the compressed bytes in a new array
// extra compression parameters can be set by `params`
// Example: b := 'abcdef'.repeat(1000).bytes(); cmpr := zstd.compress(b, compression_level: 10)!; assert cmpr.len < b.len; dc := zstd.decompress(cmpr)!; assert b == dc
pub fn compress(data []u8, params CompressParams) ![]u8 {
    dst_capacity := C.ZSTD_compressBound(data.len)
    check_error(dst_capacity)!
    mut dst := []u8{len: int(dst_capacity)}
    mut cctx := new_cctx(params)!
    defer {
        cctx.free_cctx()
    }
    size := C.ZSTD_compress2(cctx.ctx, dst.data, dst.len, data.data, data.len)
    check_error(size)!
    return dst[..int(size)]
}

@[params]
pub struct DecompressParams {
pub:
    window_log_max int
}

// decompresses an array of bytes using zstd and returns the decompressed bytes in a new array
// extra decompression parameters can be set by `params`
// Example: b := 'abcdef'.repeat(1000).bytes(); cmpr := zstd.compress(b, compression_level: 10)!; assert cmpr.len < b.len; dc := zstd.decompress(cmpr)!; assert b == dc
pub fn decompress(data []u8, params DecompressParams) ![]u8 {
    dst_capacity := C.ZSTD_getFrameContentSize(data.data, frame_header_size_max)
    if dst_capacity == content_size_unknown {
        return error('The size cannot be determined, try use streaming mode to decompress data?')
    } else if dst_capacity == content_size_error {
        return error('An error occurred (e.g. invalid magic number, srcSize too small)')
    } else if dst_capacity == 0 {
        return error('The frame is valid but empty')
    }
    mut dst := []u8{len: int(dst_capacity)}
    decompressed_size := C.ZSTD_decompress(dst.data, dst.len, data.data, data.len)
    check_error(decompressed_size)!
    return dst[..int(decompressed_size)]
}

pub struct CCtx {
mut:
    ctx &C.ZSTD_CCtx_s
}

// new_cctx create a compression context.
// extra compression parameters can be set by `params`
pub fn new_cctx(params CompressParams) !&CCtx {
    mut ctx := C.ZSTD_createCCtx()
    if isnil(ctx) {
        return error('new_cctx() failed!')
    }
    mut cctx := &CCtx{ctx}
    cctx.set(.compression_level, params.compression_level)!
    $if !(tinyc && windows) {
        // TODO: tinyc on windows doesn't support multiple thread
        cctx.set(.nb_workers, params.nb_threads)!
    }
    cctx.set(.checksum_flag, if params.checksum_flag { 1 } else { 0 })!
    cctx.set(.strategy, int(params.strategy))!
    return cctx
}

// set_parameter set compression parameter `c_param` to value `val`
pub fn (mut c CCtx) set(c_param CParameter, val int) ! {
    check_error(C.ZSTD_CCtx_setParameter(c.ctx, c_param, val))!
}

// compress_stream2 do stream compress on `input`, and store compressed data in `output`.
// `mode`:
//     .zstd_e_continue => continue stream compression.
//     .zstd_e_flush => flush data
//     .zstd_e_end => it is the last frame
pub fn (mut c CCtx) compress_stream2(output &OutBuffer, input &InBuffer, mode EndDirective) !usize {
    res := C.ZSTD_compressStream2(c.ctx, output, input, mode)
    check_error(res)!
    return res
}

// free_cctx free a compression context.
pub fn (mut c CCtx) free_cctx() usize {
    return C.ZSTD_freeCCtx(c.ctx)
}

struct C.ZSTD_CCtx_s {}

struct C.ZSTD_DCtx_s {}

pub struct DCtx {
mut:
    ctx &C.ZSTD_DCtx_s
}

// new_dctx creates a decompression context.
// extra decompression parameters can be set by `params`
pub fn new_dctx(params DecompressParams) !&DCtx {
    mut ctx := C.ZSTD_createDCtx()
    if isnil(ctx) {
        return error('new_dctx() failed!')
    }
    mut dctx := &DCtx{ctx}
    dctx.set(.window_log_max, params.window_log_max)!
    return dctx
}

// set_parameter sets decompression parameter `d_param` to value `val`
pub fn (mut d DCtx) set(d_param DParameter, val int) ! {
    check_error(C.ZSTD_DCtx_setParameter(d.ctx, d_param, val))!
}

// decompress_stream do stream decompress on `input`, and store decompressed data in `output`.
// return remaining bytes in `input` stream
pub fn (mut d DCtx) decompress_stream(output &OutBuffer, input &InBuffer) !usize {
    res := C.ZSTD_decompressStream(d.ctx, output, input)
    check_error(res)!
    return res
}

// free_cctx free a compression context
pub fn (mut d DCtx) free_dctx() usize {
    return C.ZSTD_freeDCtx(d.ctx)
}

// store_array compress an `array`'s data, and store it to file `fname`.
// extra compression parameters can be set by `params`
// WARNING: Because struct padding, some data in struct may be marked unused.
// So, when `store_array`, it will cause memory fsanitize fail with 'use-of-uninitialized-value'.
// It can be safely ignored.
// For example, following struct may cause memory fsanitize fail:
// struct MemoryTrace {
//     operation u8
//     address   u64
//     size      u8
// }
// By changing it into following , you can pass the memory fsanitize check :
// struct MemoryTrace {
//     operation u64
//     address   u64
//     size      u64
// }
pub fn store_array[T](fname string, array []T, params CompressParams) ! {
    mut fout := os.open_file(fname, 'wb')!
    mut cctx := new_cctx(params)!
    defer {
        cctx.free_cctx()
        fout.close()
    }

    mut buf_out := []u8{len: buf_out_size}
    mut input := &InBuffer{}
    mut output := &OutBuffer{}
    // first, write the array.len to file
    mut len_buf := []u8{len: 8}
    binary.little_endian_put_u64(mut len_buf, u64(array.len))
    input.src = len_buf.data
    input.size = 8
    input.pos = 0
    output.dst = buf_out.data
    output.size = buf_out_size
    output.pos = 0
    mut remaining := cctx.compress_stream2(output, input, .flush)!
    fout.write(buf_out[..int(output.pos)])!
    // then, write the array.data to file
    input.src = array.data
    input.size = usize(array.len * int(sizeof(T)))
    input.pos = 0
    output.dst = buf_out.data
    output.size = buf_out_size
    output.pos = 0
    for {
        output.dst = buf_out.data
        output.size = buf_out_size
        output.pos = 0
        remaining = cctx.compress_stream2(output, input, .end)!
        fout.write(buf_out[..int(output.pos)])!
        if remaining == 0 {
            break
        }
    }
}

// load_array return an array which data is decompressed from a file `fname`.
// extra decompression parameters can be set by `params`
pub fn load_array[T](fname string, params DecompressParams) ![]T {
    mut fin := os.open_file(fname, 'rb')!
    mut dctx := new_dctx(params)!
    defer {
        dctx.free_dctx()
        fin.close()
    }

    mut buf_in := []u8{len: buf_in_size}
    mut len_buf := []u8{len: 8}
    mut input := &InBuffer{}
    mut output := &OutBuffer{}
    mut last_ret := usize(0)
    mut last_chunk := false
    // first, read the array.len from file
    mut read_len := fin.read(mut buf_in)!
    last_chunk = read_len < buf_in.len
    input.src = buf_in.data
    input.size = usize(read_len)
    input.pos = 0
    output.dst = len_buf.data
    output.size = usize(len_buf.len)
    output.pos = 0
    last_ret = dctx.decompress_stream(output, input)!
    len := binary.little_endian_u64(len_buf)
    // then, read the array.data from file
    mut result := []T{len: int(len)}
    output.dst = result.data
    output.size = usize(result.len) * sizeof(T)
    output.pos = 0
    last_ret = dctx.decompress_stream(output, input)!
    for !last_chunk {
        read_len = fin.read(mut buf_in)!
        last_chunk = read_len < buf_in.len
        input.size = usize(read_len)
        input.pos = 0
        for input.pos < input.size {
            last_ret = dctx.decompress_stream(output, input)!
        }
        if read_len < buf_in.len {
            break
        }
    }
    if last_ret != 0 {
        // The last return value from ZSTD_decompressStream did not end on a
        // frame, but we reached the end of the file! We assume this is an
        // error, and the input was truncated.
        return error('EOF before end of stream: ${last_ret}')
    }
    return result
}