Primitive APIs in xf::compression

blockPacker

#include "block_packer.hpp"
template <int DATAWIDTH = 512>
void blockPacker (
    hls::stream <ap_uint <DATAWIDTH>>& inStream,
    hls::stream <uint32_t>& inStreamSize,
    hls::stream <ap_uint <DATAWIDTH>>& outStream,
    hls::stream <bool>& outStreamEos,
    hls::stream <uint32_t>& outCompressedSize
    )

Compression Packer module packs the compressed data.

Parameters:

DATAWIDTH input data width
inStream input data
outStream output data
inStreamSize size of the data in input stream
outStreamEos output end of stream
outCompressedSize total compressed packed data size

huffmanDecoder

#include "huffman_decoder.hpp"
template <
    eHuffmanType DECODER = FULL,
    int ByteGenLoopII = 1
    >
void huffmanDecoder (
    hls::stream <ap_uint <16>>& inStream,
    hls::stream <ap_uint <32>>& outStream,
    hls::stream <bool>& endOfStream,
    uint32_t input_size
    )

This module is ZLIB/GZIP Fixed, Dynamic and Stored block supported decoder. It takes ZLIB/GZIP Huffman encoded data as input and generates decoded data in LZ77 format (Literal, Length, Offset).

Parameters:

DECODER Fixed, Full, Dynamic huffman block support
ByteGenLoopII core bytegenerator loop initiation interval
USE_GZIP switch that controls GZIP/ZLIB header processing
inStream input bit packed data
outStream output lz77 compressed output in the form of 32bit packets (Literals, Match Length, Distances)
endOfStream output completion of execution
input_size input data size

huffmanEncoderStream

#include "huffman_encoder.hpp"
void huffmanEncoderStream (
    hls::stream <ap_uint <32>>& inStream,
    hls::stream <uint16_t>& outStream,
    hls::stream <uint8_t>& outStreamSize,
    hls::stream <uint32_t>& inputSize,
    hls::stream <uint16_t>& inStreamCodes,
    hls::stream <uint8_t>& inStreamCodeSize
    )

This module does zlib/gzip dynamic huffman encoding.

Parameters:

inStream input packet of 32bit size which contains either literal or match length and distance information. Example: [Literal (1 Byte) | ML (1 Byte) | DIST (2 Bytes)]
outStream output bit encoded LZ77 compressed data
outStreamSize output Stream Size
inputSize input size stream
inStreamCodes Huffman Codes
inStreamCodeSize HuffmanCode Lengths

huffmanEncoder

#include "huffman_encoder.hpp"
void huffmanEncoder (
    hls::stream <ap_uint <32>>& inStream,
    hls::stream <uint16_t>& outStream,
    hls::stream <uint8_t>& outStreamSize,
    uint32_t input_size,
    hls::stream <uint16_t>& inStreamCodes,
    hls::stream <uint8_t>& inStreamCodeSize
    )

This module does zlib/gzip dynamic huffman encoding.

Parameters:

inStream input packet of 32bit size which contains either literal or match length and distance information. Example: [Literal (1 Byte) | ML (1 Byte) | DIST (2 Bytes)]
outStream output bit encoded LZ77 compressed data
outStreamSize output Stream Size
input_size uncompressed data input size
inStreamCodes Huffman Codes
inStreamCodeSize HuffmanCode Lengths

lz4Compress

#include "lz4_compress.hpp"
template <
    int MAX_LIT_COUNT,
    int PARALLEL_UNITS
    >
static void lz4Compress (
    hls::stream <ap_uint <32>>& inStream,
    hls::stream <ap_uint <8>>& outStream,
    uint32_t max_lit_limit [PARALLEL_UNITS],
    uint32_t input_size,
    hls::stream <bool>& endOfStream,
    hls::stream <uint32_t>& compressdSizeStream,
    uint32_t index
    )

This is the core compression module which seperates the input stream into two output streams, one literal stream and other offset stream, then lz4 encoding is done.

Parameters:

inStream Input data stream
outStream Output data stream
max_lit_limit Size for compressed stream
input_size Size of input data
endOfStream Stream indicating that all data is processed or not
compressdSizeStream Gives the compressed size for each 64K block

lz4Decompress

#include "lz4_decompress.hpp"
void lz4Decompress (
    hls::stream <ap_uint <8>>& inStream,
    hls::stream <ap_uint <32>>& outStream,
    uint32_t input_size
    )

This module reads the compressed data from input stream and decodes the offset, match length and literals by processing in various decompress states.

Parameters:

inStream Input stream 8bit
outStream Output stream 32bit
input_size Input size

lzDecompress

#include "lz_decompress.hpp"
template <
    int HISTORY_SIZE,
    int LOW_OFFSET = 8
    >
void lzDecompress (
    hls::stream <ap_uint <32>>& inStream,
    hls::stream <ap_uint <8>>& outStream,
    uint32_t original_size
    )

This module writes the literals to the output stream as it is and when match length and offset are read, the literals will be read from the local dictionary based on offset until match length.

Parameters:

LOW_OFFSET low offset
HISTORY_SIZE history size
inStream input stream
outStream output stream
original_size original size

lzMultiByteDecompress

#include "lz_decompress.hpp"
template <
    int PARALLEL_BYTES,
    int HISTORY_SIZE,
    class SIZE_DT = uint8_t,
    class SIZE_OFFSET = ap_uint<16>
    >
void lzMultiByteDecompress (
    hls::stream <SIZE_DT>& litlenStream,
    hls::stream <ap_uint <PARALLEL_BYTES*8>>& litStream,
    hls::stream <SIZE_OFFSET>& offsetStream,
    hls::stream <SIZE_DT>& matchlenStream,
    hls::stream <ap_uint <PARALLEL_BYTES*8>>& outStream,
    hls::stream <bool>& endOfStream,
    hls::stream <uint64_t>& sizeOutStream
    )

This module writes the literals to the output stream as it is and when match length and offset are read, the literals will be read from the local dictionary based on offset until match length. This module can process data in parallel defined by PARALLEL_BYTES template argument.

Parameters:

PARALLEL_BYTES number of bytes processed in parallel (4, 8)
HISTORY_SIZE history size
SIZE_DT input data type
SIZE_OFFSET offset data type
litlenStream literal length stream
litStream literals only stream
offsetStream offset only stream
matchlenStream match length only stream
outStream output stream
endOfStream end of stream
sizeOutStream output size stream

lzBestMatchFilter

#include "lz_optional.hpp"
template <
    int MATCH_LEN,
    int OFFSET_WINDOW
    >
void lzBestMatchFilter (
    hls::stream <compressd_dt>& inStream,
    hls::stream <compressd_dt>& outStream,
    uint32_t input_size
    )

Objective of this module is to pick character with higher match length in the offset window range.

Parameters:

inStream input stream
outStream output stream
input_size input stream size

lzBooster

lzBooster overload (1)

#include "lz_optional.hpp"
template <
    int MAX_MATCH_LEN,
    int BOOSTER_OFFSET_WINDOW = 16 * 1024,
    int LEFT_BYTES = 64
    >
void lzBooster (
    hls::stream <compressd_dt>& inStream,
    hls::stream <compressd_dt>& outStream,
    hls::stream <uint32_t>& inSize
    )

This module helps in improving the compression ratio. Finds a better match length by performing more character matches with supported max match, while maintaining an offset window. Booster offset Window template argument (default value is 16K) internally consume BRAM memory to implement history window. Higher the booster value can give better compression ratio but will consume more BRAM resources.

Parameters:

MAX_MATCH_LEN maximum length allowed for character match
BOOSTER_OFFSET_WINDOW offset window to store/match the character
inStream input stream 32bit per read
outStream output stream 32bit per write
inSize input size stream
outSize output size stream
left_bytes last 64 left over bytes

lzBooster overload (2)

#include "lz_optional.hpp"
template <
    int MAX_MATCH_LEN,
    int BOOSTER_OFFSET_WINDOW = 16 * 1024,
    int LEFT_BYTES = 64
    >
void lzBooster (
    hls::stream <compressd_dt>& inStream,
    hls::stream <compressd_dt>& outStream,
    uint32_t input_size
    )

This module helps in improving the compression ratio. Finds a better match length by performing more character matches with supported max match, while maintaining an offset window. Booster offset Window template argument (default value is 16K) internally consume BRAM memory to implement history window. Higher the booster value can give better compression ratio but will consume more BRAM resources.

Parameters:

MAX_MATCH_LEN maximum length allowed for character match
BOOSTER_OFFSET_WINDOW offset window to store/match the character
inStream input stream 32bit per read
outStream output stream 32bit per write
input_size input size
left_bytes last 64 left over bytes

lzFilter

#include "lz_optional.hpp"
template <int LEFT_BYTES = 64>
static void lzFilter (
    hls::stream <compressd_dt>& inStream,
    hls::stream <compressd_dt>& outStream,
    uint32_t input_size
    )

This module checks if match length exists, and if match length exists it filters the match length -1 characters writing to output stream.

Parameters:

MATCH_LEN length of matched segment
OFFSET_WINDOW output window
inStream input stream
outStream output stream
input_size input stream size
left_bytes bytes left in block

snappyCompress

#include "snappy_compress.hpp"
template <
    int MAX_LIT_COUNT,
    int MAX_LIT_STREAM_SIZE,
    int PARALLEL_UNITS
    >
static void snappyCompress (
    hls::stream <ap_uint <32>>& inStream,
    hls::stream <ap_uint <8>>& outStream,
    uint32_t max_lit_limit [PARALLEL_UNITS],
    uint32_t input_size,
    hls::stream <bool>& endOfStream,
    hls::stream <uint32_t>& compressdSizeStream,
    uint32_t index
    )

This is the core compression module which seperates the input stream into two output streams, one literal stream and other offset stream, then encoding is done based on the snappy algorithm.

Parameters:

inStream Input data stream
outStream Output data stream
max_lit_limit Size for compressed stream
input_size Size of input data
endOfStream Stream indicating that all data is processed or not
compressdSizeStream Gives the compressed size for each 64K block

snappyDecompress

#include "snappy_decompress.hpp"
static void snappyDecompress (
    hls::stream <ap_uint <8>>& inStream,
    hls::stream <ap_uint <32>>& outStream,
    uint32_t input_size
    )

This module decodes the compressed data based on the snappy decompression format.

Parameters:

inStream input stream
outStream output stream
input_size input data size

zstdDecompressStream

#include "zstd_decompress.hpp"
template <
    int PARALLEL_BYTE,
    int BLOCK_SIZE_KB,
    int LZ_MAX_OFFSET,
    int LMO_WIDTH
    >
void zstdDecompressStream (
    hls::stream <ap_uint <8*PARALLEL_BYTE>>& inStream,
    hls::stream <ap_uint <4>>& inStrobe,
    hls::stream <ap_uint <8*PARALLEL_BYTE>>& outStream,
    hls::stream <bool>& endOfStream,
    hls::stream <uint64_t>& sizeOutStream
    )

This module decompresses the ZStd compressed file read from input stream. It reads the input stream till valid strobe input is provided. It produces the decompressed data at the output stream.

Parameters:

PARALLEL_BYTE Data stream width in bytes
BLOCK_SIZE_KB ZStd block size
LZ_MAX_OFFSET LZ history size or Window size
LMO_WIDTH data width for offset data
inStream input stream
inStrobe valid input strobe stream
outStream output stream
endOfStream end of output stream flag stream
sizeOutStream output data size

zstdDecompressCore

#include "zstd_decompress.hpp"
template <
    int PARALLEL_BYTE,
    int BLOCK_SIZE_KB,
    int LZ_MAX_OFFSET,
    int LMO_WIDTH = 15
    >
void zstdDecompressCore (
    hls::stream <ap_axiu <8*PARALLEL_BYTE, 0, 0, 0>>& inStream,
    hls::stream <ap_axiu <8*PARALLEL_BYTE, 0, 0, 0>>& outStream,
    hls::stream <ap_axiu <64, 0, 0, 0>>& outSizeStream,
    uint64_t inputSize
    )

This module decompresses the ZStd compressed file read from input stream. It reads the input stream till the given input size. It produces the decompressed data at the output stream.

Parameters:

PARALLEL_BYTE Data stream width in bytes
BLOCK_SIZE_KB ZStd block size
LZ_MAX_OFFSET LZ history size or Window size
LMO_WIDTH data width for offset data
inStream input stream
outStream output stream
outSizeStream output data size
inputSize size of input data