namespace compression

// namespaces

namespace xf::compression::details

// typedefs

typedef ap_uint <32> bitBufferTypeLL
typedef ap_uint <48> bitBufferType
typedef ap_uint <12> Histogram
typedef ap_uint <MAX_FREQ_DWIDTH> Frequency
typedef struct xf::compression::lz4BlockInfo dt_lz4BlockInfo
typedef ap_uint <32> compressd_dt
typedef ap_uint <kGMemDWidth> uintMemWidth_t
typedef struct xf::compression::BlockInfo dt_blockInfo

// enums

enum FileFormat
enum eHuffmanType

// structs

struct BlockInfo
struct Codeword

template <int MAX_FREQ_DWIDTH>
struct Symbol

struct lz4BlockInfo

// global variables

static const uint8_t c_tgnSymbolBits
static const uint8_t c_tgnBitlengthBits
static const uint8_t c_lengthHistogram
static const uint16_t c_tgnSymbolSize
static const uint16_t c_tgnTreeDepth
static const uint16_t c_tgnMaxBits
const auto BSIZE_STD_64KB
const auto BSIZE_STD_256KB
const auto BSIZE_STD_1024KB
const auto BSIZE_STD_4096KB
const auto MAX_BSIZE_64KB
const auto MAX_BSIZE_256KB
const auto MAX_BSIZE_1024KB
const auto MAX_BSIZE_4096KB
const auto MAGIC_HEADER_SIZE
const auto MAGIC_BYTE_1
const auto MAGIC_BYTE_2
const auto MAGIC_BYTE_3
const auto MAGIC_BYTE_4
const auto FLG_BYTE
const auto NO_COMPRESS_BIT
const auto BSIZE_NCOMP_64
const auto BSIZE_NCOMP_256
const auto BSIZE_NCOMP_1024
const auto BSIZE_NCOMP_4096
const int kGMemDWidth
const int c_lTreeSize
const int c_dTreeSize
const int c_bLTreeSize
const int c_maxCodeSize

Global Variables

const auto BSIZE_STD_64KB

Below are the codes as per LZ4 standard for various maximum block sizes supported.

const auto MAX_BSIZE_64KB

Maximum block sizes supported by LZ4

const auto NO_COMPRESS_BIT

This value is used to set uncompressed block size value. 4th byte is always set to below and placed as uncompressed byte

const auto BSIZE_NCOMP_64

In case of uncompressed block Values below are used to set 3rd byte to following values w.r.t various maximum block sizes supported by standard

blockPacker

#include "block_packer.hpp"

template <int DATAWIDTH = 512>
void blockPacker (
    hls::stream <ap_uint <DATAWIDTH>>& inStream,
    hls::stream <uint32_t>& inStreamSize,
    hls::stream <ap_uint <DATAWIDTH>>& outStream,
    hls::stream <bool>& outStreamEos,
    hls::stream <uint32_t>& outCompressedSize
    )

Compression Packer module packs the compressed data.

Parameters:

DATAWIDTH	input data width
inStream	input data
outStream	output data
inStreamSize	size of the data in input stream
outStreamEos	output end of stream
outCompressedSize	total compressed packed data size

huffmanDecoderLL

#include "huffman_decoder.hpp"

template <
    eHuffmanType DECODER = FULL,
    FileFormat FORMAT = BOTH
    >
void huffmanDecoderLL (
    hls::stream <ap_uint <16>>& inStream,
    hls::stream <bool>& inEos,
    hls::stream <ap_uint <16>>& outStream
    )

This module is ZLIB/GZIP Fixed, Dynamic and Stored block supported decoder. It takes ZLIB/GZIP Huffman encoded data as input and generates decoded data in LZ77 format (Literal, Length, Offset).

Parameters:

DECODER	Fixed, Full, Dynamic huffman block support
FORMAT	switch that controls GZIP/ZLIB header processing
inStream	input bit packed data
inEos	input bit to mark end of file
outStream	output bytes for LZ module (Literal, Length, Offset)
checkSum	support checksum in gzip decompress

huffmanDecoder

#include "huffman_decoder.hpp"

template <eHuffmanType DECODER = FULL>
void huffmanDecoder (
    hls::stream <ap_uint <16>>& inStream,
    hls::stream <bool>& inEos,
    hls::stream <ap_uint <17>>& outStream
    )

This module is ZLIB/GZIP Fixed, Dynamic and Stored block supported decoder. It takes ZLIB/GZIP Huffman encoded data as input and generates decoded data in LZ77 format (Literal, Length, Offset).

Parameters:

DECODER	Fixed, Full, Dynamic huffman block support
inStream	input bit packed data
inEos	input end of stream
outStream	output lz77 compressed output in the form of 32bit packets (Literals, Match Length, Distances)

huffmanEncoderStream

#include "huffman_encoder.hpp"

void huffmanEncoderStream (
    hls::stream <IntVectorStream_dt <32, 1>>& inStream,
    hls::stream <DSVectorStream_dt <HuffmanCode_dt <c_maxBits>, 1>>& hufCodeInStream,
    hls::stream <DSVectorStream_dt <HuffmanCode_dt <c_maxBits>, 1>>& hufCodeOutStream
    )

This module does zlib/gzip dynamic huffman encoding.

Parameters:

inStream	input packet of 32bit size which contains either literal or match length and distance information. Example: [Literal (1 Byte) | ML (1 Byte) | DIST (2 Bytes)]
hufCodeInStream	Huffman Codes input
hufCodeOutStream	HuffmanCodes output

lz4Compress

#include "lz4_compress.hpp"

template <
    int MAX_LIT_COUNT,
    int PARALLEL_UNITS
    >
static void lz4Compress (
    hls::stream <ap_uint <32>>& inStream,
    hls::stream <ap_uint <8>>& outStream,
    uint32_t max_lit_limit [PARALLEL_UNITS],
    uint32_t input_size,
    hls::stream <bool>& endOfStream,
    hls::stream <uint32_t>& compressdSizeStream,
    uint32_t index
    )

This is the core compression module which seperates the input stream into two output streams, one literal stream and other offset stream, then lz4 encoding is done.

Parameters:

PARALLEL_UNITS	number of parallel units
MAX_LIT_COUNT	encoded literal length count
inStream	Input data stream
outStream	Output data stream
max_lit_limit	Size for compressed stream
input_size	Size of input data
endOfStream	Stream indicating that all data is processed or not
compressdSizeStream	Gives the compressed size for each 64K block
index	index value

lz4Decompress

#include "lz4_decompress.hpp"

void lz4Decompress (
    hls::stream <ap_uint <8>>& inStream,
    hls::stream <ap_uint <32>>& outStream,
    uint32_t input_size
    )

This module reads the compressed data from input stream and decodes the offset, match length and literals by processing in various decompress states.

Parameters:

inStream	Input stream 8bit
outStream	Output stream 32bit
input_size	Input size

lzDecompress

#include "lz_decompress.hpp"

template <
    int HISTORY_SIZE,
    int LOW_OFFSET = 8
    >
void lzDecompress (
    hls::stream <ap_uint <32>>& inStream,
    hls::stream <ap_uint <8>>& outStream,
    uint32_t original_size
    )

This module writes the literals to the output stream as it is and when match length and offset are read, the literals will be read from the local dictionary based on offset until match length.

Parameters:

LOW_OFFSET	low offset
HISTORY_SIZE	history size
inStream	input stream
outStream	output stream
original_size	original size

lzMultiByteDecompress

#include "lz_decompress.hpp"

template <
    int PARALLEL_BYTES,
    int HISTORY_SIZE,
    class SIZE_DT = uint8_t,
    class SIZE_OFFSET = ap_uint<16>
    >
void lzMultiByteDecompress (
    hls::stream <SIZE_DT>& litlenStream,
    hls::stream <ap_uint <PARALLEL_BYTES*8>>& litStream,
    hls::stream <SIZE_OFFSET>& offsetStream,
    hls::stream <SIZE_DT>& matchlenStream,
    hls::stream <ap_uint < (PARALLEL_BYTES*8)+PARALLEL_BYTES>>& outStream
    )

This module writes the literals to the output stream as it is and when match length and offset are read, the literals will be read from the local dictionary based on offset until match length. This module can process data in parallel defined by PARALLEL_BYTES template argument.

Parameters:

PARALLEL_BYTES	number of bytes processed in parallel (4, 8)
HISTORY_SIZE	history size
SIZE_DT	input data type
SIZE_OFFSET	offset data type
litlenStream	literal length stream
litStream	literals only stream
offsetStream	offset only stream
matchlenStream	match length only stream
outStream	output stream

lzBestMatchFilter

lzBestMatchFilter overload (1)

#include "lz_optional.hpp"

template <
    int MATCH_LEN,
    int OFFSET_WINDOW
    >
void lzBestMatchFilter (
    hls::stream <compressd_dt>& inStream,
    hls::stream <compressd_dt>& outStream,
    uint32_t input_size
    )

Objective of this module is to pick character with higher match length in the offset window range.

Parameters:

MATCH_LEN	match length
OFFSET_WINDOW	offset window range
inStream	input stream
outStream	output stream
input_size	input stream size

lzBooster

lzBooster overload (1)

#include "lz_optional.hpp"

template <
    int MAX_MATCH_LEN,
    int BLOCKSIZE = 32768,
    int BOOSTER_OFFSET_WINDOW = 16 * 1024,
    int LEFT_BYTES = 64
    >
void lzBooster (
    hls::stream <IntVectorStream_dt <32, 1>>& inStream,
    hls::stream <IntVectorStream_dt <32, 1>>& outStream
    )

This module helps in improving the compression ratio. Finds a better match length by performing more character matches with supported max match, while maintaining an offset window. Booster offset Window template argument (default value is 16K) internally consume BRAM memory to implement history window. Higher the booster value can give better compression ratio but will consume more BRAM resources.

Parameters:

MAX_MATCH_LEN	maximum length allowed for character match
BOOSTER_OFFSET_WINDOW	offset window to store/match the character
LEFT_BYTES	last 64 left over bytes
inStream	input stream 32bit per read
outStream	output stream 32bit per write

lzBooster overload (2)

#include "lz_optional.hpp"

template <
    int MAX_MATCH_LEN,
    int BOOSTER_OFFSET_WINDOW = 16 * 1024,
    int LEFT_BYTES = 64
    >
void lzBooster (
    hls::stream <compressd_dt>& inStream,
    hls::stream <compressd_dt>& outStream,
    uint32_t input_size
    )

This module helps in improving the compression ratio. Finds a better match length by performing more character matches with supported max match, while maintaining an offset window. Booster offset Window template argument (default value is 16K) internally consume BRAM memory to implement history window. Higher the booster value can give better compression ratio but will consume more BRAM resources.

Parameters:

MAX_MATCH_LEN	maximum length allowed for character match
BOOSTER_OFFSET_WINDOW	offset window to store/match the character
inStream	input stream 32bit per read
outStream	output stream 32bit per write
input_size	input size
left_bytes	last 64 left over bytes

lzFilter

#include "lz_optional.hpp"

template <int LEFT_BYTES = 64>
static void lzFilter (
    hls::stream <compressd_dt>& inStream,
    hls::stream <compressd_dt>& outStream,
    uint32_t input_size
    )

This module checks if match length exists, and if match length exists it filters the match length -1 characters writing to output stream.

Parameters:

MATCH_LEN	length of matched segment
OFFSET_WINDOW	output window
inStream	input stream
outStream	output stream
input_size	input stream size
left_bytes	bytes left in block

snappyCompress

#include "snappy_compress.hpp"

template <
    int MAX_LIT_COUNT,
    int MAX_LIT_STREAM_SIZE,
    int PARALLEL_UNITS
    >
static void snappyCompress (
    hls::stream <ap_uint <32>>& inStream,
    hls::stream <ap_uint <8>>& outStream,
    uint32_t max_lit_limit [PARALLEL_UNITS],
    uint32_t input_size,
    hls::stream <bool>& endOfStream,
    hls::stream <uint32_t>& compressdSizeStream,
    uint32_t index
    )

This is the core compression module which seperates the input stream into two output streams, one literal stream and other offset stream, then encoding is done based on the snappy algorithm.

Parameters:

MAX_LIT_COUNT	encoded literal length count
MAX_LIT_STREAM_SIZE	encoded literal stream size
PARALLEL_UNITS	number of parallel processed units
inStream	Input data stream
outStream	Output data stream
max_lit_limit	Size for compressed stream
input_size	Size of input data
endOfStream	Stream indicating that all data is processed or not
compressdSizeStream	Gives the compressed size for each 64K block
index	index value

snappyDecompress

#include "snappy_decompress.hpp"

static void snappyDecompress (
    hls::stream <ap_uint <8>>& inStream,
    hls::stream <ap_uint <32>>& outStream,
    uint32_t input_size
    )

This module decodes the compressed data based on the snappy decompression format.

Parameters:

inStream	input stream
outStream	output stream
input_size	input data size

zstdCompressCore

#include "zstd_compress.hpp"

template <
    int BLOCK_SIZE,
    int LZWINDOW_SIZE,
    int MIN_BLCK_SIZE,
    int PARALLEL_HUFFMAN = 8,
    int PARALLEL_LIT_STREAMS = 4,
    int MIN_MATCH = 3
    >
void zstdCompressCore (
    hls::stream <IntVectorStream_dt <8, 1>>& inStream,
    hls::stream <IntVectorStream_dt <8, 4>>& outStream
    )

This module compresses the input file read from input stream. It produces the ZSTD compressed data at the output stream.

Parameters:

BLOCK_SIZE	ZStd block size
LZWINDOW_SIZE	LZ77 history size or Window size
MIN_BLCK_SIZE	Minimum block size, less than that will be considered stored block
PARALLEL_HUFFMAN	Number of Huffman encoding units used
PARALLEL_LIT_STREAMS	Number of parallel literal streams encoded using huffman
MIN_MATCH	Minimum match in LZ77
inStream	input stream
outStream	output stream

zstdCompressStreaming

#include "zstd_compress.hpp"

template <
    int IN_DWIDTH,
    int OUT_DWIDTH,
    int BLOCK_SIZE,
    int LZWINDOW_SIZE,
    int MIN_BLCK_SIZE = 1020
    >
void zstdCompressStreaming (
    hls::stream <ap_axiu <IN_DWIDTH, 0, 0, 0>>& inStream,
    hls::stream <ap_axiu <OUT_DWIDTH, 0, 0, 0>>& outStream
    )

This module is top level wrapper for zstd compression core module It compresses the input file read from input axi stream. It produces the ZSTD compressed data at the output axi stream.

Parameters:

IN_DWIDTH	Input stream data bit-width
OUT_DWIDTH	Output stream data bit-width
BLOCK_SIZE	ZStd block size
LZWINDOW_SIZE	LZ77 history size or Window size
MIN_BLCK_SIZE	Minimum block size, less than that will be considered stored block
inStream	input stream
outStream	output stream

zstdCompressQuadCore

#include "zstd_compress_multicore.hpp"

template <
    int CORE_COUNT,
    int BLOCK_SIZE,
    int LZWINDOW_SIZE,
    int MIN_BLCK_SIZE,
    int MIN_MATCH = 3
    >
void zstdCompressQuadCore (
    hls::stream <IntVectorStream_dt <8, 8>>& inStream,
    hls::stream <IntVectorStream_dt <8, 8>>& outStream
    )

This module compresses the input file read from input stream using multiple lz77 modules. It produces the ZSTD compressed data at the output stream.

Parameters:

CORE_COUNT	Total number of lz77 cores
BLOCK_SIZE	ZStd block size
LZWINDOW_SIZE	LZ77 history size or Window size
MIN_BLCK_SIZE	Minimum block size, less than that will be considered stored block
PARALLEL_HUFFMAN	Number of Huffman encoding units used
MIN_MATCH	Minimum match in LZ77
inStream	input stream
outStream	output stream

zstdCompressMultiCoreStreaming

#include "zstd_compress_multicore.hpp"

template <
    int IN_DWIDTH,
    int OUT_DWIDTH,
    int BLOCK_SIZE,
    int LZWINDOW_SIZE,
    int MIN_BLCK_SIZE,
    int CORE_COUNT
    >
void zstdCompressMultiCoreStreaming (
    hls::stream <ap_axiu <IN_DWIDTH, 0, 0, 0>>& inStream,
    hls::stream <ap_axiu <OUT_DWIDTH, 0, 0, 0>>& outStream
    )

This module is top level wrapper for zstd compression core module It compresses the input file read from input axi stream. It produces the ZSTD compressed data at the output axi stream.

Parameters:

IN_DWIDTH	Input stream data bit-width
OUT_DWIDTH	Output stream data bit-width
BLOCK_SIZE	ZStd block size
LZWINDOW_SIZE	LZ77 history size or Window size
MIN_BLCK_SIZE	Minimum block size, less than that will be considered stored block
inStream	input stream
outStream	output stream

zstdDecompressStream

#include "zstd_decompress.hpp"

template <
    int IN_BYTES = 4,
    int OUT_BYTES = 8,
    int BLOCK_SIZE_KB,
    int LZ_MAX_OFFSET,
    int LMO_WIDTH,
    bool SEQ_LOW_LATENCY = false,
    bool LOW_LATENCY = false
    >
void zstdDecompressStream (
    hls::stream <ap_uint <IN_BYTES*8>>& inStream,
    hls::stream <ap_uint <4>>& inStrobe,
    hls::stream <ap_uint < (OUT_BYTES*8)+OUT_BYTES>>& outStream
    )

This module decompresses the ZStd compressed file read from input stream. It reads the input stream till valid strobe input is provided. It produces the decompressed data at the output stream.

Parameters:

PARALLEL_BYTE	Data stream width in bytes
BLOCK_SIZE_KB	ZStd block size
LZ_MAX_OFFSET	LZ history size or Window size
LMO_WIDTH	data width for offset data
inStream	input stream
inStrobe	valid input strobe stream
outStream	output stream

zstdDecompressCore

#include "zstd_decompress.hpp"

template <
    int IN_BYTES = 4,
    int OUT_BYTES = 8,
    int BLOCK_SIZE_KB,
    int LZ_MAX_OFFSET,
    int LMO_WIDTH = 15,
    bool SEQ_LOW_LATENCY = false,
    bool LOW_LATENCY = false
    >
void zstdDecompressCore (
    hls::stream <ap_axiu <IN_BYTES*8, 0, 0, 0>>& inStream,
    hls::stream <ap_axiu <OUT_BYTES*8, 0, 0, 0>>& outStream
    )

This module decompresses the ZStd compressed file read from input stream. It reads the input stream till the given input size. It produces the decompressed data at the output stream.

Parameters:

PARALLEL_BYTE	Data stream width in bytes
BLOCK_SIZE_KB	ZStd block size
LZ_MAX_OFFSET	LZ history size or Window size
LMO_WIDTH	data width for offset data
inStream	input stream
outStream	output stream