namespace internal

// global variables

const int L_BIT
const int BURST_LEN
const int H_BIT
const int H_BIT_PAGE
const int PAGE_SIZE_BYTE
const int PAGE_VNUM_BIT64
const int PAGE_VNUM_BIT32

readSchema

#include "xf_data_analytics/dataframe/csv_parser.hpp"
template <int PU_NUM>
void readSchema (
    ap_uint <8>* schema,
    ap_uint <8> type_buff [PU_NUM][256],
    ap_uint <4> type_valid_buff [PU_NUM][256],
    ap_uint <9>& num_of_column
    )

read the schema and cache the type of each enable column

mergeLineUnitL1

#include "xf_data_analytics/dataframe/csv_parser.hpp"
void mergeLineUnitL1 (
    hls::stream <Object> i_obj_array_strm [2],
    hls::stream <bool> i_ln_e_strm [2],
    hls::stream <Object>& o_obj_strm
    )

merge two obj-stream by load-balance type, and sink the line break signal

mergeLineUnitL2

#include "xf_data_analytics/dataframe/csv_parser.hpp"
void mergeLineUnitL2 (
    hls::stream <Object> i_obj_array_strm [2],
    hls::stream <Object>& o_obj_strm
    )

merge two obj-stream by load-balance type

mergeLine

#include "xf_data_analytics/dataframe/csv_parser.hpp"
template <int PU_NUM>
void mergeLine (
    hls::stream <Object> i_obj_array_strm [PU_NUM/2][2],
    hls::stream <bool> i_ln_e_strm [PU_NUM/2][2],
    hls::stream <Object>& o_obj_strm
    )

top function of merging obj-stream from PUs, support 2/4/8-to-1 only

parseCSVCore

#include "xf_data_analytics/dataframe/csv_parser.hpp"
template <int PU_NUM>
void parseCSVCore (
    ap_uint <128>* csv_buff,
    ap_uint <8> type_buff [PU_NUM][256],
    ap_uint <4> type_valid_buff [PU_NUM][256],
    const ap_uint <9> num_of_column,
    hls::stream <Object>& o_obj_strm
    )

main function of CSV parser

mergeLineUnitL1

#include "xf_data_analytics/dataframe/csv_parser_v2.hpp"
void mergeLineUnitL1 (
    ap_uint <2> pu_grp,
    hls::stream <Object> i_obj_array_strm [2],
    hls::stream <bool> i_ln_e_strm [2],
    hls::stream <Object>& o_obj_strm,
    ap_uint <32> line_cnt [2]
    )

merge two obj-stream by load-balance type, and sink the line break signal

mergeLine

#include "xf_data_analytics/dataframe/csv_parser_v2.hpp"
template <int PU_NUM>
void mergeLine (
    hls::stream <Object> i_obj_array_strm [PU_NUM/2][2],
    hls::stream <bool> i_ln_e_strm [PU_NUM/2][2],
    ap_uint <32> line_cnt [PU_NUM/2][2],
    hls::stream <Object>& o_obj_strm
    )

top function of merging obj-stream from PUs, support 2/4/8-to-1 only

parseCSVCore

#include "xf_data_analytics/dataframe/csv_parser_v2.hpp"
template <int PU_NUM>
void parseCSVCore (
    ap_uint <128>* csv_buff,
    ap_uint <8> type_buff [PU_NUM][256],
    ap_uint <4> type_valid_buff [PU_NUM][256],
    const ap_uint <9> num_of_column,
    ap_uint <32> line_cnt [PU_NUM/2][2],
    hls::stream <Object>& o_obj_strm
    )

main function of CSV parser

readSchema

#include "xf_data_analytics/dataframe/json_parser.hpp"
template <
    int PU_NUM,
    int COL_NUM
    >
void readSchema (
    ap_uint <8>* schema,
    ap_uint <9>& num_of_column,
    ap_uint <COL_NUM>& mask_cfg,
    ap_uint <8> key_buff [PU_NUM][COL_NUM][256],
    ap_uint <4> type_buff [PU_NUM][COL_NUM]
    )

read the schema and cache the type of each enable column

parseJSONCore

#include "xf_data_analytics/dataframe/json_parser.hpp"
template <
    int PU_NUM,
    int COL_NUM
    >
void parseJSONCore (
    ap_uint <128>* json_buff,
    ap_uint <9> num_of_column,
    ap_uint <COL_NUM> mask_cfg,
    ap_uint <8> key_buff [PU_NUM][COL_NUM][256],
    ap_uint <4> type_buff [PU_NUM][COL_NUM],
    hls::stream <Object>& o_obj_strm
    )

main function of JSON parser

genAddr64

#include "xf_data_analytics/dataframe/read_from_dataframe.hpp"
template <int W>
void genAddr64 (
    int field_nm,
    ap_uint <4> t_buff [1<< W],
    FieldInfo f_buff [32],
    Node linkTable [1024],
    hls::stream <int>& o_field_id_strm,
    hls::stream <int>& o_addr_strm,
    hls::stream <int>& o_nm_strm,
    hls::stream <ap_uint <4>>& o_type_strm,
    hls::stream <bool>& o_e_strm
    )

generate the reading addr of each field, except Null and Boolean

breadWrapper

#include "xf_data_analytics/dataframe/read_from_dataframe.hpp"
template <int W>
void breadWrapper (
    int field_nm,
    ap_uint <12> str_high_addr_mem [1<< W][256],
    hls::stream <int>& i_field_id_strm,
    hls::stream <int>& i_addr_strm,
    hls::stream <int>& i_nm_strm,
    hls::stream <ap_uint <4>>& i_type_strm,
    hls::stream <bool>& i_e_strm,
    ap_uint <64>* ddr_buff,
    hls::stream <ap_uint <64>> o_dat_strm [1<< W],
    hls::stream <ap_uint <4>> o_valid_strm [1<< W],
    hls::stream <bool> o_e_strm [1<< W]
    )

burst read data field by field

writeObjOut

#include "xf_data_analytics/dataframe/read_from_dataframe.hpp"
template <int W>
void writeObjOut (
    int field_nm,
    ap_uint <4> t_buff [1<< W],
    hls::stream <bool> i_null_strm [1<< W],
    hls::stream <bool> i_bool_strm [1<< W],
    hls::stream <ap_uint <64>> i_dat_strm [1<< W],
    hls::stream <ap_uint <4>> i_valid_strm [1<< W],
    hls::stream <bool> i_e_strm [1<< W],
    hls::stream <Object>& obj_strm
    )

write out all fields of each line

memToLocalRam

#include "xf_data_analytics/dataframe/read_from_dataframe.hpp"
template <int W>
void memToLocalRam (
    int field_nm,
    ap_uint <W> t_buff [1<< W],
    ap_uint <32>* s_buff,
    ap_uint <32>* n_buff,
    ap_uint <32>* l_buff,
    ap_uint <64> bit_map [4096][1<< W],
    ap_uint <64> bool_buff [4096][1<< W],
    Node linkTable [1024],
    FieldInfo f_buff [2 *(1<< W)],
    ap_uint <12> str_high_addr_mem [1<< W][256],
    ap_uint <64>* ddr_in
    )

load the header info which saves length, null, bool info into on-chip mem

processNull

#include "xf_data_analytics/dataframe/write_to_dataframe.hpp"
template <int W>
void processNull (
    hls::stream <ap_uint <4>>& i_field_id_strm,
    hls::stream <ap_uint <4>>& i_valid_strm,
    hls::stream <bool>& i_e_strm,
    ap_uint <32>* n_buff,
    ap_uint <32>* l_buff,
    ap_uint <64> bit_map [4096][1<< W]
    )

process input data, record data count, null count, flag each data whether is null

Parameters:

i_field_id_strm input index for each value.
i_valid_strm input valid info for each value
i_e_strm end flag of i_strm.
n_buff buffer to record null count .
l_buff buffer to store length.
bit_map buffer to store value.

processBoolean

#include "xf_data_analytics/dataframe/write_to_dataframe.hpp"
template <int W>
void processBoolean (
    hls::stream <ap_uint <4>>& i_field_id_strm,
    hls::stream <bool>& i_dat_strm,
    hls::stream <bool>& i_e_strm,
    ap_uint <64> bool_buff [4096][1<< W]
    )

process bool data, save the data in bool_buff

Parameters:

i_field_id_strm input index for each value.
i_dat_strm input data
i_e_strm end flag of i_strm.
bool_buff buffer to store value.

collectData

collectData overload (1)

#include "xf_data_analytics/dataframe/write_to_dataframe.hpp"
template <int W>
void collectData (
    hls::stream <ap_uint <4>>& i_str_field_id_strm,
    hls::stream <ap_uint <32>>& i_str_offset_strm,
    hls::stream <bool>& i_str_e_strm,
    hls::stream <ap_uint <32>> o_offset_strm [1<< W],
    hls::stream <bool>& o_e_strm,
    hls::stream <ap_uint <W>>& o_field_id_strm
    )

string offset read and issue to different field_id channels

Parameters:

i_str_field_id_strm input field with strign data type
i_str_offset_strm input string offset
i_e_strm end flag of i_str_*_strm.
o_offset_strm output offset strm for different field
o_field_id_strm the output field index
o_e_strm end flag of o_offset_strm.

collectData overload (2)

#include "xf_data_analytics/dataframe/write_to_dataframe.hpp"
template <int W>
void collectData (
    hls::stream <ap_uint <4>>& i_w64_field_id_strm,
    hls::stream <ap_uint <64>>& i_w64_dat_strm,
    hls::stream <ap_uint <4>>& i_w64_dt_strm,
    hls::stream <bool>& i_w64_e_strm,
    hls::stream <ap_uint <32>> o_strm [1<< W],
    hls::stream <bool>& o_e_strm,
    hls::stream <ap_uint <W>>& o_field_id_strm
    )

process 64-bits data types, double, int64, date, float32, string data, issue to different field_id channels

Parameters:

i_str_field_id_strm input field with strign data type
i_str_offset_strm input string offset
i_e_strm end flag of i_str_*_strm.
o_offset_strm output offset strm for different field
o_field_id_strm the output field index
o_e_strm end flag of o_offset_strm.

readObjStrm

#include "xf_data_analytics/dataframe/write_to_dataframe.hpp"
template <int W>
void readObjStrm (
    hls::stream <Object>& obj_strm,
    hls::stream <ap_uint <4>>& o_null_field_id_strm,
    hls::stream <ap_uint <4>>& o_null_valid_strm,
    hls::stream <bool>& o_null_e_strm,
    hls::stream <ap_uint <4>>& o_bool_field_id_strm,
    hls::stream <bool>& o_bool_dat_strm,
    hls::stream <bool>& o_bool_e_strm,
    hls::stream <ap_uint <4>>& o_w64_field_id_strm,
    hls::stream <ap_uint <64>>& o_w64_dat_strm,
    hls::stream <ap_uint <4>>& o_w64_dt_strm,
    hls::stream <bool>& o_w64_e_strm,
    hls::stream <ap_uint <32>>& o_strlen_strm,
    hls::stream <ap_uint <4>>& o_str_field_id_strm,
    hls::stream <bool>& o_str_e_strm,
    ap_uint <32>* buff
    )

read data from object stream, analyze and send to different channels according to the data type

Parameters:

obj_strm the input stream data that packed as Object Struct
o_null_field_id_strm null strm index of each data
o_null_valid_strm null flag of each data
o_null_e_strm end flag of each null data
o_bool_field_id_strm bool strm index of each data
o_bool_dat_strm bool value of each data
o_bool_e_strm end flag of each boolean data
o_w64_field_id_strm output index of each w64 data
o_w64_dat_strm output data value of types: Int64, Float32, Double, Date, String data
o_w64_dt_strm output data type
o_w64_e_strm end flag of output w64 data
o_strlen_strm output string length of string fields in each json line
o_str_field_id_strm output index of each string field
o_str_e_strm end flag of output string length

memManage

#include "xf_data_analytics/dataframe/write_to_dataframe.hpp"
template <int W>
void memManage (
    hls::stream <ap_uint <W>> i_field_id_strm [2],
    hls::stream <bool> e_strm [2],
    hls::stream <ap_uint <15>>& o_low_addr_strm,
    hls::stream <ap_uint <12>>& o_high_addr_strm,
    hls::stream <ap_uint <8>> o_rnm_strm [2],
    hls::stream <ap_uint <W>> o_field_id_strm [2],
    hls::stream <bool> o_e_strm_a [2],
    hls::stream <ap_uint <2>>& o_ch_strm,
    hls::stream <bool>& o_e_strm,
    Node linkTable [1024],
    FieldInfo f_buff [2 *(1<< W)]
    )

genearte the addr of data write requests, every 32 x 32-bit requests output 1 addr of burst write req

localRamToMem

#include "xf_data_analytics/dataframe/write_to_dataframe.hpp"
template <int W>
void localRamToMem (
    ap_uint <32>* s_buff,
    ap_uint <32>* n_buff,
    ap_uint <32>* l_buff,
    ap_uint <64> bit_map [4096][1<< W],
    ap_uint <64> bool_buff [4096][1<< W],
    Node linkTable [1024],
    FieldInfo f_buff [2 *(1<< W)],
    ap_uint <64>* ddr_out
    )

save the on-chip mem data to DDR

Parameters:

s_buff size buffer
n_buff null count buffer
l_buff len count buffer
bit_map detailed null data for each input value
bool_buff detailed bool data for each input bool value
lineTable detailed link info of each mem block node
f_buff the link info between nodes
ddr_out DDR buffer to save local mem data

memManage

#include "xf_data_analytics/dataframe/write_to_parquet.hpp"
template <int W>
void memManage (
    hls::stream <ap_uint <W>>& i_field_id_strm,
    hls::stream <bool>& i_chunk_end,
    hls::stream <bool>& e_strm,
    hls::stream <ap_uint <H_BIT>>& o_low_addr_strm,
    hls::stream <ap_uint <12>>& o_high_addr_strm,
    hls::stream <ap_uint <8>>& o_rnm_strm,
    hls::stream <ap_uint <W>>& o_field_id_strm,
    hls::stream <bool>& o_e_strm,
    hls::stream <bool>& wr_e_strm,
    hls::stream <ap_uint <12>>& o_pre_nid_strm,
    hls::stream <ap_uint <12>>& o_cur_nid_strm,
    hls::stream <ap_uint <12>>& o_naddr_strm,
    hls::stream <bool>& o_node_e_strm,
    FieldInfo f_buff [(1<< W)]
    )

genearte the addr of data write requests, every 32 x 32-bit requests output 1 addr of burst write req