File runner.hpp¶

namespace xir¶

namespace vart¶

template<typename InputType, typename OutputType = InputType> class BaseRunner

Subclassed by Runner

Public Functions

virtual ~BaseRunner() = default¶

virtual std::pair<std::uint32_t, int> execute_async(InputType input, OutputType output) = 0

execute_async

Parameters:

input – inputs with a customized type
output – outputs with a customized type

Returns:

pair<jobid, status> status 0 for exit successfully, others for customized warnings or errors

virtual int wait(int jobid, int timeout = -1) = 0

wait

modes: 1. Blocking wait for specific ID. 2. Non-blocking wait for specific ID. 3. Blocking wait for any ID. 4. Non-blocking wait for any ID

Parameters:

jobid – job id, neg for any id, others for specific job id
timeout – timeout, neg for block for ever, 0 for non-block, pos for block with a limitation(ms).

Returns:

status 0 for exit successfully, others for customized warnings or errors

struct DpuMeta : public Meta ¶

Public Members

std::string filename¶

std::vector<std::string> kernels¶

std::string config_file¶

std::string target¶

std::string lib¶

std::string dirname¶

struct Meta¶

Subclassed by DpuMeta

Public Members

std::string target¶

std::string lib¶

std::string dirname¶

class Runner : public vart::BaseRunner<const std::vector<TensorBuffer*>&>

#include <runner.hpp>

Class of the Runner, provides API to use the runner.

The runner instance has a number of member functions to control the execution and get the input and output tensors of the runner.

Sample code:

// This example assumes that you have a DPU subgraph called dpu_subgraph.
// The way to create a DPU runner to run dpu_subgraph is shown below.

// create runner
auto runner = vart::Runner::create_runner(dpu_subgraph, ”run”);
// get input tensors
auto input_tensors = runner->get_input_tensors();
// get input tensor buffers
auto input_tensor_buffers = std::vector<vart::TensorBuffer*>();
    for (auto input : input_tensors) {
        auto t = vart::alloc_cpu_flat_tensor_buffer(input);
        input_tensor_buffers.emplace_back(t.get());
}
// get output tensors
auto output_tensors = runner->get_output_tensors();
// get output tensor buffers
auto output_tensor_buffers = std::vector< vart::TensorBuffer*>();
for (auto output : output _tensors) {
    auto t = vart::alloc_cpu_flat_tensor_buffer(output);
            output_tensor_buffers.emplace_back(t.get());
}
// sync input tensor buffers
for (auto& input : input_tensor_buffers) {
    input->sync_for_write(0, input->get_tensor()->get_data_size() /
            input->get_tensor()->get_shape()[0]);
}
// run runner
auto v = runner->execute_async(input_tensor_buffers, output_tensor_buffers);
auto status = runner->wait((int)v.first, 1000000000);
// sync output tensor buffers
for (auto& output : output_tensor_buffers) {
    output->sync_for_read(0, output->get_tensor()->get_data_size() /
    output->get_tensor()->get_shape()[0]);
}

Subclassed by RunnerExt

Public Types

enum class TensorFormat¶

Values:

enumerator NHWC¶

enumerator NCHW¶

Public Functions

virtual ~Runner() = default¶

virtual std::pair<uint32_t, int> execute_async(const std::vector<TensorBuffer*> &input, const std::vector<TensorBuffer*> &output) = 0

Executes the runner.

This is a blocking function.

Parameters:

input – A vector of TensorBuffer create by all input tensors of runner.
output – A vector of TensorBuffer create by all output tensors of runner.

Returns:

pair<jobid, status> status 0 for exit successfully, others for customized warnings or errors

virtual int wait(int jobid, int timeout) = 0

Waits for the end of DPU processing.

modes: 1. Blocking wait for specific ID. 2. Non-blocking wait for specific ID. 3. Blocking wait for any ID. 4. Non-blocking wait for any ID

Parameters:

jobid – job id, neg for any id, others for specific job id
timeout – timeout, neg for block for ever, 0 for non-block, pos for block with a limitation(ms).

Returns:

status 0 for exit successfully, others for customized warnings or errors

virtual TensorFormat get_tensor_format()

Get the tensor format of runner.

Sample code:

auto format = runner->get_tensor_format();
switch (format) {
    case vart::Runner::TensorFormat::NCHW:
        // do something
        break;
    case vart::Runner::TensorFormat::NHWC:
        // do something
        break;
}

Returns:: TensorFormat : NHWC / HCHW

virtual std::vector<const xir::Tensor*> get_input_tensors() = 0

Get all input tensors of runner.

Sample code:

inputTensors = runner->get_input_tensors();
for (auto input : inputTensor) {
    input->get_name();
    input->get_shape();
    input->get_element_num();
}

Returns:: All input tensors. A vector of raw pointer to the input tensor.

virtual std::vector<const xir::Tensor*> get_output_tensors() = 0

Get all output tensors of runner.

Sample code:

outputTensors = runner->get_output_tensors();
  for (auto output : outputTensor) {
      output->get_name();
      output->get_shape();
      output->get_element_num();
}

Returns:: All output tensors. A vector of raw pointer to the output tensor.

virtual std::pair<std::uint32_t, int> execute_async(InputType input, OutputType output) = 0

execute_async

Parameters:

input – inputs with a customized type
output – outputs with a customized type

Returns:

pair<jobid, status> status 0 for exit successfully, others for customized warnings or errors

Public Static Functions

static std::unique_ptr<Runner> create_runner(const xir::Subgraph *subgraph, const std::string &mode = std::string(""))

Factory function to create an instance of DPU runner by subgraph.

Sample code:

// This API can be used like:
auto runner = vart::Runner::create_runner(subgraph, "run");

Parameters:

subgraph – XIR Subgraph
mode – 1 mode supported: ‘run’ - DPU runner.

Returns:

An instance of DPU runner.

static std::unique_ptr<Runner> create_runner_with_attrs(const xir::Subgraph *subgraph, xir::Attrs *attrs)

Factory function to create an instance of DPU runner by subgraph, and attrs.

Parameters:

subgraph – XIR Subgraph
attrs – XIR attrs object, this object is shared among all runners on the same graph.
attrs["mode"], 1 – mode supported: ‘run’ - DPU runner.

Returns:

An instance of DPU runner.

static std::vector<std::unique_ptr<Runner>> create_runner(const std::string &model_directory)