VVAS Inference Metadata

The VVAS inference metadata (GstInferenceMeta) object holds information about the metadata produced by the machine learning (ML) inference acceleration software libraries. This metadata structure stores the inferred data generated at multiple levels of ML operation into a single structure in a hierarchical way. This data structure combines, and stores metadata generated by detection and classification models.

The GStreamer plug-ins can set and get inference metadata from the GstBuffer by using the gst_buffer_add_meta () API and gst_buffer_get_meta () API, respectively.

GstInferenceMeta

GstInferenceMeta is the root node of the inference metadata. GstInferencePrediction hosts the actual inference metadata.

/**
* Implements the placeholder for inference information.
*/
typedef struct _GstInferenceMeta GstInferenceMeta;
struct _GstInferenceMeta
{
GstMeta meta;
GstInferencePrediction *prediction;
gchar *stream_id;
};

GstInferencePrediction

This structure represents the inference data generated from the ML operation. BBox represents the bounding box around the detected object from detection model. The classification parameter of this structure stores a list of classification information associated with the detected object. This structure also stores the results of the next level of ML operation performed (typically in cascaded network based ML operation).

typedef struct _VvasColorMetadata {
  uint8_t red;
  uint8_t green;
  uint8_t blue;
  uint8_t alpha;
} VvasColorMetadata;

struct _BoundingBox
{
  gint x;
  gint y;
  guint width;
  guint height;
  VvasColorMetadata box_color;
};

typedef struct _BoundingBox BoundingBox;

/**
* GstInferencePrediction:
* @prediction_id: unique id for this specific prediction
* @enabled: flag indicating whether or not this prediction should be
* used for further inference
* @bbox: the BoundingBox for this specific prediction
* @classifications: a linked list of GstInfereferenceClassification
* associated to this prediction
* @predictions: a n-ary tree of child predictions within this
* specific prediction. It is recommended to access the tree
* directly, but to use this module's API to interact with the
* children.
* @sub_buffer: A buffer created from the main buffer.
* @bbox_scaled: bbox co-ordinates scaled to root node resolution or not
* @segmentation: contains output of segmentation model wrapped in GstBuffer
* @obj_track_label: This is currently unused, kept for future use
* Abstraction that represents a prediction
*/
struct _GstInferencePrediction
{
   /*<private>*/
   GstMiniObject base;
   GMutex mutex;
   /*<public>*/
   guint64 prediction_id;
   gboolean enabled;
   BoundingBox bbox;
   GList * classifications;
   GNode * predictions;
   GstBuffer *sub_buffer;
   gboolean bbox_scaled; /* bbox co-ordinates scaled to root node resolution or not */
   Segmentation segmentation;
   gchar *obj_track_label;
   /* for future extension */
   void * reserved_1;
   void * reserved_2;
   void * reserved_3;
   void * reserved_4;
   void * reserved_5;
};

GstInferenceClassification

This structure stores the results of the ML operation by the classification network.

/**
* GstInferenceClassification:
* @classification_id: a unique id associated to this classification
* @class_id: the numerical id associated to the assigned class
* @class_prob: the resulting probability of the assigned
* class. Typically, between 0 and 1
* @class_label: the label associated to this class or NULL if not
* available
* @num_classes: the amount of classes of the entire prediction
* @probabilities: the entire array of probabilities of the prediction
* @labels: the entire array of labels of the prediction or NULL if
* not available
*/
typedef struct _GstInferenceClassification GstInferenceClassification;
struct _GstInferenceClassification
{
   /*<private>*/
   GstMiniObject base;
   GMutex mutex;
   /*<public>*/
   guint64 classification_id;
   gint class_id;
   gdouble class_prob;
   gchar *class_label;
   gint num_classes;
   gdouble *probabilities;
   gchar **labels;
   VvasColorMetadata label_color;
};