Go to the documentation of this file.
59 #define OFFSET(x) offsetof(DnnDetectContext, dnnctx.x)
60 #define OFFSET2(x) offsetof(DnnDetectContext, x)
61 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
64 #if (CONFIG_LIBTENSORFLOW == 1)
67 #if (CONFIG_LIBOPENVINO == 1)
74 {
"ssd",
"output shape [1, 1, N, 7]", 0,
AV_OPT_TYPE_CONST, { .i64 =
DDMT_SSD }, 0, 0,
FLAGS,
"model_type" },
75 {
"yolo",
"output shape [1, N*Cx*Cy*DetectionBox]", 0,
AV_OPT_TYPE_CONST, { .i64 =
DDMT_YOLOV1V2 }, 0, 0,
FLAGS,
"model_type" },
89 for (
int i = 0;
i < nb_classes;
i++) {
90 if (label_data[
i * cell_size] > max_prob) {
91 max_prob = label_data[
i * cell_size];
100 char *saveptr =
NULL, *token;
102 int nb_anchor = 0,
i = 0;
103 while(anchors_str[
i] !=
'\0') {
104 if(anchors_str[
i] ==
'&')
109 anchors_buf =
av_mallocz(nb_anchor *
sizeof(*anchors));
113 for (
int i = 0;
i < nb_anchor;
i++) {
114 token =
av_strtok(anchors_str,
"&", &saveptr);
115 anchors_buf[
i] = strtof(token,
NULL);
118 *anchors = anchors_buf;
125 float overlapping_width =
FFMIN(bbox1->
x + bbox1->
w, bbox2->
x + bbox2->
w) -
FFMAX(bbox1->
x, bbox2->
x);
126 float overlapping_height =
FFMIN(bbox1->
y + bbox1->
h, bbox2->
y + bbox2->
h) -
FFMAX(bbox1->
y, bbox2->
y);
127 float intersection_area =
128 (overlapping_width < 0 || overlapping_height < 0) ? 0 : overlapping_height * overlapping_width;
129 float union_area = bbox1->
w * bbox1->
h + bbox2->
w * bbox2->
h - intersection_area;
130 return intersection_area / union_area;
137 float conf_threshold =
ctx->confidence;
138 int detection_boxes, box_size, cell_w, cell_h, scale_w, scale_h;
139 int nb_classes =
ctx->nb_classes;
141 float *anchors =
ctx->anchors;
145 cell_w =
ctx->cell_w;
146 cell_h =
ctx->cell_h;
150 box_size = nb_classes + 5;
152 if (!cell_h || !cell_w) {
168 output[output_index].
height % (box_size * cell_w * cell_h)) {
172 detection_boxes =
output[output_index].channels *
173 output[output_index].height *
174 output[output_index].width / box_size / cell_w / cell_h;
181 for (
int box_id = 0; box_id < detection_boxes; box_id++) {
182 for (
int cx = 0; cx < cell_w; cx++)
183 for (
int cy = 0; cy < cell_h; cy++) {
184 float x, y,
w,
h, conf;
185 float *detection_boxes_data;
188 detection_boxes_data =
output_data + box_id * box_size * cell_w * cell_h;
189 conf = detection_boxes_data[cy * cell_w + cx + 4 * cell_w * cell_h];
190 if (conf < conf_threshold) {
194 x = detection_boxes_data[cy * cell_w + cx];
195 y = detection_boxes_data[cy * cell_w + cx + cell_w * cell_h];
196 w = detection_boxes_data[cy * cell_w + cx + 2 * cell_w * cell_h];
197 h = detection_boxes_data[cy * cell_w + cx + 3 * cell_w * cell_h];
199 detection_boxes_data + cy * cell_w + cx + 5 * cell_w * cell_h);
200 conf = conf * detection_boxes_data[cy * cell_w + cx + (label_id + 5) * cell_w * cell_h];
208 bbox->x = (cx + x) / cell_w *
frame->
width - bbox->w / 2;
209 bbox->y = (cy + y) / cell_h *
frame->
height - bbox->h / 2;
210 bbox->detect_confidence =
av_make_q((
int)(conf * 10000), 10000);
211 if (
ctx->labels && label_id < ctx->label_count) {
212 av_strlcpy(bbox->detect_label,
ctx->labels[label_id],
sizeof(bbox->detect_label));
214 snprintf(bbox->detect_label,
sizeof(bbox->detect_label),
"%d", label_id);
229 float conf_threshold =
ctx->confidence;
267 memcpy(bbox, candidate_bbox,
sizeof(*bbox));
290 float conf_threshold =
ctx->confidence;
291 int proposal_count =
output->height;
292 int detect_size =
output->width;
293 float *detections =
output->data;
303 for (
int i = 0;
i < proposal_count; ++
i) {
304 float conf = detections[
i * detect_size + 2];
305 if (conf < conf_threshold) {
311 if (nb_bboxes == 0) {
324 for (
int i = 0;
i < proposal_count; ++
i) {
325 int av_unused image_id = (
int)detections[
i * detect_size + 0];
326 int label_id = (
int)detections[
i * detect_size + 1];
327 float conf = detections[
i * detect_size + 2];
328 float x0 = detections[
i * detect_size + 3];
329 float y0 = detections[
i * detect_size + 4];
330 float x1 = detections[
i * detect_size + 5];
331 float y1 = detections[
i * detect_size + 6];
333 if (conf < conf_threshold) {
346 if (
ctx->labels && label_id < ctx->label_count) {
353 if (nb_bboxes == 0) {
373 switch (
ctx->model_type) {
392 float conf_threshold =
ctx->confidence;
393 float *conf, *position, *label_id, x0, y0, x1, y1;
401 position =
output[3].data;
402 label_id =
output[2].data;
410 for (
int i = 0;
i < proposal_count; ++
i) {
411 if (conf[
i] < conf_threshold)
416 if (nb_bboxes == 0) {
429 for (
int i = 0;
i < proposal_count; ++
i) {
430 y0 = position[
i * 4];
431 x0 = position[
i * 4 + 1];
432 y1 = position[
i * 4 + 2];
433 x1 = position[
i * 4 + 3];
437 if (conf[
i] < conf_threshold) {
449 if (
ctx->labels && label_id[
i] <
ctx->label_count) {
456 if (nb_bboxes == 0) {
480 for (
int i = 0;
i <
ctx->label_count;
i++) {
483 ctx->label_count = 0;
499 while (!feof(file)) {
502 if (!fgets(buf, 256, file)) {
506 line_len = strlen(buf);
508 int i = line_len - 1;
509 if (buf[
i] ==
'\n' || buf[
i] ==
'\r' || buf[
i] ==
' ') {
547 switch(backend_type) {
549 if (output_nb != 4) {
551 but get %d instead\n", output_nb);
556 if (output_nb != 1) {
558 but get %d instead\n", output_nb);
582 if (!
ctx->bboxes_fifo)
586 if (
ctx->labels_filename) {
589 if (
ctx->anchors_str) {
629 *out_pts = in_frame->
pts +
pts;
681 int64_t out_pts =
pts;
708 .
name =
"dnn_detect",
716 .priv_class = &dnn_detect_class,
static enum AVPixelFormat pix_fmts[]
AVPixelFormat
Pixel format.
static int dnn_detect_parse_yolo_output(AVFrame *frame, DNNData *output, int output_index, AVFilterContext *filter_ctx)
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
#define AVERROR_EOF
End of file.
#define FILTER_PIXFMTS_ARRAY(array)
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
int av_fifo_peek(const AVFifo *f, void *buf, size_t nb_elems, size_t offset)
Read data from a FIFO without modifying FIFO state.
This structure describes decoded (raw) audio or video data.
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
AVFILTER_DEFINE_CLASS(dnn_detect)
static int read_detect_label_file(AVFilterContext *context)
static av_cold int dnn_detect_init(AVFilterContext *context)
static int output_data(MLPDecodeContext *m, unsigned int substr, AVFrame *frame, int *got_frame_ptr)
Write the audio data into the output buffer.
#define AV_LOG_VERBOSE
Detailed information.
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
const char * name
Filter name.
A link between two filters.
#define FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink)
Forward the status on an output link to an input link.
static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
int ff_inlink_consume_frame(AVFilterLink *link, AVFrame **rframe)
Take a frame from the link's FIFO and update the link's stats.
#define AV_DETECTION_BBOX_LABEL_NAME_MAX_SIZE
char detect_label[AV_DETECTION_BBOX_LABEL_NAME_MAX_SIZE]
Detect result with confidence.
void * priv
private data for use by the filter
int av_fifo_write(AVFifo *f, const void *buf, size_t nb_elems)
Write data into a FIFO.
static float dnn_detect_IOU(AVDetectionBBox *bbox1, AVDetectionBBox *bbox2)
static FilteringContext * filter_ctx
static av_cold void dnn_detect_uninit(AVFilterContext *context)
DNNDetectionModelType model_type
static av_always_inline AVDetectionBBox * av_get_detection_bbox(const AVDetectionBBoxHeader *header, unsigned int idx)
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
int av_fifo_read(AVFifo *f, void *buf, size_t nb_elems)
Read data from a FIFO.
const AVFilterPad ff_video_default_filterpad[1]
An AVFilterPad array whose only entry has name "default" and is of type AVMEDIA_TYPE_VIDEO.
static void ff_outlink_set_status(AVFilterLink *link, int status, int64_t pts)
Set the status field of a link from the source filter.
int ff_dnn_set_detect_post_proc(DnnContext *ctx, DetectPostProc post_proc)
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
int(* init)(AVBSFContext *ctx)
static void free_detect_labels(DnnDetectContext *ctx)
DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame)
const AVFilter ff_vf_dnn_detect
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
int av_usleep(unsigned usec)
Sleep for a period of time.
#define AV_PIX_FMT_GRAYF32
#define FILTER_INPUTS(array)
static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your context
Describe the class of an AVClass context structure.
static int dnn_detect_activate(AVFilterContext *filter_ctx)
size_t av_fifo_can_read(const AVFifo *f)
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
int ff_dnn_flush(DnnContext *ctx)
int ff_inlink_acknowledge_status(AVFilterLink *link, int *rstatus, int64_t *rpts)
Test and acknowledge the change of status on the link.
AVDetectionBBoxHeader * av_detection_bbox_create_side_data(AVFrame *frame, uint32_t nb_bboxes)
Allocates memory for AVDetectionBBoxHeader, plus an array of.
#define DNN_COMMON_OPTIONS
DNNBackendType backend_type
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
static AVRational av_make_q(int num, int den)
Create an AVRational.
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
AVFilterContext * src
source filter
static const uint8_t header[24]
FF_FILTER_FORWARD_WANTED(outlink, inlink)
static const AVOption dnn_detect_options[]
static int dnn_detect_flush_frame(AVFilterLink *outlink, int64_t pts, int64_t *out_pts)
static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)
#define i(width, name, range_min, range_max)
static int check_output_nb(DnnDetectContext *ctx, DNNBackendType backend_type, int output_nb)
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
FILE * avpriv_fopen_utf8(const char *path, const char *mode)
Open a file using a UTF-8 filename.
static int dnn_detect_post_proc_yolo(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
static int av_cmp_q(AVRational a, AVRational b)
Compare two rationals.
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
AVFifo * av_fifo_alloc2(size_t nb_elems, size_t elem_size, unsigned int flags)
Allocate and initialize an AVFifo with a given element size.
AVRational detect_confidence
int av_dynarray_add_nofree(void *tab_ptr, int *nb_ptr, void *elem)
Add an element to a dynamic array.
int x
Distance in pixels from the left/top edge of the frame, together with width and height,...
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
char * av_strdup(const char *s)
Duplicate a string.
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
static int dnn_detect_fill_side_data(AVFrame *frame, AVFilterContext *filter_ctx)
Structure to hold side data for an AVFrame.
#define FILTER_OUTPUTS(array)
int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
static int dnn_detect_get_label_id(int nb_classes, int cell_size, float *label_data)
@ AV_PIX_FMT_YUV411P
planar YUV 4:1:1, 12bpp, (1 Cr & Cb sample per 4x1 Y samples)
@ AV_PIX_FMT_YUV410P
planar YUV 4:1:0, 9bpp, (1 Cr & Cb sample per 4x4 Y samples)
size_t av_strlcpy(char *dst, const char *src, size_t size)
Copy the string src to dst, but no more than size - 1 bytes, and null-terminate dst.
void av_fifo_freep2(AVFifo **f)
Free an AVFifo and reset pointer to NULL.
void ff_dnn_uninit(DnnContext *ctx)
static av_cold int uninit(AVCodecContext *avctx)
static int dnn_detect_post_proc_tf(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
int ff_dnn_execute_model(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame)
static int dnn_detect_parse_anchors(char *anchors_str, float **anchors)
#define AV_FIFO_FLAG_AUTO_GROW
Automatically resize the FIFO on writes, so that the data fits.
@ AV_FRAME_DATA_DETECTION_BBOXES
Bounding boxes for object detection and classification, as described by AVDetectionBBoxHeader.