FFmpeg
dnn_io_proc.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "dnn_io_proc.h"
22 #include "libavutil/imgutils.h"
23 #include "libswscale/swscale.h"
24 #include "libavutil/avassert.h"
25 
26 DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
27 {
28  struct SwsContext *sws_ctx;
29  int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
30  if (output->dt != DNN_FLOAT) {
31  avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT");
32  return DNN_ERROR;
33  }
34 
35  switch (frame->format) {
36  case AV_PIX_FMT_RGB24:
37  case AV_PIX_FMT_BGR24:
38  sws_ctx = sws_getContext(frame->width * 3,
39  frame->height,
40  AV_PIX_FMT_GRAYF32,
41  frame->width * 3,
42  frame->height,
43  AV_PIX_FMT_GRAY8,
44  0, NULL, NULL, NULL);
45  if (!sws_ctx) {
46  av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
47  "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
48  av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width * 3, frame->height,
49  av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height);
50  return DNN_ERROR;
51  }
52  sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
53  (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0}, 0, frame->height,
54  (uint8_t * const*)frame->data, frame->linesize);
55  sws_freeContext(sws_ctx);
56  return DNN_SUCCESS;
57  case AV_PIX_FMT_GRAYF32:
58  av_image_copy_plane(frame->data[0], frame->linesize[0],
59  output->data, bytewidth,
60  bytewidth, frame->height);
61  return DNN_SUCCESS;
62  case AV_PIX_FMT_YUV420P:
63  case AV_PIX_FMT_YUV422P:
64  case AV_PIX_FMT_YUV444P:
65  case AV_PIX_FMT_YUV410P:
66  case AV_PIX_FMT_YUV411P:
67  case AV_PIX_FMT_GRAY8:
68  case AV_PIX_FMT_NV12:
69  sws_ctx = sws_getContext(frame->width,
70  frame->height,
71  AV_PIX_FMT_GRAYF32,
72  frame->width,
73  frame->height,
74  AV_PIX_FMT_GRAY8,
75  0, NULL, NULL, NULL);
76  if (!sws_ctx) {
77  av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
78  "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
79  av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width, frame->height,
80  av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height);
81  return DNN_ERROR;
82  }
83  sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
84  (const int[4]){frame->width * sizeof(float), 0, 0, 0}, 0, frame->height,
85  (uint8_t * const*)frame->data, frame->linesize);
86  sws_freeContext(sws_ctx);
87  return DNN_SUCCESS;
88  default:
89  avpriv_report_missing_feature(log_ctx, "%s", av_get_pix_fmt_name(frame->format));
90  return DNN_ERROR;
91  }
92 
93  return DNN_SUCCESS;
94 }
95 
96 static DNNReturnType proc_from_frame_to_dnn_frameprocessing(AVFrame *frame, DNNData *input, void *log_ctx)
97 {
98  struct SwsContext *sws_ctx;
99  int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
100  if (input->dt != DNN_FLOAT) {
101  avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT");
102  return DNN_ERROR;
103  }
104 
105  switch (frame->format) {
106  case AV_PIX_FMT_RGB24:
107  case AV_PIX_FMT_BGR24:
108  sws_ctx = sws_getContext(frame->width * 3,
109  frame->height,
110  AV_PIX_FMT_GRAY8,
111  frame->width * 3,
112  frame->height,
113  AV_PIX_FMT_GRAYF32,
114  0, NULL, NULL, NULL);
115  if (!sws_ctx) {
116  av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
117  "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
118  av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height,
119  av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width * 3, frame->height);
120  return DNN_ERROR;
121  }
122  sws_scale(sws_ctx, (const uint8_t **)frame->data,
123  frame->linesize, 0, frame->height,
124  (uint8_t * const*)(&input->data),
125  (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0});
126  sws_freeContext(sws_ctx);
127  break;
128  case AV_PIX_FMT_GRAYF32:
129  av_image_copy_plane(input->data, bytewidth,
130  frame->data[0], frame->linesize[0],
131  bytewidth, frame->height);
132  break;
133  case AV_PIX_FMT_YUV420P:
134  case AV_PIX_FMT_YUV422P:
135  case AV_PIX_FMT_YUV444P:
136  case AV_PIX_FMT_YUV410P:
137  case AV_PIX_FMT_YUV411P:
138  case AV_PIX_FMT_GRAY8:
139  case AV_PIX_FMT_NV12:
140  sws_ctx = sws_getContext(frame->width,
141  frame->height,
142  AV_PIX_FMT_GRAY8,
143  frame->width,
144  frame->height,
145  AV_PIX_FMT_GRAYF32,
146  0, NULL, NULL, NULL);
147  if (!sws_ctx) {
148  av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
149  "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
150  av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height,
151  av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width, frame->height);
152  return DNN_ERROR;
153  }
154  sws_scale(sws_ctx, (const uint8_t **)frame->data,
155  frame->linesize, 0, frame->height,
156  (uint8_t * const*)(&input->data),
157  (const int [4]){frame->width * sizeof(float), 0, 0, 0});
158  sws_freeContext(sws_ctx);
159  break;
160  default:
161  avpriv_report_missing_feature(log_ctx, "%s", av_get_pix_fmt_name(frame->format));
162  return DNN_ERROR;
163  }
164 
165  return DNN_SUCCESS;
166 }
167 
168 static enum AVPixelFormat get_pixel_format(DNNData *data)
169 {
170  if (data->dt == DNN_UINT8 && data->order == DCO_BGR) {
171  return AV_PIX_FMT_BGR24;
172  }
173 
174  av_assert0(!"not supported yet.\n");
175  return AV_PIX_FMT_BGR24;
176 }
177 
178 static DNNReturnType proc_from_frame_to_dnn_analytics(AVFrame *frame, DNNData *input, void *log_ctx)
179 {
180  struct SwsContext *sws_ctx;
181  int linesizes[4];
182  enum AVPixelFormat fmt = get_pixel_format(input);
183  sws_ctx = sws_getContext(frame->width, frame->height, frame->format,
184  input->width, input->height, fmt,
185  SWS_FAST_BILINEAR, NULL, NULL, NULL);
186  if (!sws_ctx) {
187  av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
188  "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
189  av_get_pix_fmt_name(frame->format), frame->width, frame->height,
190  av_get_pix_fmt_name(fmt), input->width, input->height);
191  return DNN_ERROR;
192  }
193 
194  if (av_image_fill_linesizes(linesizes, fmt, input->width) < 0) {
195  av_log(log_ctx, AV_LOG_ERROR, "unable to get linesizes with av_image_fill_linesizes");
196  sws_freeContext(sws_ctx);
197  return DNN_ERROR;
198  }
199 
200  sws_scale(sws_ctx, (const uint8_t *const *)frame->data, frame->linesize, 0, frame->height,
201  (uint8_t *const *)(&input->data), linesizes);
202 
203  sws_freeContext(sws_ctx);
204  return DNN_SUCCESS;
205 }
206 
207 DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, DNNFunctionType func_type, void *log_ctx)
208 {
209  switch (func_type)
210  {
211  case DFT_PROCESS_FRAME:
212  return proc_from_frame_to_dnn_frameprocessing(frame, input, log_ctx);
213  case DFT_ANALYTICS_DETECT:
214  return proc_from_frame_to_dnn_analytics(frame, input, log_ctx);
215  default:
216  avpriv_report_missing_feature(log_ctx, "model function type %d", func_type);
217  return DNN_ERROR;
218  }
219 }
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:51
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:318
data
const char data[16]
Definition: mxf.c:142
AV_PIX_FMT_BGR24
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:69
sws_scale
int attribute_align_arg sws_scale(struct SwsContext *c, const uint8_t *const srcSlice[], const int srcStride[], int srcSliceY, int srcSliceH, uint8_t *const dst[], const int dstStride[])
swscale wrapper, so we don't need to export the SwsContext.
Definition: swscale.c:745
dnn_io_proc.h
proc_from_frame_to_dnn_analytics
static DNNReturnType proc_from_frame_to_dnn_analytics(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:178
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:373
SWS_FAST_BILINEAR
#define SWS_FAST_BILINEAR
Definition: swscale.h:58
ff_proc_from_dnn_to_frame
DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:26
DNN_SUCCESS
@ DNN_SUCCESS
Definition: dnn_interface.h:33
get_pixel_format
static enum AVPixelFormat get_pixel_format(DNNData *data)
Definition: dnn_io_proc.c:168
avassert.h
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
av_image_fill_linesizes
int av_image_fill_linesizes(int linesizes[4], enum AVPixelFormat pix_fmt, int width)
Fill plane linesizes for an image with pixel format pix_fmt and width width.
Definition: imgutils.c:89
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DNNReturnType
DNNReturnType
Definition: dnn_interface.h:33
DNNData
Definition: dnn_interface.h:58
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
AV_PIX_FMT_GRAYF32
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:431
NULL
#define NULL
Definition: coverity.c:32
AV_PIX_FMT_GRAY8
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition: pixfmt.h:74
AV_PIX_FMT_RGB24
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
sws_getContext
struct SwsContext * sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat, int dstW, int dstH, enum AVPixelFormat dstFormat, int flags, SwsFilter *srcFilter, SwsFilter *dstFilter, const double *param)
Allocate and return an SwsContext.
Definition: utils.c:1917
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
ff_proc_from_frame_to_dnn
DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, DNNFunctionType func_type, void *log_ctx)
Definition: dnn_io_proc.c:207
av_image_get_linesize
int av_image_get_linesize(enum AVPixelFormat pix_fmt, int width, int plane)
Compute the size of an image line with format pix_fmt and width width for the plane plane.
Definition: imgutils.c:76
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:54
DNN_ERROR
@ DNN_ERROR
Definition: dnn_interface.h:33
uint8_t
uint8_t
Definition: audio_convert.c:194
AV_PIX_FMT_NV12
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
sws_freeContext
void sws_freeContext(struct SwsContext *swsContext)
Free the swscaler context swsContext.
Definition: utils.c:2337
AV_PIX_FMT_YUV444P
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:71
AV_PIX_FMT_YUV422P
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:70
AV_PIX_FMT_YUV411P
@ AV_PIX_FMT_YUV411P
planar YUV 4:1:1, 12bpp, (1 Cr & Cb sample per 4x1 Y samples)
Definition: pixfmt.h:73
imgutils.h
AV_PIX_FMT_YUV410P
@ AV_PIX_FMT_YUV410P
planar YUV 4:1:0, 9bpp, (1 Cr & Cb sample per 4x4 Y samples)
Definition: pixfmt.h:72
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:28
proc_from_frame_to_dnn_frameprocessing
static DNNReturnType proc_from_frame_to_dnn_frameprocessing(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:96
DCO_BGR
@ DCO_BGR
Definition: dnn_interface.h:41
SwsContext
Definition: swscale_internal.h:283
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:53
swscale.h
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
Generated on Wed Aug 24 2022 21:34:29 for FFmpeg by   doxygen 1.8.17