FFmpeg
Loading...
Searching...
No Matches
dnn_io_proc.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2020
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21#include "dnn_io_proc.h"
22#include "libavutil/imgutils.h"
23#include "libavutil/mem.h"
24#include "libswscale/swscale.h"
25#include "libavutil/avassert.h"
27
29{
30 switch (dt)
31 {
32 case DNN_FLOAT:
33 return sizeof(float);
34 case DNN_UINT8:
35 return sizeof(uint8_t);
36 default:
37 av_assert0(!"not supported yet.");
38 return 1;
39 }
40}
41
42int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
43{
44 struct SwsContext *sws_ctx;
45 int ret = 0;
46 int linesize[4] = { 0 };
47 void **dst_data = NULL;
48 void *middle_data = NULL;
49 void *dst_data_buf[4] = { NULL, NULL, NULL, NULL };
50 uint8_t *planar_data[4] = { 0 };
51 int plane_size = frame->width * frame->height * sizeof(uint8_t);
52 enum AVPixelFormat src_fmt = AV_PIX_FMT_NONE;
53 int src_datatype_size = get_datatype_size(output->dt);
54 int channel_idx;
55
56 int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
57 if (bytewidth < 0) {
58 return AVERROR(EINVAL);
59 }
60 /* scale == 1 and mean == 0 and dt == UINT8: passthrough */
61 if (fabsf(output->scale - 1) < 1e-6f && fabsf(output->mean) < 1e-6 && output->dt == DNN_UINT8)
62 src_fmt = AV_PIX_FMT_GRAY8;
63 /* (scale == 255 or scale == 0) and mean == 0 and dt == FLOAT: normalization */
64 else if ((fabsf(output->scale - 255) < 1e-6f || fabsf(output->scale) < 1e-6f) &&
65 fabsf(output->mean) < 1e-6 && output->dt == DNN_FLOAT)
66 src_fmt = AV_PIX_FMT_GRAYF32;
67 else {
68 av_log(log_ctx, AV_LOG_ERROR, "dnn_process output data doesn't type: UINT8 "
69 "scale: %f, mean: %f\n", output->scale, output->mean);
70 return AVERROR(ENOSYS);
71 }
72
73 dst_data = (void **)frame->data;
74 linesize[0] = frame->linesize[0];
75 if (output->layout == DL_NCHW) {
76 middle_data = av_malloc(plane_size * output->dims[1]);
77 if (!middle_data) {
78 ret = AVERROR(ENOMEM);
79 goto err;
80 }
81 dst_data_buf[0] = middle_data;
82 dst_data = dst_data_buf;
83 linesize[0] = frame->width * 3;
84 }
85
86 switch (frame->format) {
89 channel_idx = dnn_get_channel_idx_by_layout(output->layout);
90 if (output->dims[channel_idx] != 3) {
91 av_log(log_ctx, AV_LOG_ERROR,
92 "dnn_processing to a %s frame requires a 3-channel model "
93 "output, got %d channels; channel-reducing/expanding "
94 "models (e.g. single-channel matte output) are not "
95 "supported by this filter\n",
96 av_get_pix_fmt_name(frame->format), output->dims[channel_idx]);
97 ret = AVERROR(ENOSYS);
98 goto err;
99 }
100 sws_ctx = sws_getContext(frame->width * 3,
101 frame->height,
102 src_fmt,
103 frame->width * 3,
104 frame->height,
106 0, NULL, NULL, NULL);
107 if (!sws_ctx) {
108 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
109 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
110 av_get_pix_fmt_name(src_fmt), frame->width * 3, frame->height,
111 av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height);
112 ret = AVERROR(EINVAL);
113 goto err;
114 }
115 sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
116 (const int[4]){frame->width * 3 * src_datatype_size, 0, 0, 0}, 0, frame->height,
117 (uint8_t * const*)dst_data, linesize);
118 sws_freeContext(sws_ctx);
119 // convert data from planar to packed
120 if (output->layout == DL_NCHW) {
121 sws_ctx = sws_getContext(frame->width,
122 frame->height,
124 frame->width,
125 frame->height,
126 frame->format,
127 0, NULL, NULL, NULL);
128 if (!sws_ctx) {
129 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
130 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
132 av_get_pix_fmt_name(frame->format),frame->width, frame->height);
133 ret = AVERROR(EINVAL);
134 goto err;
135 }
136 if (frame->format == AV_PIX_FMT_RGB24) {
137 planar_data[0] = (uint8_t *)middle_data + plane_size;
138 planar_data[1] = (uint8_t *)middle_data + plane_size * 2;
139 planar_data[2] = (uint8_t *)middle_data;
140 } else if (frame->format == AV_PIX_FMT_BGR24) {
141 planar_data[0] = (uint8_t *)middle_data + plane_size;
142 planar_data[1] = (uint8_t *)middle_data;
143 planar_data[2] = (uint8_t *)middle_data + plane_size * 2;
144 }
145 sws_scale(sws_ctx, (const uint8_t * const *)planar_data,
146 (const int [4]){frame->width * sizeof(uint8_t),
147 frame->width * sizeof(uint8_t),
148 frame->width * sizeof(uint8_t), 0},
149 0, frame->height, frame->data, frame->linesize);
150 sws_freeContext(sws_ctx);
151 }
152 break;
154 if (output->dt == DNN_FLOAT) {
155 av_image_copy_plane(frame->data[0], frame->linesize[0],
156 output->data, bytewidth,
157 bytewidth, frame->height);
158 } else {
159 av_assert0(output->dt == DNN_UINT8);
160 sws_ctx = sws_getContext(frame->width, frame->height,
162 frame->width, frame->height,
164 0, NULL, NULL, NULL);
165 if (!sws_ctx) {
166 ret = AVERROR(EINVAL);
167 goto err;
168 }
169 sws_scale(sws_ctx,
170 (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
171 (const int[4]){frame->width * src_datatype_size, 0, 0, 0},
172 0, frame->height,
173 (uint8_t * const*)frame->data, frame->linesize);
174 sws_freeContext(sws_ctx);
175 }
176 break;
182 case AV_PIX_FMT_GRAY8:
183 case AV_PIX_FMT_NV12:
184 sws_ctx = sws_getContext(frame->width,
185 frame->height,
186 src_fmt,
187 frame->width,
188 frame->height,
190 0, NULL, NULL, NULL);
191 if (!sws_ctx) {
192 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
193 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
194 av_get_pix_fmt_name(src_fmt), frame->width, frame->height,
196 ret = AVERROR(EINVAL);
197 goto err;
198 }
199 sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
200 (const int[4]){frame->width * src_datatype_size, 0, 0, 0}, 0, frame->height,
201 (uint8_t * const*)frame->data, frame->linesize);
202 sws_freeContext(sws_ctx);
203 break;
204 default:
206 ret = AVERROR(ENOSYS);
207 goto err;
208 }
209
210err:
211 av_free(middle_data);
212 return ret;
213}
214
215int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
216{
217 struct SwsContext *sws_ctx;
218 int ret = 0;
219 int linesize[4] = { 0 };
220 void **src_data = NULL;
221 void *middle_data = NULL;
222 void *src_data_buf[4] = { NULL, NULL, NULL, NULL };
223 uint8_t *planar_data[4] = { 0 };
224 int plane_size = frame->width * frame->height * sizeof(uint8_t);
225 enum AVPixelFormat dst_fmt = AV_PIX_FMT_NONE;
226 int dst_datatype_size = get_datatype_size(input->dt);
227 int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
228 if (bytewidth < 0) {
229 return AVERROR(EINVAL);
230 }
231 /* scale == 1 and mean == 0 and dt == UINT8: passthrough */
232 if (fabsf(input->scale - 1) < 1e-6f && fabsf(input->mean) < 1e-6 && input->dt == DNN_UINT8)
233 dst_fmt = AV_PIX_FMT_GRAY8;
234 /* (scale == 255 or scale == 0) and mean == 0 and dt == FLOAT: normalization */
235 else if ((fabsf(input->scale - 255) < 1e-6f || fabsf(input->scale) < 1e-6f) &&
236 fabsf(input->mean) < 1e-6 && input->dt == DNN_FLOAT)
237 dst_fmt = AV_PIX_FMT_GRAYF32;
238 else {
239 av_log(log_ctx, AV_LOG_ERROR, "dnn_process input data doesn't support type: UINT8 "
240 "scale: %f, mean: %f\n", input->scale, input->mean);
241 return AVERROR(ENOSYS);
242 }
243
244 src_data = (void **)frame->data;
245 linesize[0] = frame->linesize[0];
246 if (input->layout == DL_NCHW) {
247 middle_data = av_malloc(plane_size * input->dims[1]);
248 if (!middle_data) {
249 ret = AVERROR(ENOMEM);
250 goto err;
251 }
252 src_data_buf[0] = middle_data;
253 src_data = src_data_buf;
254 linesize[0] = frame->width * 3;
255 }
256
257 switch (frame->format) {
258 case AV_PIX_FMT_RGB24:
259 case AV_PIX_FMT_BGR24:
260 // convert data from planar to packed
261 if (input->layout == DL_NCHW) {
262 sws_ctx = sws_getContext(frame->width,
263 frame->height,
264 frame->format,
265 frame->width,
266 frame->height,
268 0, NULL, NULL, NULL);
269 if (!sws_ctx) {
270 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
271 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
272 av_get_pix_fmt_name(frame->format), frame->width, frame->height,
274 ret = AVERROR(EINVAL);
275 goto err;
276 }
277 if (frame->format == AV_PIX_FMT_RGB24) {
278 planar_data[0] = (uint8_t *)middle_data + plane_size;
279 planar_data[1] = (uint8_t *)middle_data + plane_size * 2;
280 planar_data[2] = (uint8_t *)middle_data;
281 } else if (frame->format == AV_PIX_FMT_BGR24) {
282 planar_data[0] = (uint8_t *)middle_data + plane_size;
283 planar_data[1] = (uint8_t *)middle_data;
284 planar_data[2] = (uint8_t *)middle_data + plane_size * 2;
285 }
286 sws_scale(sws_ctx, (const uint8_t * const *)frame->data,
287 frame->linesize, 0, frame->height, planar_data,
288 (const int [4]){frame->width * sizeof(uint8_t),
289 frame->width * sizeof(uint8_t),
290 frame->width * sizeof(uint8_t), 0});
291 sws_freeContext(sws_ctx);
292 }
293 sws_ctx = sws_getContext(frame->width * 3,
294 frame->height,
296 frame->width * 3,
297 frame->height,
298 dst_fmt,
299 0, NULL, NULL, NULL);
300 if (!sws_ctx) {
301 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
302 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
303 av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height,
304 av_get_pix_fmt_name(dst_fmt),frame->width * 3, frame->height);
305 ret = AVERROR(EINVAL);
306 goto err;
307 }
308 sws_scale(sws_ctx, (const uint8_t **)src_data,
309 linesize, 0, frame->height,
310 (uint8_t * const [4]){input->data, 0, 0, 0},
311 (const int [4]){frame->width * 3 * dst_datatype_size, 0, 0, 0});
312 sws_freeContext(sws_ctx);
313 break;
315 if (input->dt == DNN_FLOAT) {
316 av_image_copy_plane(input->data, bytewidth,
317 frame->data[0], frame->linesize[0],
318 bytewidth, frame->height);
319 } else {
320 sws_ctx = sws_getContext(frame->width, frame->height,
322 frame->width, frame->height,
324 0, NULL, NULL, NULL);
325 if (!sws_ctx) {
326 ret = AVERROR(EINVAL);
327 goto err;
328 }
329 sws_scale(sws_ctx,
330 (const uint8_t *[4]){frame->data[0], 0, 0, 0},
331 (const int[4]){frame->linesize[0], 0, 0, 0},
332 0, frame->height,
333 (uint8_t * const[4]){input->data, 0, 0, 0},
334 (const int[4]){frame->width * dst_datatype_size, 0, 0, 0});
335 sws_freeContext(sws_ctx);
336 }
337 break;
343 case AV_PIX_FMT_GRAY8:
344 case AV_PIX_FMT_NV12:
345 sws_ctx = sws_getContext(frame->width,
346 frame->height,
348 frame->width,
349 frame->height,
350 dst_fmt,
351 0, NULL, NULL, NULL);
352 if (!sws_ctx) {
353 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
354 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
356 av_get_pix_fmt_name(dst_fmt),frame->width, frame->height);
357 ret = AVERROR(EINVAL);
358 goto err;
359 }
360 sws_scale(sws_ctx, (const uint8_t **)frame->data,
361 frame->linesize, 0, frame->height,
362 (uint8_t * const [4]){input->data, 0, 0, 0},
363 (const int [4]){frame->width * dst_datatype_size, 0, 0, 0});
364 sws_freeContext(sws_ctx);
365 break;
366 default:
368 ret = AVERROR(ENOSYS);
369 goto err;
370 }
371err:
372 av_free(middle_data);
373 return ret;
374}
375
377{
378 if (data->dt == DNN_UINT8) {
379 switch (data->order) {
380 case DCO_BGR:
381 return AV_PIX_FMT_BGR24;
382 case DCO_RGB:
383 return AV_PIX_FMT_RGB24;
384 default:
385 av_assert0(!"unsupported data pixel format.\n");
386 return AV_PIX_FMT_BGR24;
387 }
388 }
389
390 av_assert0(!"unsupported data type.\n");
391 return AV_PIX_FMT_BGR24;
392}
393
394static void detect_write_tensor(DNNData *input, const uint8_t *src,
395 int src_linesize, int w, int h);
396
397int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index, void *log_ctx)
398{
400 int offsetx[4], offsety[4];
401 uint8_t *bbox_data[4];
402 struct SwsContext *sws_ctx;
403 int linesizes[4];
404 int ret = 0;
405 enum AVPixelFormat fmt;
406 int left, top, width, height;
407 int width_idx, height_idx;
409 const AVDetectionBBox *bbox;
411 int max_step[4] = { 0 };
412 av_assert0(sd);
413
414 /* (scale != 1 and scale != 0) or mean != 0 */
415 if ((fabsf(input->scale - 1) > 1e-6f && fabsf(input->scale) > 1e-6f) ||
416 fabsf(input->mean) > 1e-6f) {
417 av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support "
418 "scale: %f, mean: %f\n", input->scale, input->mean);
419 return AVERROR(ENOSYS);
420 }
421
422 width_idx = dnn_get_width_idx_by_layout(input->layout);
423 height_idx = dnn_get_height_idx_by_layout(input->layout);
424
425 header = (const AVDetectionBBoxHeader *)sd->data;
426 bbox = av_get_detection_bbox(header, bbox_index);
427
428 left = bbox->x;
429 width = bbox->w;
430 top = bbox->y;
431 height = bbox->h;
432
433 desc = av_pix_fmt_desc_get(frame->format);
434 offsetx[1] = offsetx[2] = AV_CEIL_RSHIFT(left, desc->log2_chroma_w);
435 offsetx[0] = offsetx[3] = left;
436
437 offsety[1] = offsety[2] = AV_CEIL_RSHIFT(top, desc->log2_chroma_h);
438 offsety[0] = offsety[3] = top;
439
441 for (int k = 0; frame->data[k]; k++)
442 bbox_data[k] = frame->data[k] + offsety[k] * frame->linesize[k] + offsetx[k] * max_step[k];
443
444 if (input->layout == DL_NCHW) {
445 /*
446 * For NCHW layout, scale the bbox region into a packed RGB temp buffer,
447 * then deinterleave into the NCHW tensor using detect_write_tensor().
448 */
449 int dst_w = input->dims[width_idx];
450 int dst_h = input->dims[height_idx];
451 int tmp_linesize = dst_w * 3;
452 uint8_t *tmp_buf = av_malloc(tmp_linesize * dst_h);
453 if (!tmp_buf)
454 return AVERROR(ENOMEM);
455
456 sws_ctx = sws_getContext(width, height, frame->format,
459 if (!sws_ctx) {
460 av_log(log_ctx, AV_LOG_ERROR, "Failed to create scale context for the conversion "
461 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
464 av_free(tmp_buf);
465 return AVERROR(EINVAL);
466 }
467
468 sws_scale(sws_ctx, (const uint8_t *const *)&bbox_data, frame->linesize,
469 0, height,
470 (uint8_t *const [4]){tmp_buf, 0, 0, 0},
471 (const int [4]){tmp_linesize, 0, 0, 0});
472 sws_freeContext(sws_ctx);
473
474 detect_write_tensor(input, tmp_buf, tmp_linesize, dst_w, dst_h);
475 av_free(tmp_buf);
476 } else {
477 fmt = get_pixel_format(input);
478 sws_ctx = sws_getContext(width, height, frame->format,
479 input->dims[width_idx],
480 input->dims[height_idx], fmt,
482 if (!sws_ctx) {
483 av_log(log_ctx, AV_LOG_ERROR, "Failed to create scale context for the conversion "
484 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
487 input->dims[width_idx],
488 input->dims[height_idx]);
489 return AVERROR(EINVAL);
490 }
491
492 ret = av_image_fill_linesizes(linesizes, fmt, input->dims[width_idx]);
493 if (ret < 0) {
494 av_log(log_ctx, AV_LOG_ERROR, "unable to get linesizes with av_image_fill_linesizes");
495 sws_freeContext(sws_ctx);
496 return ret;
497 }
498
499 sws_scale(sws_ctx, (const uint8_t *const *)&bbox_data, frame->linesize,
500 0, height,
501 (uint8_t *const [4]){input->data, 0, 0, 0}, linesizes);
502 sws_freeContext(sws_ctx);
503 }
504
505 return ret;
506}
507
508/*
509 * Write packed 24bit RGB/BGR samples into the tensor type/layout the model expects.
510 */
511static void detect_write_tensor(DNNData *input, const uint8_t *src,
512 int src_linesize, int w, int h)
513{
514 const size_t plane_size = (size_t)w * h;
515
516 if (input->dt == DNN_FLOAT) {
517 float *dst = input->data;
518 if (input->layout == DL_NCHW) {
519 /* FLOAT + NCHW: deinterleave into planes while widening uint8 -> float. */
520 for (int c = 0; c < 3; c++)
521 for (int y = 0; y < h; y++) {
522 const uint8_t *row = src + (ptrdiff_t)y * src_linesize;
523 for (int x = 0; x < w; x++)
524 dst[c * plane_size + (size_t)y * w + x] = row[x * 3 + c];
525 }
526 } else {
527 /* FLOAT + NHWC : keep it packed, only widen uint8 -> float byte by byte. */
528 for (int y = 0; y < h; y++) {
529 const uint8_t *row = src + (ptrdiff_t)y * src_linesize;
530 for (int x = 0; x < w * 3; x++)
531 dst[(size_t)y * w * 3 + x] = row[x];
532 }
533 }
534 } else {
535 /* UINT8 + NCHW: deinterleave into 3 uint8 planes. */
536 uint8_t *dst = input->data;
537 for (int c = 0; c < 3; c++)
538 for (int y = 0; y < h; y++) {
539 const uint8_t *row = src + (ptrdiff_t)y * src_linesize;
540 for (int x = 0; x < w; x++)
541 dst[c * plane_size + (size_t)y * w + x] = row[x * 3 + c];
542 }
543 }
544}
545
546int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx)
547{
548 struct SwsContext *sws_ctx = NULL;
549 uint8_t *tmp_buf = NULL;
550 const uint8_t *src;
551 int src_linesize;
552 int linesizes[4];
553 int ret = 0, width_idx, height_idx, channel_idx;
554 int packed_u8, w, h;
555 enum AVPixelFormat fmt;
556
557 switch (input->order) {
558 case DCO_BGR: fmt = AV_PIX_FMT_BGR24; break;
559 case DCO_RGB: fmt = AV_PIX_FMT_RGB24; break;
560 default: fmt = AV_PIX_FMT_NONE; break;
561 }
562
563 /* (scale != 1 and scale != 0) or mean != 0 */
564 if ((fabsf(input->scale - 1) > 1e-6f && fabsf(input->scale) > 1e-6f) ||
565 fabsf(input->mean) > 1e-6f) {
566 av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support "
567 "scale: %f, mean: %f\n", input->scale, input->mean);
568 return AVERROR(ENOSYS);
569 }
570
571 if (fmt == AV_PIX_FMT_NONE) {
572 av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support "
573 "color order: %d\n", input->order);
574 return AVERROR(ENOSYS);
575 }
576
577 if (input->dt != DNN_UINT8 && input->dt != DNN_FLOAT) {
578 av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support "
579 "data type: %d\n", input->dt);
580 return AVERROR(ENOSYS);
581 }
582
583 width_idx = dnn_get_width_idx_by_layout(input->layout);
584 height_idx = dnn_get_height_idx_by_layout(input->layout);
585 channel_idx = dnn_get_channel_idx_by_layout(input->layout);
586
587 w = input->dims[width_idx];
588 h = input->dims[height_idx];
589 if (w <= 0 || h <= 0) {
590 av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data has invalid "
591 "dimensions: %dx%d\n", w, h);
592 return AVERROR(EINVAL);
593 }
594
595 packed_u8 = (input->dt == DNN_UINT8) && (input->layout != DL_NCHW);
596 if (!packed_u8 && input->dims[channel_idx] != 3) {
597 av_log(log_ctx, AV_LOG_ERROR, "dnn_detect requires a 3-channel input, "
598 "but the model has %d channels\n",
599 input->dims[channel_idx]);
600 return AVERROR(ENOSYS);
601 }
602
603 ret = av_image_fill_linesizes(linesizes, fmt, w);
604 if (ret < 0) {
605 av_log(log_ctx, AV_LOG_ERROR, "unable to get linesizes with av_image_fill_linesizes");
606 return ret;
607 }
608
609 if (packed_u8) {
610 /* UINT8 + NHWC: sws_scale() writes straight into input->data in one pass. */
611 sws_ctx = sws_getContext(frame->width, frame->height, frame->format,
612 w, h, fmt, SWS_FAST_BILINEAR, NULL, NULL, NULL);
613 if (!sws_ctx) {
614 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
615 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
616 av_get_pix_fmt_name(frame->format), frame->width, frame->height,
617 av_get_pix_fmt_name(fmt), w, h);
618 return AVERROR(EINVAL);
619 }
620
621 sws_scale(sws_ctx, (const uint8_t *const *)frame->data, frame->linesize, 0, frame->height,
622 (uint8_t *const [4]){input->data, 0, 0, 0}, linesizes);
623 sws_freeContext(sws_ctx);
624 return 0;
625 }
626
627 if (frame->format == fmt && frame->width == w && frame->height == h) {
628 src = frame->data[0];
629 src_linesize = frame->linesize[0];
630 } else {
631 size_t tmp_size;
632
633 if (av_size_mult((size_t)linesizes[0], (size_t)h, &tmp_size) < 0) {
634 av_log(log_ctx, AV_LOG_ERROR, "dnn_detect temporary buffer size overflows\n");
635 return AVERROR(EINVAL);
636 }
637
638 tmp_buf = av_malloc(tmp_size);
639 if (!tmp_buf)
640 return AVERROR(ENOMEM);
641
642 sws_ctx = sws_getContext(frame->width, frame->height, frame->format,
643 w, h, fmt, SWS_FAST_BILINEAR, NULL, NULL, NULL);
644 if (!sws_ctx) {
645 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
646 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
647 av_get_pix_fmt_name(frame->format), frame->width, frame->height,
648 av_get_pix_fmt_name(fmt), w, h);
649 ret = AVERROR(EINVAL);
650 goto end;
651 }
652
653 sws_scale(sws_ctx, (const uint8_t *const *)frame->data, frame->linesize, 0, frame->height,
654 (uint8_t *const [4]){tmp_buf, 0, 0, 0}, linesizes);
655 sws_freeContext(sws_ctx);
656 sws_ctx = NULL;
657
658 src = tmp_buf;
659 src_linesize = linesizes[0];
660 }
661
662 detect_write_tensor(input, src, src_linesize, w, h);
663
664end:
665 av_freep(&tmp_buf);
666 return ret;
667}
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition dsp.h:97
simple assert() macros that are a bit more flexible than ISO C assert().
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition avassert.h:42
static int BS_FUNC left(const BSCTX *bc)
Return the number of the bits left in a buffer.
#define AV_CEIL_RSHIFT(a, b)
Definition common.h:60
#define NULL
Definition coverity.c:32
static __device__ float fabsf(float a)
static AVFrame * frame
static av_always_inline AVDetectionBBox * av_get_detection_bbox(const AVDetectionBBoxHeader *header, unsigned int idx)
static int dnn_get_height_idx_by_layout(DNNLayout layout)
@ DL_NCHW
static int dnn_get_width_idx_by_layout(DNNLayout layout)
DNNDataType
@ DNN_UINT8
@ DNN_FLOAT
@ DCO_RGB
@ DCO_BGR
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
static int get_datatype_size(DNNDataType dt)
Definition dnn_io_proc.c:28
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx)
static void detect_write_tensor(DNNData *input, const uint8_t *src, int src_linesize, int w, int h)
int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index, void *log_ctx)
static enum AVPixelFormat get_pixel_format(DNNData *data)
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition dnn_io_proc.c:42
DNN input&output process between AVFrame and DNNData.
int ret
Definition ffplay.c:391
#define AVERROR(e)
Definition error.h:45
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition frame.c:662
@ AV_FRAME_DATA_DETECTION_BBOXES
Bounding boxes for object detection and classification, as described by AVDetectionBBoxHeader.
Definition frame.h:194
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition log.h:210
int av_size_mult(size_t a, size_t b, size_t *r)
Multiply two size_t values checking for overflow.
Definition mem.c:671
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition imgutils.c:374
int av_image_get_linesize(enum AVPixelFormat pix_fmt, int width, int plane)
Compute the size of an image line with format pix_fmt and width width for the plane plane.
Definition imgutils.c:76
int av_image_fill_linesizes(int linesizes[4], enum AVPixelFormat pix_fmt, int width)
Fill plane linesizes for an image with pixel format pix_fmt and width width.
Definition imgutils.c:89
void av_image_fill_max_pixsteps(int max_pixsteps[4], int max_pixstep_comps[4], const AVPixFmtDescriptor *pixdesc)
Compute the max pixel step for each plane of an image with a format described by pixdesc.
Definition imgutils.c:35
int attribute_align_arg sws_scale(SwsContext *sws, const uint8_t *const srcSlice[], const int srcStride[], int srcSliceY, int srcSliceH, uint8_t *const dst[], const int dstStride[])
swscale wrapper, so we don't need to export the SwsContext.
Definition swscale.c:1695
SwsContext * sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat, int dstW, int dstH, enum AVPixelFormat dstFormat, int flags, SwsFilter *srcFilter, SwsFilter *dstFilter, const double *param)
Allocate and return an SwsContext.
Definition utils.c:2007
void sws_freeContext(SwsContext *swsContext)
Free the swscaler context swsContext.
Definition utils.c:2338
@ SWS_FAST_BILINEAR
Scaler selection options.
Definition swscale.h:197
misc image utilities
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
const char * desc
Definition libsvtav1.c:83
uint8_t w
Definition llvidencdsp.c:39
Memory handling functions.
const char data[16]
Definition mxf.c:155
#define av_malloc(s)
Definition ops_static.c:52
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition pixdesc.c:3404
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition pixdesc.c:3484
#define AV_PIX_FMT_GRAYF32
Definition pixfmt.h:591
AVPixelFormat
Pixel format.
Definition pixfmt.h:71
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition pixfmt.h:96
@ AV_PIX_FMT_NONE
Definition pixfmt.h:72
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition pixfmt.h:75
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition pixfmt.h:73
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition pixfmt.h:77
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition pixfmt.h:81
@ AV_PIX_FMT_YUV410P
planar YUV 4:1:0, 9bpp, (1 Cr & Cb sample per 4x4 Y samples)
Definition pixfmt.h:79
@ AV_PIX_FMT_YUV411P
planar YUV 4:1:1, 12bpp, (1 Cr & Cb sample per 4x1 Y samples)
Definition pixfmt.h:80
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition pixfmt.h:78
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition pixfmt.h:76
@ AV_PIX_FMT_GBRP
planar GBR 4:4:4 24bpp
Definition pixfmt.h:165
static const uint8_t header[24]
Definition sdr2.c:68
int x
Distance in pixels from the left/top edge of the frame, together with width and height,...
Structure to hold side data for an AVFrame.
Definition frame.h:334
uint8_t * data
Definition frame.h:336
This structure describes decoded (raw) audio or video data.
Definition frame.h:479
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition pixdesc.h:69
float scale
DNNDataType dt
int dims[4]
DNNColorOrder order
void * data
DNNLayout layout
float mean
Main external API structure.
Definition swscale.h:227
int dst_h
Width and height of the destination frame.
Definition swscale.h:273
int dst_w
Definition swscale.h:273
external API header
#define av_free(p)
#define av_freep(p)
#define av_log(a,...)
#define src
Definition vp8dsp.c:248
#define height
Definition dsp.h:99
#define width
Definition dsp.h:99
static double c[64]