Go to the documentation of this file.
35 #include "../internal.h"
41 #include <tensorflow/c/c_api.h>
83 #define OFFSET(x) offsetof(TFContext, x)
84 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
121 for (uint32_t
i = 0;
i < nb_output; ++
i) {
140 if (!infer_request) {
147 return infer_request;
176 if (TF_GetCode(request->
status) != TF_OK) {
197 TF_DeleteStatus(request->
status);
224 TF_Buffer *graph_buf;
225 unsigned char *graph_data =
NULL;
227 long size, bytes_read;
240 bytes_read =
avio_read(model_file_context, graph_data,
size);
242 if (bytes_read !=
size){
247 graph_buf = TF_NewBuffer();
248 graph_buf->data = graph_data;
249 graph_buf->length =
size;
273 return TF_AllocateTensor(dt, input_dims, 4,
274 input_dims[1] * input_dims[2] * input_dims[3] *
size);
285 tf_output.oper = TF_GraphOperationByName(tf_model->
graph, input_name);
286 if (!tf_output.oper) {
292 input->dt = TF_OperationOutputType(tf_output);
296 TF_GraphGetTensorShape(tf_model->
graph, tf_output, dims, 4,
status);
297 if (TF_GetCode(
status) != TF_OK){
308 input->channels = dims[3];
313 static int get_output_tf(
void *model,
const char *input_name,
int input_width,
int input_height,
314 const char *output_name,
int *output_width,
int *output_height)
323 .output_names = &output_name,
357 #define SPACE_CHARS " \t\r\n"
369 if (
c >=
'0' &&
c <=
'9')
371 else if (
c >=
'A' &&
c <=
'F')
390 TF_Buffer *graph_def;
391 TF_ImportGraphDefOptions *graph_opts;
392 TF_SessionOptions *sess_opts;
393 const TF_Operation *init_op;
394 uint8_t *sess_config =
NULL;
395 int sess_config_length = 0;
430 tf_model->
graph = TF_NewGraph();
431 tf_model->
status = TF_NewStatus();
432 graph_opts = TF_NewImportGraphDefOptions();
433 TF_GraphImportGraphDef(tf_model->
graph, graph_def, graph_opts, tf_model->
status);
434 TF_DeleteImportGraphDefOptions(graph_opts);
435 TF_DeleteBuffer(graph_def);
436 if (TF_GetCode(tf_model->
status) != TF_OK){
442 init_op = TF_GraphOperationByName(tf_model->
graph,
"init");
443 sess_opts = TF_NewSessionOptions();
446 TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->
status);
448 if (TF_GetCode(tf_model->
status) != TF_OK) {
449 TF_DeleteSessionOptions(sess_opts);
457 TF_DeleteSessionOptions(sess_opts);
458 if (TF_GetCode(tf_model->
status) != TF_OK)
471 if (TF_GetCode(tf_model->
status) != TF_OK)
482 #define NAME_BUFFER_SIZE 256
489 TF_OperationDescription *op_desc;
491 int64_t strides[] = {1, 1, 1, 1};
492 TF_Tensor *kernel_tensor =
NULL, *biases_tensor =
NULL;
502 op_desc = TF_NewOperation(tf_model->
graph,
"Const", name_buffer);
503 TF_SetAttrType(op_desc,
"dtype", TF_FLOAT);
509 kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len,
size *
sizeof(
float));
510 memcpy(TF_TensorData(kernel_tensor), params->
kernel,
size *
sizeof(
float));
511 TF_SetAttrTensor(op_desc,
"value", kernel_tensor, tf_model->
status);
512 if (TF_GetCode(tf_model->
status) != TF_OK){
515 op = TF_FinishOperation(op_desc, tf_model->
status);
516 if (TF_GetCode(tf_model->
status) != TF_OK){
521 op_desc = TF_NewOperation(tf_model->
graph,
"Transpose", name_buffer);
523 TF_AddInput(op_desc,
input);
524 input.oper = transpose_op;
525 TF_AddInput(op_desc,
input);
526 TF_SetAttrType(op_desc,
"T", TF_FLOAT);
527 TF_SetAttrType(op_desc,
"Tperm", TF_INT32);
528 op = TF_FinishOperation(op_desc, tf_model->
status);
529 if (TF_GetCode(tf_model->
status) != TF_OK){
534 op_desc = TF_NewOperation(tf_model->
graph,
"Conv2D", name_buffer);
535 input.oper = *cur_op;
536 TF_AddInput(op_desc,
input);
538 TF_AddInput(op_desc,
input);
539 TF_SetAttrType(op_desc,
"T", TF_FLOAT);
540 TF_SetAttrIntList(op_desc,
"strides", strides, 4);
541 TF_SetAttrString(op_desc,
"padding",
"VALID", 5);
542 *cur_op = TF_FinishOperation(op_desc, tf_model->
status);
543 if (TF_GetCode(tf_model->
status) != TF_OK){
548 op_desc = TF_NewOperation(tf_model->
graph,
"Const", name_buffer);
549 TF_SetAttrType(op_desc,
"dtype", TF_FLOAT);
552 biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->
output_num *
sizeof(
float));
553 memcpy(TF_TensorData(biases_tensor), params->
biases, params->
output_num *
sizeof(
float));
554 TF_SetAttrTensor(op_desc,
"value", biases_tensor, tf_model->
status);
555 if (TF_GetCode(tf_model->
status) != TF_OK){
558 op = TF_FinishOperation(op_desc, tf_model->
status);
559 if (TF_GetCode(tf_model->
status) != TF_OK){
564 op_desc = TF_NewOperation(tf_model->
graph,
"BiasAdd", name_buffer);
565 input.oper = *cur_op;
566 TF_AddInput(op_desc,
input);
568 TF_AddInput(op_desc,
input);
569 TF_SetAttrType(op_desc,
"T", TF_FLOAT);
570 *cur_op = TF_FinishOperation(op_desc, tf_model->
status);
571 if (TF_GetCode(tf_model->
status) != TF_OK){
578 op_desc = TF_NewOperation(tf_model->
graph,
"Relu", name_buffer);
581 op_desc = TF_NewOperation(tf_model->
graph,
"Tanh", name_buffer);
584 op_desc = TF_NewOperation(tf_model->
graph,
"Sigmoid", name_buffer);
590 input.oper = *cur_op;
591 TF_AddInput(op_desc,
input);
592 TF_SetAttrType(op_desc,
"T", TF_FLOAT);
593 *cur_op = TF_FinishOperation(op_desc, tf_model->
status);
594 if (TF_GetCode(tf_model->
status) != TF_OK){
600 TF_DeleteTensor(kernel_tensor);
601 TF_DeleteTensor(biases_tensor);
610 TF_OperationDescription *op_desc;
615 op_desc = TF_NewOperation(tf_model->
graph,
"DepthToSpace", name_buffer);
616 input.oper = *cur_op;
618 TF_AddInput(op_desc,
input);
619 TF_SetAttrType(op_desc,
"T", TF_FLOAT);
620 TF_SetAttrInt(op_desc,
"block_size", params->
block_size);
621 *cur_op = TF_FinishOperation(op_desc, tf_model->
status);
622 if (TF_GetCode(tf_model->
status) != TF_OK){
636 TF_OperationDescription *op_desc;
639 int64_t pads_shape[] = {4, 2};
644 op_desc = TF_NewOperation(tf_model->
graph,
"Const", name_buffer);
645 TF_SetAttrType(op_desc,
"dtype", TF_INT32);
646 tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 *
sizeof(
int32_t));
647 pads = (
int32_t *)TF_TensorData(tensor);
656 TF_SetAttrTensor(op_desc,
"value", tensor, tf_model->
status);
657 if (TF_GetCode(tf_model->
status) != TF_OK){
658 TF_DeleteTensor(tensor);
662 op = TF_FinishOperation(op_desc, tf_model->
status);
663 if (TF_GetCode(tf_model->
status) != TF_OK){
664 TF_DeleteTensor(tensor);
669 op_desc = TF_NewOperation(tf_model->
graph,
"MirrorPad",
"mirror_pad");
670 input.oper = *cur_op;
672 TF_AddInput(op_desc,
input);
674 TF_AddInput(op_desc,
input);
675 TF_SetAttrType(op_desc,
"T", TF_FLOAT);
676 TF_SetAttrType(op_desc,
"Tpaddings", TF_INT32);
677 TF_SetAttrString(op_desc,
"mode",
"SYMMETRIC", 9);
678 *cur_op = TF_FinishOperation(op_desc, tf_model->
status);
679 if (TF_GetCode(tf_model->
status) != TF_OK){
680 TF_DeleteTensor(tensor);
694 TF_OperationDescription *op_desc;
701 op_desc = TF_NewOperation(tf_model->
graph,
"Const", name_buffer);
702 TF_SetAttrType(op_desc,
"dtype", TF_FLOAT);
703 tensor = TF_AllocateTensor(TF_FLOAT,
NULL, 0, TF_DataTypeSize(TF_FLOAT));
704 y = (
float *)TF_TensorData(tensor);
706 TF_SetAttrTensor(op_desc,
"value", tensor, tf_model->
status);
707 if (TF_GetCode(tf_model->
status) != TF_OK){
708 TF_DeleteTensor(tensor);
712 op = TF_FinishOperation(op_desc, tf_model->
status);
713 if (TF_GetCode(tf_model->
status) != TF_OK){
714 TF_DeleteTensor(tensor);
720 op_desc = TF_NewOperation(tf_model->
graph,
"Maximum", name_buffer);
721 input.oper = *cur_op;
723 TF_AddInput(op_desc,
input);
725 TF_AddInput(op_desc,
input);
726 TF_SetAttrType(op_desc,
"T", TF_FLOAT);
727 *cur_op = TF_FinishOperation(op_desc, tf_model->
status);
728 if (TF_GetCode(tf_model->
status) != TF_OK){
729 TF_DeleteTensor(tensor);
741 TF_OperationDescription *op_desc;
743 TF_Operation *transpose_op;
744 TF_Tensor *tensor =
NULL;
747 int64_t transpose_perm_shape[] = {4};
748 int64_t input_shape[] = {1, -1, -1, -1};
759 native_model = model->
model;
760 tf_model->
graph = TF_NewGraph();
761 tf_model->
status = TF_NewStatus();
763 #define CLEANUP_ON_ERROR(tf_model) \
765 TF_DeleteTensor(tensor); \
766 TF_DeleteGraph(tf_model->graph); \
767 TF_DeleteStatus(tf_model->status); \
768 av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
769 return DNN_GENERIC_ERROR; \
772 op_desc = TF_NewOperation(tf_model->
graph,
"Placeholder",
"x");
773 TF_SetAttrType(op_desc,
"dtype", TF_FLOAT);
774 TF_SetAttrShape(op_desc,
"shape", input_shape, 4);
775 op = TF_FinishOperation(op_desc, tf_model->
status);
776 if (TF_GetCode(tf_model->
status) != TF_OK){
780 op_desc = TF_NewOperation(tf_model->
graph,
"Const",
"transpose_perm");
781 TF_SetAttrType(op_desc,
"dtype", TF_INT32);
782 tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 *
sizeof(
int32_t));
788 TF_SetAttrTensor(op_desc,
"value", tensor, tf_model->
status);
789 if (TF_GetCode(tf_model->
status) != TF_OK){
792 transpose_op = TF_FinishOperation(op_desc, tf_model->
status);
793 if (TF_GetCode(tf_model->
status) != TF_OK){
797 for (layer = 0; layer < native_model->
layers_num; ++layer){
822 if (layer_add_res != 0){
827 op_desc = TF_NewOperation(tf_model->
graph,
"Identity",
"y");
830 TF_AddInput(op_desc,
input);
831 TF_FinishOperation(op_desc, tf_model->
status);
832 if (TF_GetCode(tf_model->
status) != TF_OK){
857 model->
model = tf_model;
858 tf_model->
model = model;
860 ctx->class = &dnn_tensorflow_class;
875 if (
ctx->options.nireq <= 0) {
879 #if !HAVE_PTHREAD_CANCEL
880 if (
ctx->options.async) {
881 ctx->options.async = 0;
891 for (
int i = 0;
i <
ctx->options.nireq;
i++) {
903 item->
status = TF_NewStatus();
966 if (!infer_request->
tf_input->oper){
1102 task = lltask->
task;
1103 tf_model = task->
model;
1223 tf_model = (*model)->
model;
1244 if (tf_model->
graph){
1245 TF_DeleteGraph(tf_model->
graph);
1252 TF_DeleteStatus(tf_model->
status);
AVFILTER_DEFINE_CLASS(dnn_tensorflow)
DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)
#define AV_LOG_WARNING
Something somehow does not look correct.
Stores execution parameters for single call to the TensorFlow C API.
static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue)
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
void av_opt_set_defaults(void *s)
Set the values of all AVOption fields to their default values.
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Common Async Execution Mechanism for the DNN Backends.
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
static int add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
#define DNN_GENERIC_ERROR
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
This structure describes decoded (raw) audio or video data.
DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Double-ended queue with mutex locks ensuring data consistency while multithreading.
FramePrePostProc frame_pre_proc
static int load_tf_model(TFModel *tf_model, const char *model_filename)
SafeQueue * request_queue
void(* callback)(void *args)
Completion Callback for the backend.
int64_t avio_size(AVIOContext *s)
Get the filesize.
static int load_native_model(TFModel *tf_model, const char *model_filename)
static void destroy_request_item(TFRequestItem **arg)
Free the TFRequestItem completely.
AVFilterContext * filter_ctx
Queue * ff_queue_create(void)
Create a Queue instance.
int ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
static FilteringContext * filter_ctx
static int get_input_tf(void *model, DNNData *input, const char *input_name)
Linear double-ended data structure.
static int add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
#define DNN_BACKEND_COMMON_OPTIONS
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
static int fill_model_input_tf(TFModel *tf_model, TFRequestItem *request)
DNNAsyncExecModule exec_module
static TF_Buffer * read_graph(const char *model_filename)
static int op(uint8_t **dst, const uint8_t *dst_end, GetByteContext *gb, int pixel, int count, int *x, int width, int linesize)
Perform decode operation.
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
#define av_assert0(cond)
assert() equivalent, that is always enabled.
static const AVFilterPad outputs[]
DNNActivationFunc activation
static const AVOption dnn_tensorflow_options[]
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
void ff_dnn_free_model_native(DNNModel **model)
int ff_dnn_flush_tf(const DNNModel *model)
Describe the class of an AVClass context structure.
int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx)
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
FramePrePostProc frame_post_proc
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
static TFInferRequest * tf_create_inference_request(void)
Create a TensorFlow inference request.
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
static void infer_completion_callback(void *args)
static void tf_free_request(TFInferRequest *request)
Free the contents of TensorFlow inference request.
static void transpose_perm(int16_t *out, int16_t *in, int num_vect, const uint8_t line_len[2], int length_div)
Interpret the input data as in the following table:
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
const OptionDef options[]
DetectPostProc detect_post_proc
DNNFunctionType func_type
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
static int hex_to_data(uint8_t *data, const char *p)
static int tf_start_inference(void *args)
Start synchronous inference for the TensorFlow model.
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
DNNModel * ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
static int add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)
#define i(width, name, range_min, range_max)
TF_Tensor ** output_tensors
TFInferRequest * infer_request
#define av_malloc_array(a, b)
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
void * args
Argument for the execution functions.
static av_const int av_toupper(int c)
Locale-independent conversion of ASCII characters to uppercase.
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
const char ** output_names
void * av_calloc(size_t nmemb, size_t size)
int(* get_input)(void *model, DNNData *input, const char *input_name)
#define AV_INPUT_BUFFER_PADDING_SIZE
static TF_Tensor * allocate_input_tensor(const DNNData *input)
LastLevelTaskItem * lltask
union DnnLayerMaximumParams::@221 val
int avio_read(AVIOContext *s, unsigned char *buf, int size)
Read size bytes from AVIOContext into buf.
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
void ff_dnn_free_model_tf(DNNModel **model)
int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
#define AVIO_FLAG_READ
read-only
static void free_buffer(void *data, size_t length)
static int get_output_tf(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
int(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
#define CLEANUP_ON_ERROR(tf_model)
static int add_pad_layer(TFModel *tf_model, TF_Operation **cur_op, LayerPadParams *params, const int layer)
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)