[FFmpeg-devel] [PATCH v2 1/8] avutil/hwcontext: Add hwdevice type for V4L2 Request API
Lynne
dev at lynne.ee
Tue Aug 6 15:46:16 EEST 2024
On 06/08/2024 11:06, Jonas Karlman wrote:
> Add a hwdevice type for V4L2 Request API with transfer_data_from support
> for AV_PIX_FMT_DRM_PRIME, based on AV_HWDEVICE_TYPE_DRM.
>
> AVV4L2RequestDeviceContext.media_fd can be set by the application or a
> media device path can be supplied when hwdevice is created. When none
> is supplied it default to -1 and hwaccel will auto-detect a media device
> with a capable video device.
>
> Signed-off-by: Jonas Karlman <jonas at kwiboo.se>
> ---
> configure | 7 +
> libavutil/Makefile | 3 +
> libavutil/hwcontext.c | 4 +
> libavutil/hwcontext.h | 1 +
> libavutil/hwcontext_internal.h | 1 +
> libavutil/hwcontext_v4l2request.c | 261 ++++++++++++++++++++++++++++++
> libavutil/hwcontext_v4l2request.h | 41 +++++
> 7 files changed, 318 insertions(+)
> create mode 100644 libavutil/hwcontext_v4l2request.c
> create mode 100644 libavutil/hwcontext_v4l2request.h
>
> diff --git a/configure b/configure
> index 37178d7d81..23d00edc48 100755
> --- a/configure
> +++ b/configure
> @@ -358,6 +358,7 @@ External library support:
> --enable-omx-rpi enable OpenMAX IL code for Raspberry Pi [no]
> --enable-rkmpp enable Rockchip Media Process Platform code [no]
> --disable-v4l2-m2m disable V4L2 mem2mem code [autodetect]
> + --enable-v4l2-request enable V4L2 Request API code [no]
> --disable-vaapi disable Video Acceleration API (mainly Unix/Intel) code [autodetect]
> --disable-vdpau disable Nvidia Video Decode and Presentation API for Unix code [autodetect]
> --disable-videotoolbox disable VideoToolbox code [autodetect]
> @@ -2023,6 +2024,7 @@ HWACCEL_LIBRARY_LIST="
> mmal
> omx
> opencl
> + v4l2_request
> "
>
> DOCUMENT_LIST="
> @@ -3148,6 +3150,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32"
> ffnvcodec_deps_any="libdl LoadLibrary"
> mediacodec_deps="android mediandk"
> nvdec_deps="ffnvcodec"
> +v4l2_request_deps="linux_media_h v4l2_timeval_to_ns"
> vaapi_x11_deps="xlib_x11"
> videotoolbox_hwaccel_deps="videotoolbox pthreads"
> videotoolbox_hwaccel_extralibs="-framework QuartzCore"
> @@ -7172,6 +7175,10 @@ if enabled v4l2_m2m; then
> check_cc vp9_v4l2_m2m linux/videodev2.h "int i = V4L2_PIX_FMT_VP9;"
> fi
>
> +if enabled v4l2_request; then
> + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns
> +fi
> +
> check_headers sys/videoio.h
> test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete
>
> diff --git a/libavutil/Makefile b/libavutil/Makefile
> index 6e6fa8d800..1ce46157dd 100644
> --- a/libavutil/Makefile
> +++ b/libavutil/Makefile
> @@ -48,6 +48,7 @@ HEADERS = adler32.h \
> hwcontext_qsv.h \
> hwcontext_mediacodec.h \
> hwcontext_opencl.h \
> + hwcontext_v4l2request.h \
> hwcontext_vaapi.h \
> hwcontext_videotoolbox.h \
> hwcontext_vdpau.h \
> @@ -201,6 +202,7 @@ OBJS-$(CONFIG_MACOS_KPERF) += macos_kperf.o
> OBJS-$(CONFIG_MEDIACODEC) += hwcontext_mediacodec.o
> OBJS-$(CONFIG_OPENCL) += hwcontext_opencl.o
> OBJS-$(CONFIG_QSV) += hwcontext_qsv.o
> +OBJS-$(CONFIG_V4L2_REQUEST) += hwcontext_v4l2request.o
> OBJS-$(CONFIG_VAAPI) += hwcontext_vaapi.o
> OBJS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.o
> OBJS-$(CONFIG_VDPAU) += hwcontext_vdpau.o
> @@ -222,6 +224,7 @@ SKIPHEADERS-$(CONFIG_D3D12VA) += hwcontext_d3d12va.h
> SKIPHEADERS-$(CONFIG_DXVA2) += hwcontext_dxva2.h
> SKIPHEADERS-$(CONFIG_QSV) += hwcontext_qsv.h
> SKIPHEADERS-$(CONFIG_OPENCL) += hwcontext_opencl.h
> +SKIPHEADERS-$(CONFIG_V4L2_REQUEST) += hwcontext_v4l2request.h
> SKIPHEADERS-$(CONFIG_VAAPI) += hwcontext_vaapi.h
> SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.h
> SKIPHEADERS-$(CONFIG_VDPAU) += hwcontext_vdpau.h
> diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c
> index fa99a0d8a4..7fae9381da 100644
> --- a/libavutil/hwcontext.c
> +++ b/libavutil/hwcontext.c
> @@ -65,6 +65,9 @@ static const HWContextType * const hw_table[] = {
> #endif
> #if CONFIG_VULKAN
> &ff_hwcontext_type_vulkan,
> +#endif
> +#if CONFIG_V4L2_REQUEST
> + &ff_hwcontext_type_v4l2request,
> #endif
> NULL,
> };
> @@ -77,6 +80,7 @@ static const char *const hw_type_names[] = {
> [AV_HWDEVICE_TYPE_D3D12VA] = "d3d12va",
> [AV_HWDEVICE_TYPE_OPENCL] = "opencl",
> [AV_HWDEVICE_TYPE_QSV] = "qsv",
> + [AV_HWDEVICE_TYPE_V4L2REQUEST] = "v4l2request",
> [AV_HWDEVICE_TYPE_VAAPI] = "vaapi",
> [AV_HWDEVICE_TYPE_VDPAU] = "vdpau",
> [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox",
> diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h
> index bac30debae..8cf50ddbd0 100644
> --- a/libavutil/hwcontext.h
> +++ b/libavutil/hwcontext.h
> @@ -38,6 +38,7 @@ enum AVHWDeviceType {
> AV_HWDEVICE_TYPE_MEDIACODEC,
> AV_HWDEVICE_TYPE_VULKAN,
> AV_HWDEVICE_TYPE_D3D12VA,
> + AV_HWDEVICE_TYPE_V4L2REQUEST,
> };
>
> /**
> diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h
> index e32b786238..fd0cf29c6e 100644
> --- a/libavutil/hwcontext_internal.h
> +++ b/libavutil/hwcontext_internal.h
> @@ -158,6 +158,7 @@ extern const HWContextType ff_hwcontext_type_drm;
> extern const HWContextType ff_hwcontext_type_dxva2;
> extern const HWContextType ff_hwcontext_type_opencl;
> extern const HWContextType ff_hwcontext_type_qsv;
> +extern const HWContextType ff_hwcontext_type_v4l2request;
> extern const HWContextType ff_hwcontext_type_vaapi;
> extern const HWContextType ff_hwcontext_type_vdpau;
> extern const HWContextType ff_hwcontext_type_videotoolbox;
> diff --git a/libavutil/hwcontext_v4l2request.c b/libavutil/hwcontext_v4l2request.c
> new file mode 100644
> index 0000000000..833fbf9f40
> --- /dev/null
> +++ b/libavutil/hwcontext_v4l2request.c
> @@ -0,0 +1,261 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "config.h"
> +
> +#include <fcntl.h>
> +#include <linux/dma-buf.h>
> +#include <linux/media.h>
> +#include <sys/ioctl.h>
> +#include <sys/mman.h>
> +#include <unistd.h>
> +
> +#include "avassert.h"
> +#include "hwcontext_drm.h"
> +#include "hwcontext_internal.h"
> +#include "hwcontext_v4l2request.h"
> +#include "mem.h"
> +
> +static void v4l2request_device_free(AVHWDeviceContext *hwdev)
> +{
> + AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
> +
> + if (hwctx->media_fd >= 0) {
> + close(hwctx->media_fd);
> + hwctx->media_fd = -1;
> + }
> +}
> +
> +static int v4l2request_device_create(AVHWDeviceContext *hwdev, const char *device,
> + AVDictionary *opts, int flags)
> +{
> + AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
> +
> + hwctx->media_fd = -1;
> + hwdev->free = v4l2request_device_free;
> +
> + // Use auto-detect
> + if (!device || !device[0])
> + return 0;
> +
> + hwctx->media_fd = open(device, O_RDWR);
> + if (hwctx->media_fd < 0)
> + return AVERROR(errno);
> +
> + return 0;
> +}
> +
> +static int v4l2request_device_init(AVHWDeviceContext *hwdev)
> +{
> + AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx;
> + struct media_device_info device_info;
> +
> + // Use auto-detect
> + if (hwctx->media_fd < 0)
> + return 0;
> +
> + if (ioctl(hwctx->media_fd, MEDIA_IOC_DEVICE_INFO, &device_info) < 0)
> + return AVERROR(errno);
> +
> + av_log(hwdev, AV_LOG_VERBOSE, "Using V4L2 media driver %s (%u.%u.%u)\n",
> + device_info.driver,
> + device_info.driver_version >> 16,
> + (device_info.driver_version >> 8) & 0xff,
> + device_info.driver_version & 0xff);
> +
> + return 0;
> +}
> +
> +static int v4l2request_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
> +{
> + frame->buf[0] = av_buffer_pool_get(hwfc->pool);
> + if (!frame->buf[0])
> + return AVERROR(ENOMEM);
> +
> + frame->data[0] = (uint8_t *)frame->buf[0]->data;
> +
> + frame->format = AV_PIX_FMT_DRM_PRIME;
> + frame->width = hwfc->width;
> + frame->height = hwfc->height;
> +
> + return 0;
> +}
> +
> +typedef struct DRMMapping {
> + // Address and length of each mmap()ed region.
> + int nb_regions;
> + int object[AV_DRM_MAX_PLANES];
> + void *address[AV_DRM_MAX_PLANES];
> + size_t length[AV_DRM_MAX_PLANES];
> +} DRMMapping;
> +
> +static void v4l2request_unmap_frame(AVHWFramesContext *hwfc,
> + HWMapDescriptor *hwmap)
> +{
> + DRMMapping *map = hwmap->priv;
> +
> + for (int i = 0; i < map->nb_regions; i++) {
> + struct dma_buf_sync sync = {
> + .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ,
> + };
> + ioctl(map->object[i], DMA_BUF_IOCTL_SYNC, &sync);
> + munmap(map->address[i], map->length[i]);
> + }
> +
> + av_free(map);
> +}
> +
> +static int v4l2request_map_frame(AVHWFramesContext *hwfc,
> + AVFrame *dst, const AVFrame *src)
> +{
> + const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
> + struct dma_buf_sync sync = {
> + .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ,
> + };
> + DRMMapping *map;
> + int ret, i, p, plane;
> + void *addr;
> +
> + map = av_mallocz(sizeof(*map));
> + if (!map)
> + return AVERROR(ENOMEM);
> +
> + av_assert0(desc->nb_objects <= AV_DRM_MAX_PLANES);
> + for (i = 0; i < desc->nb_objects; i++) {
> + addr = mmap(NULL, desc->objects[i].size, AV_HWFRAME_MAP_READ, MAP_SHARED,
> + desc->objects[i].fd, 0);
> + if (addr == MAP_FAILED) {
> + av_log(hwfc, AV_LOG_ERROR, "Failed to map DRM object %d to memory: %s (%d)\n",
> + desc->objects[i].fd, strerror(errno), errno);
> + ret = AVERROR(errno);
> + goto fail;
> + }
> +
> + map->address[i] = addr;
> + map->length[i] = desc->objects[i].size;
> + map->object[i] = desc->objects[i].fd;
> +
> + /*
> + * We're not checking for errors here because the kernel may not
> + * support the ioctl, in which case its okay to carry on
> + */
> + ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_SYNC, &sync);
> + }
> + map->nb_regions = i;
> +
> + plane = 0;
> + for (i = 0; i < desc->nb_layers; i++) {
> + const AVDRMLayerDescriptor *layer = &desc->layers[i];
> + for (p = 0; p < layer->nb_planes; p++) {
> + dst->data[plane] =
> + (uint8_t *)map->address[layer->planes[p].object_index] +
> + layer->planes[p].offset;
> + dst->linesize[plane] = layer->planes[p].pitch;
> + ++plane;
> + }
> + }
> + av_assert0(plane <= AV_DRM_MAX_PLANES);
> +
> + dst->width = src->width;
> + dst->height = src->height;
> +
> + ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src,
> + v4l2request_unmap_frame, map);
> + if (ret < 0)
> + goto fail;
> +
> + return 0;
> +
> +fail:
> + for (i = 0; i < desc->nb_objects; i++) {
> + if (map->address[i])
> + munmap(map->address[i], map->length[i]);
> + }
> + av_free(map);
> + return ret;
> +}
> +
> +static int v4l2request_transfer_get_formats(AVHWFramesContext *hwfc,
> + enum AVHWFrameTransferDirection dir,
> + enum AVPixelFormat **formats)
> +{
> + enum AVPixelFormat *pix_fmts;
> +
> + if (dir == AV_HWFRAME_TRANSFER_DIRECTION_TO)
> + return AVERROR(ENOSYS);
> +
> + pix_fmts = av_malloc_array(2, sizeof(*pix_fmts));
> + if (!pix_fmts)
> + return AVERROR(ENOMEM);
> +
> + pix_fmts[0] = hwfc->sw_format;
> + pix_fmts[1] = AV_PIX_FMT_NONE;
> +
> + *formats = pix_fmts;
> + return 0;
> +}
> +
> +static int v4l2request_transfer_data_from(AVHWFramesContext *hwfc,
> + AVFrame *dst, const AVFrame *src)
> +{
> + AVFrame *map;
> + int ret;
> +
> + if (dst->width > hwfc->width || dst->height > hwfc->height)
> + return AVERROR(EINVAL);
> +
> + map = av_frame_alloc();
> + if (!map)
> + return AVERROR(ENOMEM);
> + map->format = dst->format;
> +
> + ret = v4l2request_map_frame(hwfc, map, src);
> + if (ret)
> + goto fail;
> +
> + map->width = dst->width;
> + map->height = dst->height;
> +
> + ret = av_frame_copy(dst, map);
> + if (ret)
> + goto fail;
> +
> + ret = 0;
> +fail:
> + av_frame_free(&map);
> + return ret;
> +}
> +
> +const HWContextType ff_hwcontext_type_v4l2request = {
> + .type = AV_HWDEVICE_TYPE_V4L2REQUEST,
> + .name = "V4L2 Request API",
> +
> + .device_hwctx_size = sizeof(AVV4L2RequestDeviceContext),
> + .device_create = v4l2request_device_create,
> + .device_init = v4l2request_device_init,
> +
> + .frames_get_buffer = v4l2request_get_buffer,
> +
> + .transfer_get_formats = v4l2request_transfer_get_formats,
> + .transfer_data_from = v4l2request_transfer_data_from,
> +
> + .pix_fmts = (const enum AVPixelFormat[]) {
> + AV_PIX_FMT_DRM_PRIME,
> + AV_PIX_FMT_NONE
> + },
> +};
> diff --git a/libavutil/hwcontext_v4l2request.h b/libavutil/hwcontext_v4l2request.h
> new file mode 100644
> index 0000000000..0fe42f97b4
> --- /dev/null
> +++ b/libavutil/hwcontext_v4l2request.h
> @@ -0,0 +1,41 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#ifndef AVUTIL_HWCONTEXT_V4L2REQUEST_H
> +#define AVUTIL_HWCONTEXT_V4L2REQUEST_H
> +
> +/**
> + * @file
> + * An API-specific header for AV_HWDEVICE_TYPE_V4L2REQUEST.
> + */
> +
> +/**
> + * V4L2 Request API device details.
> + *
> + * Allocated as AVHWDeviceContext.hwctx
> + */
> +typedef struct AVV4L2RequestDeviceContext {
> + /**
> + * File descriptor of media device.
> + *
> + * Defaults to -1 for auto-detect.
> + */
> + int media_fd;
> +} AVV4L2RequestDeviceContext;
> +
> +#endif /* AVUTIL_HWCONTEXT_V4L2REQUEST_H */
Any reason you're not using hwcontext_drm.h?
-------------- next part --------------
A non-text attachment was scrubbed...
Name: OpenPGP_0xA2FEA5F03F034464.asc
Type: application/pgp-keys
Size: 624 bytes
Desc: OpenPGP public key
URL: <https://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20240806/1d26526c/attachment.key>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: OpenPGP_signature.asc
Type: application/pgp-signature
Size: 236 bytes
Desc: OpenPGP digital signature
URL: <https://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20240806/1d26526c/attachment.sig>
More information about the ffmpeg-devel
mailing list