[FFmpeg-devel] [PATCH] lavfi: add a Vulkan Chromatic Aberration filter
Rostislav Pehlivanov
atomnuker at gmail.com
Fri Mar 30 17:44:11 EEST 2018
This implements a simple chromatic aberration filter. The curve could
use a little bit of tweaking, right now its not as curvy as it should
be but nevertheless it models the imperfections of lenses.
Meant to be applied on top of my previous patches. RFC quality again.
Signed-off-by: Rostislav Pehlivanov <atomnuker at gmail.com>
---
You want something more interesting, you get something more interesting!
configure | 1 +
libavfilter/Makefile | 1 +
libavfilter/allfilters.c | 1 +
libavfilter/vf_avgblur_vulkan.c | 3 -
libavfilter/vf_chromaticaberration_vulkan.c | 334 ++++++++++++++++++++++++++++
libavfilter/vulkan.c | 7 +-
libavfilter/vulkan.h | 2 +-
7 files changed, 342 insertions(+), 7 deletions(-)
create mode 100644 libavfilter/vf_chromaticaberration_vulkan.c
diff --git a/configure b/configure
index 388e45fed1..577c8e0aba 100755
--- a/configure
+++ b/configure
@@ -3298,6 +3298,7 @@ azmq_filter_deps="libzmq"
blackframe_filter_deps="gpl"
boxblur_filter_deps="gpl"
bs2b_filter_deps="libbs2b"
+chromaticabberation_vulkan_filter_deps="vulkan libshaderc"
colormatrix_filter_deps="gpl"
convolution_opencl_filter_deps="opencl"
convolve_filter_deps="avcodec"
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index f0a47320c8..4798f002a8 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -151,6 +151,7 @@ OBJS-$(CONFIG_BLEND_FILTER) += vf_blend.o framesync.o
OBJS-$(CONFIG_BOXBLUR_FILTER) += vf_boxblur.o
OBJS-$(CONFIG_BWDIF_FILTER) += vf_bwdif.o
OBJS-$(CONFIG_CHROMAKEY_FILTER) += vf_chromakey.o
+OBJS-$(CONFIG_CHROMATICABERRATION_VULKAN_FILTER) += vf_chromaticaberration_vulkan.o vulkan.o
OBJS-$(CONFIG_CIESCOPE_FILTER) += vf_ciescope.o
OBJS-$(CONFIG_CODECVIEW_FILTER) += vf_codecview.o
OBJS-$(CONFIG_COLORBALANCE_FILTER) += vf_colorbalance.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 3cbaecd726..3baf32ae7a 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -160,6 +160,7 @@ static void register_all(void)
REGISTER_FILTER(BOXBLUR, boxblur, vf);
REGISTER_FILTER(BWDIF, bwdif, vf);
REGISTER_FILTER(CHROMAKEY, chromakey, vf);
+ REGISTER_FILTER(CHROMATICABERRATION_VULKAN, chromaticaberration_vulkan, vf);
REGISTER_FILTER(CIESCOPE, ciescope, vf);
REGISTER_FILTER(CODECVIEW, codecview, vf);
REGISTER_FILTER(COLORBALANCE, colorbalance, vf);
diff --git a/libavfilter/vf_avgblur_vulkan.c b/libavfilter/vf_avgblur_vulkan.c
index a2c0fddd98..cef855d891 100644
--- a/libavfilter/vf_avgblur_vulkan.c
+++ b/libavfilter/vf_avgblur_vulkan.c
@@ -79,9 +79,6 @@ static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
int err;
AvgBlurVulkanContext *s = ctx->priv;
- /* Create sampler */
- ff_vk_init_sampler(ctx, NULL);
-
{ /* Create the shader */
const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
diff --git a/libavfilter/vf_chromaticaberration_vulkan.c b/libavfilter/vf_chromaticaberration_vulkan.c
new file mode 100644
index 0000000000..e7497e7384
--- /dev/null
+++ b/libavfilter/vf_chromaticaberration_vulkan.c
@@ -0,0 +1,334 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/opt.h"
+#include "vulkan.h"
+#include "internal.h"
+
+typedef struct ChromaticAberrationVulkanContext {
+ VulkanFilterContext vkctx;
+
+ int initialized;
+ AVVkExecContext exec;
+ AVVkBuffer shader_buf;
+
+ /* Shader updators, must be in the main filter struct */
+ VkDescriptorImageInfo input_images[3];
+ VkDescriptorImageInfo output_images[3];
+
+ float dist_x;
+ float dist_y;
+} ChromaticAberrationVulkanContext;
+
+#define RET(x) \
+ do { \
+ if ((err = (x)) < 0) \
+ goto fail; \
+ } while (0)
+
+static const char distort_chroma_kernel[] = {
+ C(0, void distort_chroma(int idx, ivec2 size, ivec2 pos) )
+ C(0, { )
+ C(1, vec2 p = ((vec2(pos)/vec2(size)) - 0.5f)*2.0f; )
+ C(1, float d = sqrt(p.x*p.x + p.y*p.y); )
+ C(1, p *= d / (d*FILTER_DIST); )
+ C(1, vec4 res = texture(input_img[idx], (p/2.0f) + 0.5f); )
+ C(1, imageStore(output_img[idx], pos, res); )
+ C(0, } )
+};
+
+static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
+{
+ int err;
+ ChromaticAberrationVulkanContext *s = ctx->priv;
+
+ /* Create sampler */
+ ff_vk_init_sampler(ctx, NULL, 0);
+
+ { /* Create the shader */
+ float dist_x = (s->dist_x/100.0f) + 1.0f;
+ float dist_y = (s->dist_y/100.0f) + 1.0f;
+ const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
+
+ SPIRVShader *shd = ff_vk_init_shader(ctx, "chromaticaberration_compute",
+ VK_SHADER_STAGE_COMPUTE_BIT);
+ ff_vk_set_compute_shader_sizes(ctx, shd, (int [3]){ 16, 16, 1 });
+
+ VulkanDescriptorSetBinding desc_i[2] = {
+ {
+ .name = "input_img",
+ .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+ .dimensions = 2,
+ .elems = planes,
+ .stages = VK_SHADER_STAGE_COMPUTE_BIT,
+ .updater = s->input_images,
+ },
+ {
+ .name = "output_img",
+ .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
+ .mem_layout = "rgba8",
+ .mem_quali = "writeonly",
+ .dimensions = 2,
+ .elems = planes,
+ .stages = VK_SHADER_STAGE_COMPUTE_BIT,
+ .updater = s->output_images,
+ },
+ };
+
+ RET(ff_vk_add_descriptor_set(ctx, shd, desc_i, 2, 0)); /* set 0 */
+
+ GLSLF(0, #define FILTER_DIST vec2(%f, %f), dist_x, dist_y);
+ GLSLD( distort_chroma_kernel );
+ GLSLC(0, #define IS_WITHIN(v1, v2) ((v1.x < v2.x) && (v1.y < v2.y)) );
+ GLSLC(0, void main() );
+ GLSLC(0, { );
+ GLSLC(1, ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
+ GLSLF(1, int planes = %i;, planes);
+ GLSLC(1, for (int i = 0; i < planes; i++) { );
+ GLSLC(2, ivec2 size = imageSize(output_img[i]); );
+ GLSLC(2, if (!IS_WITHIN(pos, size)) );
+ GLSLC(3, continue; );
+ GLSLC(2, if ((planes == 1) || (i > 0)) { );
+ GLSLC(3, distort_chroma(i, size, pos); );
+ GLSLC(2, } else { );
+ GLSLC(3, vec2 npos = vec2(pos)/vec2(size); );
+ GLSLC(3, vec4 res = texture(input_img[i], npos); );
+ GLSLC(3, imageStore(output_img[i], pos, res); );
+ GLSLC(2, } );
+ GLSLC(1, } );
+ GLSLC(0, } );
+
+ RET(ff_vk_compile_shader(ctx, shd, "main"));
+ }
+
+ RET(ff_vk_init_pipeline_layout(ctx));
+
+ /* Execution context */
+ RET(av_vk_create_exec_ctx(s->vkctx.device, &s->exec,
+ s->vkctx.hwctx->queue_family_comp_index));
+
+ /* The pipeline */
+ RET(ff_vk_init_compute_pipeline(ctx));
+
+ s->initialized = 1;
+
+ return 0;
+
+fail:
+ return err;
+}
+
+static int process_frames(AVFilterContext *avctx, AVVkFrame *out, AVVkFrame *in)
+{
+ int err;
+ ChromaticAberrationVulkanContext *s = avctx->priv;
+ int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
+
+ VkCommandBufferBeginInfo cmd_start = {
+ .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
+ .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
+ };
+
+ VkComponentMapping null_map = {
+ .r = VK_COMPONENT_SWIZZLE_IDENTITY,
+ .g = VK_COMPONENT_SWIZZLE_IDENTITY,
+ .b = VK_COMPONENT_SWIZZLE_IDENTITY,
+ .a = VK_COMPONENT_SWIZZLE_IDENTITY,
+ };
+
+ for (int i = 0; i < planes; i++) {
+ RET(ff_vk_create_imageview(avctx, &s->input_images[i].imageView, in,
+ ff_vk_plane_rep_fmt(s->vkctx.input_format, i),
+ ff_vk_aspect_flags(s->vkctx.input_format, i),
+ null_map, NULL));
+
+ RET(ff_vk_create_imageview(avctx, &s->output_images[i].imageView, out,
+ ff_vk_plane_rep_fmt(s->vkctx.output_format, i),
+ ff_vk_aspect_flags(s->vkctx.output_format, i),
+ null_map, NULL));
+
+ s->input_images[i].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
+ s->input_images[i].sampler = s->vkctx.sampler;
+
+ s->output_images[i].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
+ }
+
+ ff_vk_update_descriptor_set(avctx, 0);
+
+ vkBeginCommandBuffer(s->exec.buf, &cmd_start);
+
+ {
+ VkImageMemoryBarrier bar[2] = {
+ {
+ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+ .srcAccessMask = 0,
+ .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT,
+ .oldLayout = in->layout,
+ .newLayout = VK_IMAGE_LAYOUT_GENERAL,
+ .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .image = in->img,
+ .subresourceRange.aspectMask = ff_vk_aspect_flags(s->vkctx.input_format, -1),
+ .subresourceRange.levelCount = 1,
+ .subresourceRange.layerCount = 1,
+ },
+ {
+ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+ .srcAccessMask = 0,
+ .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT,
+ .oldLayout = out->layout,
+ .newLayout = VK_IMAGE_LAYOUT_GENERAL,
+ .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .image = out->img,
+ .subresourceRange.aspectMask = ff_vk_aspect_flags(s->vkctx.input_format, -1),
+ .subresourceRange.levelCount = 1,
+ .subresourceRange.layerCount = 1,
+ },
+ };
+
+ vkCmdPipelineBarrier(s->exec.buf, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
+ VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0,
+ 0, NULL, 0, NULL, 2, bar);
+
+ in->layout = bar[0].newLayout;
+ in->access = bar[0].dstAccessMask;
+
+ out->layout = bar[1].newLayout;
+ out->access = bar[1].dstAccessMask;
+ }
+
+ vkCmdBindPipeline(s->exec.buf, VK_PIPELINE_BIND_POINT_COMPUTE, s->vkctx.pipeline);
+ vkCmdBindDescriptorSets(s->exec.buf, VK_PIPELINE_BIND_POINT_COMPUTE, s->vkctx.pipeline_layout, 0, s->vkctx.descriptor_sets_num, s->vkctx.desc_set, 0, 0);
+ vkCmdDispatch(s->exec.buf,
+ FFALIGN(s->vkctx.output_width, s->vkctx.shaders[0].local_size[0])/s->vkctx.shaders[0].local_size[0],
+ FFALIGN(s->vkctx.output_height, s->vkctx.shaders[0].local_size[1])/s->vkctx.shaders[0].local_size[1], 1);
+
+ vkEndCommandBuffer(s->exec.buf);
+
+ VkSubmitInfo s_info = {
+ .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
+ .commandBufferCount = 1,
+ .pCommandBuffers = &s->exec.buf,
+ };
+
+ VkResult ret = vkQueueSubmit(s->exec.queue, 1, &s_info, s->exec.fence);
+ if (ret != VK_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Unable to submit command buffer: %s\n",
+ av_vk_ret2str(ret));
+ return AVERROR_EXTERNAL;
+ } else {
+ vkWaitForFences(s->vkctx.hwctx->act_dev, 1, &s->exec.fence, VK_TRUE, UINT64_MAX);
+ vkResetFences(s->vkctx.hwctx->act_dev, 1, &s->exec.fence);
+ }
+
+fail:
+
+ for (int i = 0; i < planes; i++) {
+ ff_vk_destroy_imageview(avctx, s->input_images[i].imageView);
+ ff_vk_destroy_imageview(avctx, s->output_images[i].imageView);
+ }
+
+ return err;
+}
+
+static int chromaticaberration_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
+{
+ int err;
+ AVFilterContext *ctx = link->dst;
+ ChromaticAberrationVulkanContext *s = ctx->priv;
+ AVFilterLink *outlink = ctx->outputs[0];
+
+ AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+ if (!out) {
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+
+ if (!s->initialized)
+ RET(init_filter(ctx, in));
+
+ RET(process_frames(ctx, (AVVkFrame *)out->data[0],
+ (AVVkFrame *) in->data[0]));
+
+ err = av_frame_copy_props(out, in);
+ if (err < 0)
+ goto fail;
+
+ av_frame_free(&in);
+
+ return ff_filter_frame(outlink, out);
+
+fail:
+ av_frame_free(&in);
+ av_frame_free(&out);
+ return err;
+}
+
+static void chromaticaberration_vulkan_uninit(AVFilterContext *avctx)
+{
+ ChromaticAberrationVulkanContext *s = avctx->priv;
+
+ av_vk_free_exec_ctx(s->vkctx.device, &s->exec);
+ av_vk_free_buf(s->vkctx.device, &s->shader_buf);
+ ff_vk_filter_uninit(avctx);
+
+ s->initialized = 0;
+}
+
+#define OFFSET(x) offsetof(ChromaticAberrationVulkanContext, x)
+#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
+static const AVOption chromaticaberration_vulkan_options[] = {
+ { "dist_x", "Set horizontal distortion amount", OFFSET(dist_x), AV_OPT_TYPE_FLOAT, {.dbl = 0.0f}, 0.0f, 10.0f, .flags = FLAGS },
+ { "dist_y", "Set vertical distortion amount", OFFSET(dist_y), AV_OPT_TYPE_FLOAT, {.dbl = 0.0f}, 0.0f, 10.0f, .flags = FLAGS },
+ { NULL },
+};
+
+AVFILTER_DEFINE_CLASS(chromaticaberration_vulkan);
+
+static const AVFilterPad chromaticaberration_vulkan_inputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .filter_frame = &chromaticaberration_vulkan_filter_frame,
+ .config_props = &ff_vk_filter_config_input,
+ },
+ { NULL }
+};
+
+static const AVFilterPad chromaticaberration_vulkan_outputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .config_props = &ff_vk_filter_config_output,
+ },
+ { NULL }
+};
+
+AVFilter ff_vf_chromaticaberration_vulkan = {
+ .name = "chromaticaberration_vulkan",
+ .description = NULL_IF_CONFIG_SMALL("Apply chromaticaberration mask to input video"),
+ .priv_size = sizeof(ChromaticAberrationVulkanContext),
+ .init = &ff_vk_filter_init,
+ .uninit = &chromaticaberration_vulkan_uninit,
+ .query_formats = &ff_vk_filter_query_formats,
+ .inputs = chromaticaberration_vulkan_inputs,
+ .outputs = chromaticaberration_vulkan_outputs,
+ .priv_class = &chromaticaberration_vulkan_class,
+ .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
+};
diff --git a/libavfilter/vulkan.c b/libavfilter/vulkan.c
index c2e02f5d0a..c4da2ebe4a 100644
--- a/libavfilter/vulkan.c
+++ b/libavfilter/vulkan.c
@@ -430,7 +430,7 @@ static VkSamplerYcbcrModelConversion conv_primaries(enum AVColorPrimaries color_
return VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY;
}
-int ff_vk_init_sampler(AVFilterContext *avctx, AVFrame *input)
+int ff_vk_init_sampler(AVFilterContext *avctx, AVFrame *input, int unnorm_coords)
{
VkResult ret;
VulkanFilterContext *s = avctx->priv;
@@ -475,14 +475,15 @@ int ff_vk_init_sampler(AVFilterContext *avctx, AVFrame *input)
.pNext = input ? &s->yuv_sampler : NULL,
.magFilter = VK_FILTER_LINEAR,
.minFilter = VK_FILTER_LINEAR,
- .mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST,
+ .mipmapMode = unnorm_coords ? VK_SAMPLER_MIPMAP_MODE_NEAREST :
+ VK_SAMPLER_MIPMAP_MODE_LINEAR,
.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
.addressModeV = sampler.addressModeU,
.addressModeW = sampler.addressModeU,
.anisotropyEnable = VK_FALSE,
.compareOp = VK_COMPARE_OP_NEVER,
.borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK,
- .unnormalizedCoordinates = 1,
+ .unnormalizedCoordinates = unnorm_coords,
};
ret = vkCreateSampler(s->hwctx->act_dev, &sampler, NULL, &s->sampler);
diff --git a/libavfilter/vulkan.h b/libavfilter/vulkan.h
index 6e059731b7..a7becf9ae0 100644
--- a/libavfilter/vulkan.h
+++ b/libavfilter/vulkan.h
@@ -154,7 +154,7 @@ int ff_vk_add_push_constant(AVFilterContext *avctx, int offset, int size,
int ff_vk_init_pipeline_layout(AVFilterContext *avctx);
/* Create a Vulkan sampler, if input isn't NULL the sampler will convert to RGB */
-int ff_vk_init_sampler(AVFilterContext *avctx, AVFrame *input);
+int ff_vk_init_sampler(AVFilterContext *avctx, AVFrame *input, int unnorm_coords);
/* Creates a compute pipeline */
int ff_vk_init_compute_pipeline(AVFilterContext *avctx);
--
2.16.3
More information about the ffmpeg-devel
mailing list