[FFmpeg-devel] [PATCH] Fix segfault in x11grab when drawing Cursor on Xservers that don't support the XFixes extension

[Måns Rullgård]

Brenden Bain <brenden.bain at gmail.com> writes:

> Index: libavdevice/x11grab.c
> ===================================================================
> --- libavdevice/x11grab.c	(revision 23755)
> +++ libavdevice/x11grab.c	(working copy)
> @@ -91,7 +91,7 @@
>      XImage *image;
>      int x_off = 0;
>      int y_off = 0;
> -    int use_shm;
> +    int use_shm, ignore;
>      char *param, *offset;
>  
>      param = av_strdup(s1->filename);
> @@ -115,6 +115,11 @@
>          return AVERROR(EIO);
>      }
>  
> +    if (!XFixesQueryExtension(dpy, &ignore, &ignore)) {
> +        av_log(s1, AV_LOG_INFO, "Disabling cursor recording. Unable to query cursor shape.\n");
> +        x11grab->nomouse = 1;
> +    }
> +
>      st = av_new_stream(s1, 0);
>      if (!st) {
>          return AVERROR(ENOMEM);
> @@ -245,8 +250,9 @@
>   *          coordinates
>   * @param x Mouse pointer coordinate
>   * @param y Mouse pointer coordinate
> + * @return 0 if an error occurred, 1 otherwise.

Usual convention is 0 for success, negative for error.

>   */
> -static void
> +static int
>  paint_mouse_pointer(XImage *image, struct x11_grab *s)
>  {
>      int x_off = s->x_off;
> @@ -262,6 +268,10 @@
>  
>      xcim = XFixesGetCursorImage(dpy);
>  
> +    if (xcim == NULL) {
> +        return 0;
> +    }
> +
>      x = xcim->x - xcim->xhot;
>      y = xcim->y - xcim->yhot;
>  
> @@ -284,6 +294,7 @@
>  
>      XFree(xcim);
>      xcim = NULL;
> +    return 1;
>  }
>  
>  
> @@ -388,7 +399,11 @@
>      }
>  
>      if(!s->nomouse){
> -        paint_mouse_pointer(image, s);
> +        if (!paint_mouse_pointer(image, s)) {
> +            /* An error occured and will likely occur again. Just disable mouse capture from now on */

That comment is superfluous.  Anyone with half a brain can see pointer
capture is being turned off.

> +            av_log(s1, AV_LOG_INFO, "Couldn't load mouse image. Unable to render cursor in video.\n");
> +            s->nomouse = 1;
> +        }
>      }
>  
>  

-- 
M?ns Rullg?rd
mans at mansr.com