[FFmpeg-devel] [Toy] LucasArts SMUSH demuxer and decoder

Måns Rullgård mans
Sun Jan 23 17:38:33 CET 2011


Vitor Sessak <vitor1001 at gmail.com> writes:

> On 01/29/2009 09:47 AM, Kostya wrote:
>> Again, for the record and not for review.
>>
>> I just took very old patch and extended it to decode palettized LucasArts FMVs
>> from the games like Full Throttle, Curse of Monkey Island, The Dig, etc.
>>
>> I would be glad if somebody pick and finish it (there are minor glitches during
>> decoding and fade in/fade out scenes are rendered as madly flipping frames
>> instead; oh, and the code style is rather bad).
>
> Git-friendly patch attached. Hope that patchwork will catch it up.
>
> -Vitor
>
> From 94c41ff457f032d1327b67868f39e9e804eecb87 Mon Sep 17 00:00:00 2001
> From: Kostya Shishkov <kostya.shishkov at gmail.com>
> Date: Sun, 23 Jan 2011 15:36:23 +0100
> Subject: [PATCH] Add LucasArts SMUSH demuxer and decoder.
>
> ---
>  libavcodec/Makefile      |    1 +
>  libavcodec/allcodecs.c   |    1 +
>  libavcodec/avcodec.h     |    1 +
>  libavcodec/sanm.c        | 1381 ++++++++++++++++++++++++++++++++++++++++++++++
>  libavformat/Makefile     |    1 +
>  libavformat/allformats.c |    1 +
>  libavformat/sanm.c       |  395 +++++++++++++
>  7 files changed, 1781 insertions(+), 0 deletions(-)
>  create mode 100644 libavcodec/sanm.c
>  create mode 100644 libavformat/sanm.c
>
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 9abedfc..cf782a6 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -185,6 +185,7 @@ OBJS-$(CONFIG_RV20_DECODER)            += rv10.o h263.o mpeg12data.o mpegvideo.o
>  OBJS-$(CONFIG_RV20_ENCODER)            += rv10.o mpegvideo_enc.o motion_est.o ratecontrol.o h263.o mpeg12data.o mpegvideo.o error_resilience.o
>  OBJS-$(CONFIG_RV30_DECODER)            += rv30.o rv34.o h264pred.o rv30dsp.o
>  OBJS-$(CONFIG_RV40_DECODER)            += rv40.o rv34.o h264pred.o rv40dsp.o
> +OBJS-$(CONFIG_SANM_DECODER)            += sanm.o
>  OBJS-$(CONFIG_SGI_DECODER)             += sgidec.o
>  OBJS-$(CONFIG_SGI_ENCODER)             += sgienc.o rle.o
>  OBJS-$(CONFIG_SHORTEN_DECODER)         += shorten.o
> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
> index f369c0a..8da8077 100644
> --- a/libavcodec/allcodecs.c
> +++ b/libavcodec/allcodecs.c
> @@ -137,6 +137,7 @@ void avcodec_register_all(void)
>      REGISTER_ENCDEC  (RV20, rv20);
>      REGISTER_DECODER (RV30, rv30);
>      REGISTER_DECODER (RV40, rv40);
> +    REGISTER_DECODER (SANM, sanm);
>      REGISTER_ENCDEC  (SGI, sgi);
>      REGISTER_DECODER (SMACKER, smacker);
>      REGISTER_DECODER (SMC, smc);
> diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
> index 6aa41b5..135b2bc 100644
> --- a/libavcodec/avcodec.h
> +++ b/libavcodec/avcodec.h
> @@ -190,6 +190,7 @@ enum CodecID {
>      CODEC_ID_MOTIONPIXELS,
>      CODEC_ID_TGV,
>      CODEC_ID_TGQ,
> +    CODEC_ID_SANM,
>  
>      /* various PCM "codecs" */
>      CODEC_ID_PCM_S16LE= 0x10000,
> diff --git a/libavcodec/sanm.c b/libavcodec/sanm.c
> new file mode 100644
> index 0000000..f01ab11
> --- /dev/null
> +++ b/libavcodec/sanm.c
> @@ -0,0 +1,1381 @@
> +/*
> + * LucasArts Smush v2 decoder
> + * Copyright (c) 2006 Cyril Zorin
> + * <firstname dot lastname at gmail dot com>
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + *
> + */
> +
> +/**
> + * @file sanm.c
> + * LucasArts Smush v2 decoder
> + */
> +
> +/*
> + * Based on http://wiki.multimedia.cx/index.php?title=SANM
> + */
> +
> +#include "avcodec.h"
> +#include "bytestream.h"
> +
> +#define GLYPH_COORD_VECT_SIZE 16
> +static const int8_t glyph4_x[GLYPH_COORD_VECT_SIZE] = { 0, 1, 2, 3, 3, 3, 3, 2, 1, 0, 0, 0, 1, 2, 2, 1 };
> +static const int8_t glyph4_y[GLYPH_COORD_VECT_SIZE] = { 0, 0, 0, 0, 1, 2, 3, 3, 3, 3, 2, 1, 1, 1, 2, 2 };
> +static const int8_t glyph8_x[GLYPH_COORD_VECT_SIZE] = { 0, 2, 5, 7, 7, 7, 7, 7, 7, 5, 2, 0, 0, 0, 0, 0 };
> +static const int8_t glyph8_y[GLYPH_COORD_VECT_SIZE] = { 0, 0, 0, 0, 1, 3, 4, 6, 7, 7, 7, 7, 6, 4, 3, 1 };
> +
> +static const int8_t motion_vectors[256][2] =
> +{
> +    {0,   0}, {-1, -43}, {6, -43},  {-9, -42},  {13, -41},
> +    {-16, -40},  {19, -39}, {-23, -36},  {26, -34},  {-2, -33},
> +    {4, -33}, {-29, -32},  {-9, -32},  {11, -31}, {-16, -29},
> +    {32, -29},  {18, -28}, {-34, -26}, {-22, -25},  {-1, -25},

[...]

> +};

This table could be made prettier.  For your c&p pleasure:

    {  0,   0}, { -1, -43}, {  6, -43}, { -9, -42}, { 13, -41},
    {-16, -40}, { 19, -39}, {-23, -36}, { 26, -34}, { -2, -33},
    {  4, -33}, {-29, -32}, { -9, -32}, { 11, -31}, {-16, -29},
    { 32, -29}, { 18, -28}, {-34, -26}, {-22, -25}, { -1, -25},
    {  3, -25}, { -7, -24}, {  8, -24}, { 24, -23}, { 36, -23},
    {-12, -22}, { 13, -21}, {-38, -20}, {  0, -20}, {-27, -19},
    { -4, -19}, {  4, -19}, {-17, -18}, { -8, -17}, {  8, -17},
    { 18, -17}, { 28, -17}, { 39, -17}, {-12, -15}, { 12, -15},
    {-21, -14}, { -1, -14}, {  1, -14}, {-41, -13}, { -5, -13},
    {  5, -13}, { 21, -13}, {-31, -12}, {-15, -11}, { -8, -11},
    {  8, -11}, { 15, -11}, { -2, -10}, {  1, -10}, { 31, -10},
    {-23,  -9}, {-11,  -9}, { -5,  -9}, {  4,  -9}, { 11,  -9},
    { 42,  -9}, {  6,  -8}, { 24,  -8}, {-18,  -7}, { -7,  -7},
    { -3,  -7}, { -1,  -7}, {  2,  -7}, { 18,  -7}, {-43,  -6},
    {-13,  -6}, { -4,  -6}, {  4,  -6}, {  8,  -6}, {-33,  -5},
    { -9,  -5}, { -2,  -5}, {  0,  -5}, {  2,  -5}, {  5,  -5},
    { 13,  -5}, {-25,  -4}, { -6,  -4}, { -3,  -4}, {  3,  -4},
    {  9,  -4}, {-19,  -3}, { -7,  -3}, { -4,  -3}, { -2,  -3},
    { -1,  -3}, {  0,  -3}, {  1,  -3}, {  2,  -3}, {  4,  -3},
    {  6,  -3}, { 33,  -3}, {-14,  -2}, {-10,  -2}, { -5,  -2},
    { -3,  -2}, { -2,  -2}, { -1,  -2}, {  0,  -2}, {  1,  -2},
    {  2,  -2}, {  3,  -2}, {  5,  -2}, {  7,  -2}, { 14,  -2},
    { 19,  -2}, { 25,  -2}, { 43,  -2}, { -7,  -1}, { -3,  -1},
    { -2,  -1}, { -1,  -1}, {  0,  -1}, {  1,  -1}, {  2,  -1},
    {  3,  -1}, { 10,  -1}, { -5,   0}, { -3,   0}, { -2,   0},
    { -1,   0}, {  1,   0}, {  2,   0}, {  3,   0}, {  5,   0},
    {  7,   0}, {-10,   1}, { -7,   1}, { -3,   1}, { -2,   1},
    { -1,   1}, {  0,   1}, {  1,   1}, {  2,   1}, {  3,   1},
    {-43,   2}, {-25,   2}, {-19,   2}, {-14,   2}, { -5,   2},
    { -3,   2}, { -2,   2}, { -1,   2}, {  0,   2}, {  1,   2},
    {  2,   2}, {  3,   2}, {  5,   2}, {  7,   2}, { 10,   2},
    { 14,   2}, {-33,   3}, { -6,   3}, { -4,   3}, { -2,   3},
    { -1,   3}, {  0,   3}, {  1,   3}, {  2,   3}, {  4,   3},
    { 19,   3}, { -9,   4}, { -3,   4}, {  3,   4}, {  7,   4},
    { 25,   4}, {-13,   5}, { -5,   5}, { -2,   5}, {  0,   5},
    {  2,   5}, {  5,   5}, {  9,   5}, { 33,   5}, { -8,   6},
    { -4,   6}, {  4,   6}, { 13,   6}, { 43,   6}, {-18,   7},
    { -2,   7}, {  0,   7}, {  2,   7}, {  7,   7}, { 18,   7},
    {-24,   8}, { -6,   8}, {-42,   9}, {-11,   9}, { -4,   9},
    {  5,   9}, { 11,   9}, { 23,   9}, {-31,  10}, { -1,  10},
    {  2,  10}, {-15,  11}, { -8,  11}, {  8,  11}, { 15,  11},
    { 31,  12}, {-21,  13}, { -5,  13}, {  5,  13}, { 41,  13},
    { -1,  14}, {  1,  14}, { 21,  14}, {-12,  15}, { 12,  15},
    {-39,  17}, {-28,  17}, {-18,  17}, { -8,  17}, {  8,  17},
    { 17,  18}, { -4,  19}, {  0,  19}, {  4,  19}, { 27,  19},
    { 38,  20}, {-13,  21}, { 12,  22}, {-36,  23}, {-24,  23},
    { -8,  24}, {  7,  24}, { -3,  25}, {  1,  25}, { 22,  25},
    { 34,  26}, {-18,  28}, {-32,  29}, { 16,  29}, {-11,  31},
    {  9,  32}, { 29,  32}, { -4,  33}, {  2,  33}, {-26,  34},
    { 23,  36}, {-19,  39}, { 16,  40}, {-13,  41}, {  9,  42},
    { -6,  43}, {  1,  43}, {  0,   0}, {  0,   0}, {  0,   0},

> +#define NGLYPHS 256
> +static int8_t p4x4glyphs[NGLYPHS][16];
> +static int8_t p8x8glyphs[NGLYPHS][64];
> +
> +typedef struct sanm_ctx
> +{
> +    AVCodecContext *avctx;
> +    const uint8_t* psrc;
> +
> +    int version, subversion;
> +    uint32_t pal[256];
> +    int16_t delta_pal[768];
> +
> +    int pitch;
> +    int width, height;
> +    int aligned_width, aligned_height;
> +    int prev_seq;
> +
> +    AVFrame frame, *output;
> +    uint16_t* pdb0, * pdb1, * pdb2;
> +    int rotate_code;
> +
> +    long npixels, buf_size;
> +
> +    uint16_t* pcodebook;
> +    uint16_t* psmall_codebook;
> +} sanm_ctx;

I prefer keeping stars attached to the name they apply to rather than
the type name.  The latter is confusing.

> +typedef struct sanm_frame_header
> +{
> +    int seq_num, codec, rotate_code, rle_output_size;
> +
> +    uint16_t bg_color;
> +    uint32_t width, height;
> +
> +    const uint8_t* pvstream;
> +} sanm_frame_header;
> +
> +typedef enum glyph_edge
> +{
> +   left_edge,
> +   top_edge,
> +   right_edge,
> +   bottom_edge,
> +   no_edge
> +} glyph_edge;
> +
> +typedef enum glyph_dir
> +{
> +    dir_left = 0,
> +    dir_up,
> +    dir_right,
> +    dir_down
> +} glyph_dir;

Uppercase enum values is common practice.  We also don't tend to
typedef enums, but that's getting into nitpicking territory.

> +static uint16_t* point_at(uint16_t* pbuf, int x, int y, int width)
> +{
> +    return &pbuf[y * width + x];
> +}
> +
> +static glyph_edge which_edge(int x, int y, int edge_size)
> +{
> +    glyph_edge edge;
> +    const int edge_max = edge_size - 1;
> +
> +    if (!y)
> +    {
> +        edge = bottom_edge;
> +    }
> +    else if (edge_max == y)
> +    {
> +        edge = top_edge;
> +    }
> +    else if (!x)
> +    {
> +        edge = left_edge;
> +    }
> +    else if (edge_max == x)
> +    {
> +        edge = right_edge;
> +    }
> +    else
> +    {
> +        edge = no_edge;
> +    }
> +
> +    return edge;
> +}

Brace placement is inconsistent with usual ffmpeg style.

> +static glyph_dir which_direction(glyph_edge edge0, glyph_edge edge1)
> +{
> +    glyph_dir dir = -1;
> +
> +    if ((left_edge == edge0 && right_edge == edge1) ||
> +        (left_edge == edge1 && right_edge == edge0) ||
> +        (bottom_edge == edge0 && top_edge != edge1) ||
> +        (bottom_edge == edge1 && top_edge != edge0))
> +    {
> +        dir = dir_up;
> +    }
> +    else if ((top_edge == edge0 && bottom_edge != edge1) ||
> +             (top_edge == edge1 && bottom_edge != edge0))
> +    {
> +        dir = dir_down;
> +    }
> +    else if ((left_edge == edge0 && right_edge != edge1) ||
> +             (left_edge == edge1 && right_edge != edge0))
> +    {
> +        dir = dir_left;
> +    }
> +    else if ((top_edge == edge0 && bottom_edge == edge1) ||
> +             (top_edge == edge1 && bottom_edge == edge0) ||
> +             (right_edge == edge0 && left_edge != edge1) ||
> +             (right_edge == edge1 && left_edge != edge0))
> +    {
> +        dir = dir_right;
> +    }
> +
> +    return dir;
> +}

Shift, or, and a switch or table lookup seems simpler.

> +static void interp_point(int8_t* ppoint, int x0, int y0, int x1, int y1, int ipoint, int npoints)
> +{
> +    if (npoints)
> +    {
> +        ppoint[0] = (x0 * ipoint + x1 * (npoints - ipoint) + npoints / 2) / npoints;
> +        ppoint[1] = (y0 * ipoint + y1 * (npoints - ipoint) + npoints / 2) / npoints;
> +    }
> +    else
> +    {
> +        ppoint[0] = x0;
> +        ppoint[1] = y0;
> +    }
> +}
> +
> +static void make_glyphs(int8_t* pglyphs, const int8_t* pxvector, const int8_t* pyvector, const int side_length)
> +{
> +    const int glyph_size = side_length * side_length;
> +    int8_t* pglyph = pglyphs;
> +
> +    int i, j;
> +    for (i = 0; i != GLYPH_COORD_VECT_SIZE; ++i)
> +    {
> +        int x0 = pxvector[i], y0 = pyvector[i];
> +        glyph_edge edge0 = which_edge(x0, y0, side_length);
> +
> +        for (j = 0; j != GLYPH_COORD_VECT_SIZE; ++j, pglyph += glyph_size)
> +        {
> +            int x1 = pxvector[j], y1 = pyvector[j];
> +            glyph_edge edge1 = which_edge(x1, y1, side_length);
> +            glyph_dir dir = which_direction(edge0, edge1);
> +            int ipoint, npoints = FFMAX(FFABS(x1 - x0), FFABS(y1 - y0));
> +
> +            memset(pglyph, 0, glyph_size * sizeof(pglyph[0]));
> +            for (ipoint = 0; ipoint <= npoints; ++ipoint)
> +            {
> +                int8_t point[2];
> +                int irow, icol;
> +
> +                interp_point(point, x0, y0, x1, y1, ipoint, npoints);
> +
> +                switch (dir)
> +                {
> +                case dir_up:
> +                    for (irow = point[1]; irow >= 0; --irow)
> +                    {
> +                        pglyph[point[0] + irow*side_length] = 1;
> +                    }
> +                    break;
> +
> +                case dir_down:
> +                    for (irow = point[1]; irow < side_length; ++irow)
> +                    {
> +                        pglyph[point[0] + irow*side_length] = 1;
> +                    }
> +                    break;
> +
> +                case dir_left:
> +                    for (icol = point[0]; icol >= 0; --icol)
> +                    {
> +                        pglyph[icol + point[1]*side_length] = 1;
> +                    }
> +                    break;
> +
> +                case dir_right:
> +                    for (icol = point[0]; icol < side_length; ++icol)
> +                    {
> +                        pglyph[icol + point[1]*side_length] = 1;
> +                    }
> +                    break;
> +                }
> +            }
> +        }
> +    }
> +}
> +
> +static void init_sizes(sanm_ctx* pctx, int width, int height)
> +{
> +    pctx->width = width;
> +    pctx->height = height;
> +    pctx->npixels = width * height;
> +
> +    pctx->aligned_width = (width + 7) & ~7;
> +    pctx->aligned_height = (height + 7) & ~7;

FFALIGN()

> +    pctx->buf_size = pctx->aligned_width * pctx->aligned_height * sizeof(pctx->pdb0[0]);
> +    pctx->pitch = width;
> +}
> +
> +static void init_buffers(sanm_ctx* pctx)
> +{
> +    pctx->pdb0 = av_realloc(pctx->pdb0, pctx->buf_size);
> +    memset(pctx->pdb0, 0, pctx->buf_size);
> +
> +    pctx->pdb1 = av_realloc(pctx->pdb1, pctx->buf_size);
> +    memset(pctx->pdb1, 0, pctx->buf_size);
> +
> +    pctx->pdb2 = av_realloc(pctx->pdb2, pctx->buf_size);
> +    memset(pctx->pdb2, 0, pctx->buf_size);
> +}

Unchecked mallocs.

> +static void destroy_buffers(sanm_ctx* pctx)
> +{
> +    av_free(pctx->pdb0);
> +    av_free(pctx->pdb1);
> +    av_free(pctx->pdb2);
> +}
> +
> +static int decode_init(AVCodecContext* pav_ctx)
> +{
> +    sanm_ctx* pctx = pav_ctx->priv_data;
> +
> +    pctx->avctx = pav_ctx;
> +    /*if (avcodec_check_dimensions(pav_ctx, pav_ctx->height, pav_ctx->width) < 0)
> +    {
> +        return -1;
> +    }*/

This commented out bit should be either fixed or removed.

> +    pctx->version = !pav_ctx->extradata_size;
> +
> +    pav_ctx->has_b_frames = 0;
> +    pav_ctx->pix_fmt = pctx->version ? PIX_FMT_RGB565 : PIX_FMT_PAL8;
> +
> +    init_sizes(pctx, pav_ctx->width, pav_ctx->height);
> +    init_buffers(pctx);
> +    pctx->output = &pctx->frame;
> +    pctx->output->data[0] = 0;
> +    pctx->output->quality = 1;



> +    make_glyphs(&p4x4glyphs[0][0], glyph4_x, glyph4_y, 4);
> +    make_glyphs(&p8x8glyphs[0][0], glyph8_x, glyph8_y, 8);
> +
> +    if(!pctx->version){
> +        int i;
> +        pctx->subversion = AV_RL16(pav_ctx->extradata);
> +        for(i = 0; i < 256; i++)
> +            pctx->pal[i] = AV_RL32(pav_ctx->extradata + 2 + i*4);
> +    }
> +
> +    return 0;
> +}
> +
> +static int decode_end(AVCodecContext* pav_ctx)
> +{
> +    sanm_ctx* pctx = pav_ctx->priv_data;
> +    int i;
> +
> +    destroy_buffers(pctx);
> +
> +    if (pctx->frame.data[0]){
> +        pav_ctx->release_buffer(pav_ctx, &pctx->frame);
> +        pctx->frame.data[0] = 0;
> +    }
> +
> +    return 0;
> +}
> +
> +static int read_frame_header(sanm_ctx* pctx, const uint8_t* pinput, sanm_frame_header* pheader)
> +{
> +    pinput += 8; // skip pad
> +
> +    pheader->width = AV_RL32(pinput); pinput += 4;
> +    pheader->height = AV_RL32(pinput); pinput += 4;

Use bytestream_get_le32() and friends here and below.

> +    if (pheader->width != pctx->width || pheader->height != pctx->height)
> +    {
> +        av_log(0, AV_LOG_ERROR, "sanm decoder: variable size frames are not implemented. video may be garbled until next keyframe.\n");
> +        return -1;
> +    }
> +
> +    pheader->seq_num = AV_RL16(pinput); pinput += 2;
> +    pheader->codec = *pinput; pinput += 1;
> +    pheader->rotate_code = *pinput; pinput += 1;
> +
> +    pinput += 4; // skip pad
> +
> +    pctx->psmall_codebook = (uint16_t*) pinput; pinput += 8;
> +    pheader->bg_color = AV_RL16(pinput); pinput += 2;
> +
> +    pinput += 2; // skip pad
> +
> +    pheader->rle_output_size = AV_RL32(pinput); pinput += 4;
> +    pctx->pcodebook = (uint16_t*) pinput; pinput += 512; // 256 entries in codebook
> +
> +    pinput += 8; // skip pad
> +
> +    pheader->pvstream = pinput;
> +
> +    return 0;
> +}
> +
> +static void fill_db(uint16_t* pbuf, int buf_size, uint16_t color)

db?

> +{
> +    while (buf_size--)
> +    {
> +        *pbuf++ = color;
> +    }
> +}

Fill functions for various sizes is something we should implement in a
common location.  Just an observation.

> +static void swap(uint16_t** ppleft, uint16_t** ppright)
> +{
> +    uint16_t* ptemp = *ppleft;
> +    *ppleft = *ppright;
> +    *ppright = ptemp;
> +}

FFSWAP()

> +static void rotate_bufs(sanm_ctx* pctx, int rotate_code)
> +{
> +    if (1 == rotate_code)
> +    {
> +        swap(&pctx->pdb0, &pctx->pdb2);
> +    }
> +    else if (2 == rotate_code)
> +    {
> +        swap(&pctx->pdb1, &pctx->pdb2);
> +        swap(&pctx->pdb2, &pctx->pdb0);
> +    }
> +}
> +
> +static void codec0(sanm_ctx* pctx, const uint8_t* pinput)
> +{
> +    uint16_t* pdb0 = pctx->pdb0;
> +
> +    int x, y;
> +    for (y = 0; y != pctx->height; ++y)
> +    {
> +        for (x = 0; x != pctx->width; ++x)
> +        {
> +            *point_at(pdb0, x, y, pctx->pitch) = AV_RL16(pinput); pinput += 2;
> +        }
> +    }
> +}
> +
> +static void codec6(sanm_ctx* pctx, const uint8_t* pinput)
> +{
> +    int npixels = pctx->npixels;
> +    uint16_t* pdb0 = pctx->pdb0;
> +    uint16_t* pcodebook = pctx->pcodebook;
> +
> +    int index;
> +    while (npixels--)
> +    {
> +        index = *pinput++;
> +        *pdb0++ = AV_RL16(&pcodebook[index]);
> +    }
> +}
> +
> +static void copy_block(uint16_t* pdest, uint16_t* psrc, int block_size, int pitch)
> +{
> +    int y;
> +    for (y = 0; y != block_size; ++y, pdest += pitch, psrc += pitch)
> +    {
> +        memcpy(pdest, psrc, block_size * sizeof(pdest[0]));
> +    }
> +}
> +
> +static void fill_block(uint16_t* pdest, uint16_t color, int block_size, int pitch)
> +{
> +    int x, y;
> +    pitch -= block_size;
> +    for (y = 0; y != block_size; ++y, pdest += pitch)
> +    {
> +        for (x = 0; x != block_size; ++x)
> +        {
> +            *pdest++ = color;
> +        }
> +    }
> +}

These two functions also look rather generic.  Perhaps candidates for
sharing.

> +static void draw_glyph(uint16_t* pdest, int index, uint16_t fg_color, uint16_t bg_color, int block_size, int pitch)
> +{
> +    int8_t* pglyph;
> +    uint16_t colors[2] = { fg_color, bg_color };
> +    int x, y;
> +
> +    if (index > NGLYPHS)
> +    {
> +        av_log(0, AV_LOG_ERROR, "sanm decoder: ignoring nonexistent glyph #%u.\n", index);
> +        return;
> +    }
> +
> +    pglyph = (8 == block_size) ? p8x8glyphs[index] : p4x4glyphs[index];
> +    pitch -= block_size;
> +
> +    for (y = 0; y != block_size; ++y, pdest += pitch)
> +    {
> +        for (x = 0; x != block_size; ++x)
> +        {
> +            *pdest++ = colors[*pglyph++];

What happens if input contains something other than 0 and 1?

> +        }
> +    }
> +}
> +
> +static void opcode_0xf7(sanm_ctx* pctx, int cx, int cy, int block_size, int pitch)
> +{
> +    if (2 == block_size)
> +    {
> +        uint16_t* pcodebook = pctx->pcodebook;
> +        uint16_t* pdest = point_at(pctx->pdb0, cx, cy, pctx->pitch);
> +        uint32_t indices = AV_RL32(pctx->psrc); pctx->psrc += 4;
> +
> +        pdest[0] = AV_RL16(&pcodebook[indices & 0xff]); indices >>= 8;
> +        pdest[1] = AV_RL16(&pcodebook[indices & 0xff]); indices >>= 8;
> +        pdest[pitch] = AV_RL16(&pcodebook[indices & 0xff]); indices >>= 8;
> +        pdest[pitch + 1] = AV_RL16(&pcodebook[indices & 0xff]);
> +    }
> +    else
> +    {
> +        uint16_t fgcolor, bgcolor;
> +
> +        int index = *pctx->psrc++;
> +        uint16_t color_indices = AV_RL16(pctx->psrc); pctx->psrc += 2;
> +        fgcolor = AV_RL16(&pctx->pcodebook[color_indices >> 8]);
> +        bgcolor = AV_RL16(&pctx->pcodebook[color_indices & 0xff]);
> +
> +        draw_glyph(point_at(pctx->pdb0, cx, cy, pctx->pitch), index, fgcolor, bgcolor, block_size, pitch);
> +    }
> +}
> +
> +static void opcode_0xf8(sanm_ctx* pctx, int cx, int cy, int block_size, int pitch)
> +{
> +    if (2 == block_size)
> +    {
> +        uint16_t* pdest = point_at(pctx->pdb0, cx, cy, pctx->pitch);
> +        pdest[0] = AV_RL16(pctx->psrc); pctx->psrc += 2;
> +        pdest[1] = AV_RL16(pctx->psrc); pctx->psrc += 2;
> +        pdest[pitch] = AV_RL16(pctx->psrc); pctx->psrc += 2;
> +        pdest[pitch + 1] = AV_RL16(pctx->psrc); pctx->psrc += 2;
> +    }
> +    else
> +    {
> +        uint16_t fgcolor, bgcolor;
> +
> +        int index = *pctx->psrc++;
> +        uint32_t colors = AV_RL32(pctx->psrc); pctx->psrc += 4;
> +        fgcolor = colors >> 16;
> +        bgcolor = colors & 0xffff;
> +
> +        draw_glyph(point_at(pctx->pdb0, cx, cy, pctx->pitch), index, fgcolor, bgcolor, block_size, pitch);
> +    }
> +}
> +
> +static int good_mvec(sanm_ctx* pctx, int cx, int cy, int mx, int my, int block_size)
> +{
> +    int good = point_at(pctx->pdb2, cx + mx + block_size - 1, cy + my + block_size - 1, pctx->pitch) < (&pctx->pdb2[pctx->buf_size / 2]);

This sanity check looks rather incomplete.

> +    if (!good)
> +    {
> +        av_log(0, AV_LOG_ERROR, "sanm decoder: ignoring segfault-inducing motion vector (%i, %i)->(%u, %u), block size = %u.\n",
> +               cx + mx, cy + my, cx, cy, block_size);
> +    }
> +
> +    return good;
> +}
> +
> +static void codec2level(sanm_ctx* pctx, int cx, int cy, int block_size)
> +{
> +    int16_t mx, my, index;
> +    uint16_t color;
> +
> +    int opcode = *pctx->psrc++;
> +
> +    switch (opcode)
> +    {
> +    default:
> +        mx = motion_vectors[opcode][0];
> +        my = motion_vectors[opcode][1];
> +
> +        if (good_mvec(pctx, cx, cy, mx, my, block_size))
> +        {
> +            copy_block(point_at(pctx->pdb0, cx, cy, pctx->pitch),
> +                       point_at(pctx->pdb2, cx + mx, cy + my, pctx->pitch),
> +                       block_size, pctx->pitch);
> +        }
> +        break;
> +
> +    case 0xf5:
> +        index = AV_RL16(pctx->psrc); pctx->psrc += 2;
> +
> +        mx = index % pctx->width;
> +        my = index / pctx->width;
> +
> +        if (good_mvec(pctx, cx, cy, mx, my, block_size))
> +        {
> +            copy_block(point_at(pctx->pdb0, cx, cy, pctx->pitch),
> +                       point_at(pctx->pdb2, cx + mx, cy + my, pctx->pitch),
> +                       block_size, pctx->pitch);
> +        }
> +        break;
> +
> +    case 0xf6:
> +        copy_block(point_at(pctx->pdb0, cx, cy, pctx->pitch),
> +                   point_at(pctx->pdb1, cx, cy, pctx->pitch),
> +                   block_size, pctx->pitch);
> +        break;
> +
> +    case 0xf7:
> +        opcode_0xf7(pctx, cx, cy, block_size, pctx->pitch);
> +        break;
> +
> +    case 0xf8:
> +        opcode_0xf8(pctx, cx, cy, block_size, pctx->pitch);
> +        break;
> +
> +    case 0xf9:
> +    case 0xfa:
> +    case 0xfb:
> +    case 0xfc:
> +        color = AV_RL16(&pctx->psmall_codebook[opcode - 0xf9]);
> +        fill_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), color, block_size, pctx->pitch);
> +        break;
> +
> +    case 0xfd:
> +        index = *pctx->psrc++;
> +        color = AV_RL16(&pctx->pcodebook[index]);
> +        fill_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), color, block_size, pctx->pitch);
> +        break;
> +
> +    case 0xfe:
> +        color = AV_RL16(pctx->psrc); pctx->psrc += 2;
> +        fill_block(point_at(pctx->pdb0, cx, cy, pctx->pitch), color, block_size, pctx->pitch);
> +        break;
> +
> +    case 0xff:
> +        if (2 == block_size)
> +        {
> +            opcode_0xf8(pctx, cx, cy, block_size, pctx->pitch);
> +        }
> +        else
> +        {
> +            block_size >>= 1;
> +            codec2level(pctx, cx             , cy             , block_size);
> +            codec2level(pctx, cx + block_size, cy             , block_size);
> +            codec2level(pctx, cx             , cy + block_size, block_size);
> +            codec2level(pctx, cx + block_size, cy + block_size, block_size);
> +        }
> +        break;
> +    }
> +}
> +
> +static void rle_decode(uint8_t* pdest, const uint8_t* pinput, const int out_size)
> +{
> +    int opcode, color, run_len, remaining = out_size;
> +
> +    assert(out_size > 0);
> +
> +    while (remaining)
> +    {
> +        opcode = *pinput++;
> +        run_len = (opcode >> 1) + 1;
> +        assert(run_len <= remaining);
> +
> +        if (opcode & 1) // rle strip
> +        {
> +            color = *pinput++;
> +            memset(pdest, color, run_len);
> +        }
> +        else
> +        {
> +            memcpy(pdest, pinput, run_len);
> +            pinput += run_len;
> +        }
> +
> +        pdest += run_len;
> +        remaining -= run_len;
> +    }
> +}
> +
> +static void codec5(sanm_ctx* pctx, const uint8_t* pinput, const int decoded_size)
> +{
> +    uint8_t* pdest = (uint8_t*) pctx->pdb0;
> +    int npixels = pctx->npixels;
> +    uint16_t* pdb0 = pctx->pdb0;
> +
> +    assert(!(decode_size & 1));
> +
> +    rle_decode(pdest, pinput, decoded_size);
> +
> +    while (npixels--)
> +    {
> +        *pdb0 = AV_RL16(pdb0);
> +        ++pdb0;
> +    }
> +}

Yet another candidate for a shared bswap16_buf().

> +static void codec2(sanm_ctx* pctx, const uint8_t* pinput)
> +{
> +    const int width = pctx->aligned_width, height = pctx->aligned_height;
> +    int cx, cy;
> +
> +    pctx->psrc = pinput;
> +    for (cy = 0; cy != height; cy += 8)
> +    {
> +        for (cx = 0; cx != width; cx += 8)
> +        {
> +            codec2level(pctx, cx, cy, 8);
> +        }
> +    }
> +}
> +
> +static void codec8(sanm_ctx* pctx, const uint8_t* pinput, const int decoded_size)
> +{
> +    uint16_t* pdest = pctx->pdb0;
> +    uint8_t* pindices = av_malloc(decoded_size);

Unchecked malloc.

> +    uint8_t* pidx_cursor = pindices;
> +    long npixels = pctx->npixels;
> +
> +    rle_decode(pindices, pinput, decoded_size);
> +
> +    while (npixels--)
> +    {
> +        int index = *pidx_cursor++;
> +        *pdest++ = AV_RL16(&pctx->pcodebook[index]);
> +    }
> +
> +    av_free(pindices);
> +}

I have a feeling this could be done without the temp buffer.

> +static void copy_output(sanm_ctx* pctx, sanm_frame_header* pheader)
> +{
> +    uint8_t* poutput;
> +    const uint8_t* pinput = (uint8_t*) pctx->pdb0;
> +
> +    int height = pctx->height;
> +    long inpitch = pctx->pitch * (pheader ? sizeof(pctx->pdb0[0]) : 1);
> +    long outpitch;
> +
> +    if (pctx->avctx->get_buffer(pctx->avctx, pctx->output))
> +    {
> +        av_log(pctx->avctx, AV_LOG_ERROR, "sanm decoder: get_buffer() failed.\n");
> +        return;
> +    }
> +    poutput = pctx->output->data[0];
> +    outpitch = pctx->output->linesize[0];
> +    while (height--)
> +    {
> +        memcpy(poutput, pinput, inpitch);
> +        pinput += inpitch;
> +        poutput += outpitch;
> +    }
> +}

Why not decode directly into the get_buffer() buffers?

> +static void codec1(sanm_ctx* pctx, const uint8_t* pinput, int top, int left, int width, int height)
> +{
> +    uint8_t *dst = ((uint8_t*)pctx->pdb0) + left + top * pctx->pitch;
> +    const uint8_t* end;
> +    int i, j, len, flag, code, val, pos;
> +    for(i = 0; i < height; i++){
> +        pos = 0;
> +        len = bytestream_get_le16(&pinput);
> +        end = pinput + len;
> +        while(pinput < end){
> +            code = bytestream_get_byte(&pinput);
> +            flag = code & 1;
> +            code = (code >> 1) + 1;
> +            if(flag){
> +                val = bytestream_get_byte(&pinput);
> +                if(val)
> +                    memset(dst + pos, val, code);
> +                pos += code;
> +            }else{
> +                for(j = 0; j < code; j++){
> +                    val = bytestream_get_byte(&pinput);
> +                    if(val)
> +                        dst[pos] = val;
> +                    pos++;
> +                }
> +            }

This is missing output buffer bounds checks.

> +        }
> +        dst += pctx->pitch;
> +    }
> +    pctx->rotate_code = 0;
> +}
> +

[...]

> +static void codec37(sanm_ctx* pctx, const uint8_t* src, int top, int left, int width, int height)
> +{
> +    int stride = pctx->pitch;
> +    uint8_t *dst;
> +    uint8_t *prev;
> +    uint8_t *buf;
> +    int compr = src[0];
> +    int mvoff = src[1];
> +    int seq = src[2];
> +    int i, j, k, t;
> +    int flags = src[12];
> +    int skip_run = 0;
> +
> +    pctx->rotate_code = 0;
> +    if(((seq & 1) || !(flags & 1)) && (compr >= 3))
> +        rotate_bufs(pctx, 1);
> +    dst = ((uint8_t*)pctx->pdb0) + left + top * stride;
> +    prev = pctx->pdb2;
> +    src += 16;
> +    switch(compr){
> +    case 0:
> +        for(i = 0; i < height; i++){
> +            memcpy(dst, src, width);
> +            src += width;
> +            dst += stride;
> +        }
> +        memset(pctx->pdb1, 0, pctx->height * stride);
> +        memset(pctx->pdb2, 0, pctx->height * stride);
> +        break;
> +    case 1:
> +        av_log(NULL,0,"Codec 37 compr 1: TODO\n");

av_log_missing_feature() here and elsewhere.

> +        break;
> +    case 2:
> +        t = width * height;
> +        buf = av_malloc(t+16);
> +        rle_decode(buf, src, t);
> +        src = buf;
> +        for(j = 0; j < height; j++){
> +            for(i = 0; i < width; i++)
> +                memcpy(dst, src, width);
> +            dst += stride;
> +            src += width;
> +        }
> +        memset(pctx->pdb1, 0, pctx->height * stride);
> +        memset(pctx->pdb2, 0, pctx->height * stride);
> +        av_free(buf);
> +        break;
> +    case 3:
> +    case 4:
> +        if(flags & 4){
> +            for(j = 0; j < height; j += 4){
> +                for(i = 0; i < width; i += 4){
> +                    int code;
> +                    if(skip_run){
> +                        skip_run--;
> +                        for(k = 0; k < 4; k++)
> +                            memcpy(dst + i + k*stride, prev + i + k*stride, 4);
> +                        continue;
> +                    }
> +                    code = bytestream_get_byte(&src);
> +                    switch(code){
> +                    case 0xFF:
> +                        for(k = 0; k < 4; k++)
> +                            bytestream_get_buffer(&src, dst + i + k*stride, 4);
> +                        break;
> +                    case 0xFE:
> +                        for(k = 0; k < 4; k++)
> +                            memset(dst + i + k*stride, bytestream_get_byte(&src), 4);
> +                        break;
> +                    case 0xFD:
> +                        t = bytestream_get_byte(&src);
> +                        for(k = 0; k < 4; k++)
> +                            memset(dst + i + k*stride, t, 4);
> +                        break;
> +                    default:
> +                        if(compr == 4 && !code){
> +                            skip_run = bytestream_get_byte(&src) + 1;
> +                            i -= 4;
> +                        }else{
> +                            int mx, my;
> +                            mx = c37_mv[(mvoff*255 + code)*2];
> +                            my = c37_mv[(mvoff*255 + code)*2+1];
> +                            for(k = 0; k < 4; k++)
> +                                memcpy(dst + i + k*stride, prev + i + mx + (k + my)*stride, 4);
> +                        }
> +                    }
> +                }
> +                dst  += stride * 4;
> +                prev += stride * 4;
> +            }
> +        }else{
> +            for(j = 0; j < height; j += 4){
> +                for(i = 0; i < width; i += 4){
> +                    int code;
> +                    if(skip_run){
> +                        skip_run--;
> +                        for(k = 0; k < 4; k++)
> +                            memcpy(dst + i + k*stride, prev + i + k*stride, 4);
> +                        continue;
> +                    }
> +                    code = bytestream_get_byte(&src);
> +                    if(code == 0xFF){
> +                        for(k = 0; k < 4; k++)
> +                            bytestream_get_buffer(&src, dst + i + k*stride, 4);
> +                    }else if(compr == 4 && !code){
> +                        skip_run = bytestream_get_byte(&src) + 1;
> +                        i -= 4;
> +                    }else{
> +                        int mx, my;
> +                        mx = c37_mv[(mvoff*255 + code)*2];
> +                        my = c37_mv[(mvoff*255 + code)*2+1];
> +                        if(my + j >= 0)
> +                        for(k = 0; k < 4; k++)
> +                            memcpy(dst + i + k*stride, prev + i + mx + (k + my)*stride, 4);
> +                    }
> +                }
> +                dst  += stride * 4;
> +                prev += stride * 4;
> +            }
> +        }
> +        break;
> +    }
> +}
> +
> +static void process_block(uint8_t *dst, uint8_t *prev1, uint8_t *prev2, int stride, const uint8_t **src, const uint8_t *tbl, int size)
> +{
> +    int code;
> +    int k, t;
> +
> +    code = bytestream_get_byte(src);
> +    if(code >= 0xF8){
> +        switch(code){
> +        case 0xFF:
> +            if(size == 2){
> +                dst[0] = bytestream_get_byte(src);
> +                dst[1] = bytestream_get_byte(src);
> +                dst[0+stride] = bytestream_get_byte(src);
> +                dst[1+stride] = bytestream_get_byte(src);
> +            }else{
> +                size >>= 1;
> +                process_block(dst,      prev1,      prev2,      stride, src, tbl, size);
> +                process_block(dst+size, prev1+size, prev2+size, stride, src, tbl, size);
> +                dst   += size * stride;
> +                prev1 += size * stride;
> +                prev2 += size * stride;
> +                process_block(dst,      prev1,      prev2,      stride, src, tbl, size);
> +                process_block(dst+size, prev1+size, prev2+size, stride, src, tbl, size);
> +            }
> +            break;
> +        case 0xFE:
> +            t = bytestream_get_byte(src);
> +            for(k = 0; k < size; k++)
> +                memset(dst + k*stride, t, size);
> +            break;
> +        case 0xFD:
> +            {
> +                int code = bytestream_get_byte(src);
> +                int8_t *pglyph = (size == 8) ? p8x8glyphs[code] : p4x4glyphs[code];
> +
> +                for(k = 0; k < size; k++)
> +                    for(t = 0; t < size; t++)
> +                        dst[t + k*stride] = src[0][!(*pglyph++)];
> +                *src += 2;
> +            }
> +            break;
> +        case 0xFC:
> +            for(k = 0; k < size; k++)
> +                memcpy(dst + k*stride, prev1 + k*stride, size);
> +            break;
> +        default:
> +            t = tbl[code & 7];
> +            for(k = 0; k < size; k++)
> +                memset(dst + k*stride, t, size);
> +        }
> +    }else{
> +        int mx = motion_vectors[code][0];
> +        int my = motion_vectors[code][1];
> +        for(k = 0; k < size; k++)
> +            memcpy(dst + k*stride, prev2 + mx + (my+k)*stride, size);
> +    }
> +}
> +
> +static void codec47(sanm_ctx* pctx, const uint8_t* pinput, int top, int left, int width, int height)
> +{
> +    int compr = pinput[2];
> +    const uint8_t *src = pinput + 26;
> +    int stride = pctx->pitch;
> +    uint8_t *dst = pctx->pdb0 + left + top * stride;
> +    uint8_t *prev1 = pctx->pdb1;
> +    uint8_t *prev2 = pctx->pdb2;
> +    int i, j, t;
> +    uint8_t *buf;
> +    int seq = AV_RL16(pinput);
> +    if(pinput[4] & 1)
> +        src += 0x8080;
> +    if(!seq)
> +        pctx->prev_seq = -1;
> +    switch(compr){
> +    case 0:
> +        for(j = 0; j < height; j++){
> +            for(i = 0; i < width; i++)
> +                memcpy(dst, src, width);
> +            dst += stride;
> +            src += width;
> +        }
> +        break;
> +    case 1:
> +        av_log(NULL,0,"Codec 47 compr 1: TODO\n");
> +        break;
> +    case 2:
> +        if(seq == pctx->prev_seq + 1){
> +        for(j = 0; j < height; j += 8){
> +            for(i = 0; i < width; i += 8){
> +                process_block(dst + i, prev1 + i, prev2 + i, stride, &src, pinput + 8, 8);
> +            }
> +            dst  += stride * 8;
> +            prev1 += stride * 8;
> +            prev2 += stride * 8;
> +        }
> +        }
> +        break;
> +    case 3:
> +        memcpy(pctx->pdb0, pctx->pdb2, pctx->pitch * pctx->height);
> +        break;
> +    case 4:
> +        memcpy(pctx->pdb0, pctx->pdb1, pctx->pitch * pctx->height);
> +        break;
> +    case 5:
> +        t = AV_RL32(pinput + 14);
> +        buf = av_malloc(t+16);

Unchecked malloc and with size provided by input to boot.

> +        rle_decode(buf, src, t);
> +        src = buf;
> +        for(j = 0; j < height; j++){
> +            for(i = 0; i < width; i++)
> +                memcpy(dst, src, width);
> +            dst += stride;
> +            src += width;
> +        }
> +        av_free(buf);
> +        break;
> +    }
> +    if(seq == pctx->prev_seq + 1)
> +        pctx->rotate_code = pinput[3];
> +    else
> +        pctx->rotate_code = 0;
> +    pctx->prev_seq = seq;
> +}
> +
> +
> +static void process_npal(sanm_ctx* pctx, const uint8_t* pinput)
> +{
> +    int i;
> +    for(i = 0; i < 256; i++)
> +        pctx->pal[i] = bytestream_get_be24(&pinput);
> +}

I'm sure I've seen this pattern somewhere before.

> +static void process_xpal(sanm_ctx* pctx, const uint8_t* pinput, int size)
> +{
> +    int i, j;
> +
> +    if(size == 6){
> +        for(i = 0; i < 256; i++){
> +            uint8_t tmp[4];
> +            for(j = 0; j < 3; j++){
> +                int t = (pctx->pal[i] >> (16 - j*8)) & 0xFF;
> +                tmp[2-j] = av_clip_uint8((t*129 + pctx->delta_pal[i*3+j]) / 128);
> +            }
> +            pctx->pal[i] = AV_RL24(tmp);
> +        }
> +    }else{
> +        pinput += 4;
> +        for(i = 0; i < 768; i++)
> +            pctx->delta_pal[i] = bytestream_get_le16(&pinput);
> +        for(i = 0; i < 256; i++)
> +            pctx->pal[i] = bytestream_get_be24(&pinput);
> +    }
> +}
> +
> +static void process_fobj(sanm_ctx* pctx, const uint8_t* pinput)
> +{
> +    int i;
> +    int codec, top, left, w, h;
> +    codec = bytestream_get_le16(&pinput);
> +    top = bytestream_get_le16(&pinput);
> +    left = bytestream_get_le16(&pinput);
> +    w = bytestream_get_le16(&pinput);
> +    h = bytestream_get_le16(&pinput);
> +    if(pctx->width < left + w || pctx->height < top + h){
> +        pctx->avctx->width = left + w;
> +        pctx->avctx->height = top + h;
> +        init_sizes(pctx, left + w, top + h);
> +        init_buffers(pctx);
> +    }
> +    pinput += 4;
> +    switch(codec){
> +    case 1:
> +        codec1(pctx, pinput, top, left, w, h);
> +        break;
> +    case 37:
> +        codec37(pctx, pinput, top, left, w, h);
> +        break;
> +    case 47:
> +        codec47(pctx, pinput, top, left, w, h);
> +        break;
> +    default:
> +        av_log(NULL,0,"Unknown codec %d\n",codec);
> +    }
> +}
> +
> +static int dispatch_codec1(sanm_ctx* pctx, const uint8_t* pinput, const int isize)
> +{
> +    const uint8_t *frame_end = pinput + isize;
> +    while(pinput < frame_end - 8){
> +        int sig  = AV_RB32(pinput);
> +        int size = AV_RB32(pinput+4);
> +        pinput += 8;
> +        if(size < 0 || pinput + size > frame_end){
> +            av_log(NULL,0,"Incorrect size (%d|%X, %d)\n",size,size,frame_end-pinput);
> +            break;
> +        }
> +        switch(sig){
> +        case MKBETAG('N', 'P', 'A', 'L'):
> +            process_npal(pctx, pinput);
> +            break;
> +        case MKBETAG('F', 'O', 'B', 'J'):
> +            process_fobj(pctx, pinput);
> +            break;
> +        case MKBETAG('X', 'P', 'A', 'L'):
> +            process_xpal(pctx, pinput, size);
> +            break;
> +        }
> +
> +        pinput += size;
> +        if(size & 1)
> +            pinput++;
> +    }
> +    copy_output(pctx, NULL);
> +    rotate_bufs(pctx, pctx->rotate_code);
> +    memcpy(pctx->output->data[1], pctx->pal, 1024);
> +    return 0;
> +}
> +
> +static int dispatch_codec(sanm_ctx* pctx, const uint8_t* pinput)
> +{
> +    sanm_frame_header header;
> +    if (read_frame_header(pctx, pinput, &header))
> +    {
> +        return -1;
> +    }
> +
> +    if ((pctx->output->key_frame = !header.seq_num))
> +    {
> +        pctx->output->pict_type = FF_I_TYPE;
> +        fill_db(pctx->pdb1, pctx->npixels, header.bg_color);
> +        fill_db(pctx->pdb2, pctx->npixels, header.bg_color);
> +    }
> +    else
> +    {
> +        pctx->output->pict_type = FF_P_TYPE;
> +    }
> +
> +    switch (header.codec)
> +    {
> +    case 0:
> +        codec0(pctx, header.pvstream);
> +        break;
> +
> +    case 2:
> +        codec2(pctx, header.pvstream);
> +        break;
> +
> +    case 3:
> +        memcpy(pctx->pdb0, pctx->pdb2, pctx->buf_size);
> +        break;
> +
> +    case 4:
> +        memcpy(pctx->pdb0, pctx->pdb1, pctx->buf_size);
> +        break;
> +
> +    case 5:
> +        codec5(pctx, header.pvstream, header.rle_output_size);
> +        break;
> +
> +    case 6:
> +        codec6(pctx, header.pvstream);
> +        break;
> +
> +    case 8:
> +        codec8(pctx, header.pvstream, pctx->npixels);
> +        break;
> +
> +    default:
> +        av_log(0, AV_LOG_ERROR, "sanm decoder: subcodec %u is not implemented. video may be garbled until next keyframe.\n", header.codec);
> +        break;
> +    }

Perhaps a table of function pointers would be better.

> +    copy_output(pctx, &header);
> +    rotate_bufs(pctx, header.rotate_code);
> +
> +    return 0;
> +}
> +
> +static int decode_frame(AVCodecContext* pav_ctx, void* poutput, int* poutput_size, uint8_t* pinput, int input_size)
> +{
> +    sanm_ctx* pctx = pav_ctx->priv_data;
> +
> +    if (pctx->output->data[0])
> +    {
> +        pav_ctx->release_buffer(pav_ctx, pctx->output);
> +    }
> +
> +    if (!pctx->version && dispatch_codec1(pctx, pinput, input_size))
> +    {
> +        return -1;
> +    }
> +
> +    if (pctx->version && dispatch_codec(pctx, pinput))
> +    {
> +        return -1;
> +    }
> +
> +    *poutput_size = sizeof(AVFrame);
> +    *((AVFrame*) poutput) = *pctx->output;
> +
> +    return input_size;
> +}
> +
> +AVCodec sanm_decoder =
> +{
> +    "sanm",
> +    CODEC_TYPE_VIDEO,
> +    CODEC_ID_SANM,
> +    sizeof(sanm_ctx),
> +    decode_init,
> +    0,
> +    decode_end,
> +    decode_frame
> +};

This decoder is lacking error/sanity checks in many places.  I suspect
a fuzz attack would make it fall over rather quickly.  Needs more work.

Leaving the demuxer for later.

-- 
M?ns Rullg?rd
mans at mansr.com



More information about the ffmpeg-devel mailing list