[FFmpeg-devel] [PATCH]Fix pnm encoding with bpc set

[Carl Eugen Hoyos]

Hi!

Attached patch fixes pnm encoding for me if bits_per_raw_sample is set to a 
value different from the pix_fmt's native value.

Please comment, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/pnmenc.c b/libavcodec/pnmenc.c
index 9198ddb..8f74459 100644
--- a/libavcodec/pnmenc.c
+++ b/libavcodec/pnmenc.c
@@ -27,7 +27,7 @@ static int pnm_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                             const AVFrame *p, int *got_packet)
 {
     uint8_t *bytestream, *bytestream_start, *bytestream_end;
-    int i, h, h1, c, n, linesize, ret;
+    int i, j, h, h1, c, n, linesize, ret;
     uint8_t *ptr, *ptr1, *ptr2;
 
     if ((ret = ff_alloc_packet2(avctx, pkt, avpicture_get_size(avctx->pix_fmt,
@@ -84,7 +84,8 @@ static int pnm_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     bytestream += strlen(bytestream);
     if (avctx->pix_fmt != AV_PIX_FMT_MONOWHITE) {
         int maxdepth = (1 << (av_pix_fmt_desc_get(avctx->pix_fmt)->comp[0].depth_minus1 + 1)) - 1;
-        if( avctx->bits_per_raw_sample )
+        if (   avctx->bits_per_raw_sample
+            && av_pix_fmt_desc_get(avctx->pix_fmt)->comp[0].depth_minus1 > 7)
             maxdepth = (1 << avctx->bits_per_raw_sample) - 1;
         snprintf(bytestream, bytestream_end - bytestream,
                  "%d\n", maxdepth);
@@ -94,7 +95,13 @@ static int pnm_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     ptr      = p->data[0];
     linesize = p->linesize[0];
     for (i = 0; i < h; i++) {
-        memcpy(bytestream, ptr, n);
+        int bps_delta = av_pix_fmt_desc_get(avctx->pix_fmt)->comp[0].depth_minus1 + 1 - avctx->bits_per_raw_sample;
+        if (!avctx->bits_per_raw_sample || !bps_delta) {
+            memcpy(bytestream, ptr, n);
+        } else {
+            for (j = 0; j < avctx->width; j++)
+                ((uint16_t *)bytestream)[j] = ((uint16_t *)ptr)[j] >> bps_delta;
+        }
         bytestream += n;
         ptr        += linesize;
     }