[FFmpeg-cvslog] j2k: change fixed point of stepsize to 16.16

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michaelni at gmx.at> | Mon May 27 17:58:06 2013 +0200| [243cc38d944b1a457178d5984d250321361301b3] | committer: Michael Niedermayer

j2k: change fixed point of stepsize to 16.16

This seems more natural and matches jpeg2000

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=243cc38d944b1a457178d5984d250321361301b3
---

 libavcodec/j2k.c    |    2 +-
 libavcodec/j2kdec.c |    2 +-
 libavcodec/j2kenc.c |    4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/libavcodec/j2k.c b/libavcodec/j2k.c
index 171a101..2eca845 100644
--- a/libavcodec/j2k.c
+++ b/libavcodec/j2k.c
@@ -289,7 +289,7 @@ int ff_j2k_init_component(Jpeg2000Component *comp,
             }
             /* BITEXACT computing case --> convert to int */
 //             if (avctx->flags & CODEC_FLAG_BITEXACT)
-            band->stepsize = stepsize * (1 << 13);
+            band->stepsize = stepsize * (1 << 16);
 
             /* computation of tbx_0, tbx_1, tby_0, tby_1
              * see ISO/IEC 15444-1:2002 B.5 eq. B-15 and tbl B.1
diff --git a/libavcodec/j2kdec.c b/libavcodec/j2kdec.c
index b349c97..7bf0e9e 100644
--- a/libavcodec/j2kdec.c
+++ b/libavcodec/j2kdec.c
@@ -830,7 +830,7 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
                                 int *datap = &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * (y+j) + x];
                                 int *ptr = t1.data[j];
                                 for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
-                                    int tmp = ((int64_t)ptr[i]) * ((int64_t)band->stepsize) >> 13, tmp2;
+                                    int tmp = ((int64_t)ptr[i]) * ((int64_t)band->stepsize) >> 16, tmp2;
                                     tmp2 = FFABS(tmp>>1) + (tmp&1);
                                     datap[i] = tmp < 0 ? -tmp2 : tmp2;
                                 }
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c
index aea12f3..7807552 100644
--- a/libavcodec/j2kenc.c
+++ b/libavcodec/j2kenc.c
@@ -802,7 +802,7 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile)
                         Jpeg2000Cblk *cblk = prec->cblk + cblkno;
 
                         cblk->ninclpasses = getcut(cblk, s->lambda,
-                                (int64_t)dwt_norms[codsty->transform][bandpos][lev] * (int64_t)band->stepsize >> 13);
+                                (int64_t)dwt_norms[codsty->transform][bandpos][lev] * (int64_t)band->stepsize >> 16);
                     }
                 }
             }
@@ -863,7 +863,7 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno
                                 int *ptr = t1.data[y-yy0];
                                 for (x = xx0; x < xx1; x++){
                                     *ptr = (comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x]);
-                                    *ptr = (int64_t)*ptr * (int64_t)(8192 * 8192 / band->stepsize) >> 13 - NMSEDEC_FRACBITS;
+                                    *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->stepsize) >> 14 - NMSEDEC_FRACBITS;
                                     ptr++;
                                 }
                             }