[FFmpeg-cvslog] avcodec/cfhd: improve decompanding quality with reference implementation

[Paul B Mahol]

ffmpeg | branch: master | Paul B Mahol <onemda at gmail.com> | Sat Aug  1 10:47:07 2020 +0200| [131d2a3e1cd2b7687e8cf751214c3884be4c154b] | committer: Paul B Mahol

avcodec/cfhd: improve decompanding quality with reference implementation

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=131d2a3e1cd2b7687e8cf751214c3884be4c154b
---

 libavcodec/cfhd.c     |  4 ++--
 tests/ref/fate/cfhd-3 | 20 ++++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/libavcodec/cfhd.c b/libavcodec/cfhd.c
index aa6e2ebee7..0327b456ac 100644
--- a/libavcodec/cfhd.c
+++ b/libavcodec/cfhd.c
@@ -117,8 +117,8 @@ static inline int dequant_and_decompand(int level, int quantisation, int codeboo
 {
     if (codebook == 0 || codebook == 1) {
         int64_t abslevel = abs(level);
-        if (level < 264)
-            return (abslevel + ((768 * abslevel * abslevel * abslevel) / (255 * 255 * 255))) *
+        if (abslevel < 256)
+            return (abslevel + ((768 * abslevel * abslevel * abslevel) / (256 * 256 * 256))) *
                FFSIGN(level) * quantisation;
         else
             return level * quantisation;
diff --git a/tests/ref/fate/cfhd-3 b/tests/ref/fate/cfhd-3
index 59fdc92260..79133a14b0 100644
--- a/tests/ref/fate/cfhd-3
+++ b/tests/ref/fate/cfhd-3
@@ -3,13 +3,13 @@
 #codec_id 0: rawvideo
 #dimensions 0: 496x241
 #sar 0: 0/1
-0,          0,          0,        1,   478144, 0x48a01dbb
-0,          1,          1,        1,   478144, 0x48a01dbb
-0,          2,          2,        1,   478144, 0x48a01dbb
-0,          3,          3,        1,   478144, 0xb978a72f
-0,          4,          4,        1,   478144, 0x7bbb4679
-0,          5,          5,        1,   478144, 0xc3fd3f59
-0,          6,          6,        1,   478144, 0xfd2a4816
-0,          7,          7,        1,   478144, 0x207f65d3
-0,          8,          8,        1,   478144, 0x207f65d3
-0,          9,          9,        1,   478144, 0x207f65d3
+0,          0,          0,        1,   478144, 0x1e5a0d6c
+0,          1,          1,        1,   478144, 0x1e5a0d6c
+0,          2,          2,        1,   478144, 0x1e5a0d6c
+0,          3,          3,        1,   478144, 0x88788c7d
+0,          4,          4,        1,   478144, 0x78643db8
+0,          5,          5,        1,   478144, 0x84303909
+0,          6,          6,        1,   478144, 0x8ddd4828
+0,          7,          7,        1,   478144, 0xc0845d58
+0,          8,          8,        1,   478144, 0xc0845d58
+0,          9,          9,        1,   478144, 0xc0845d58