[FFmpeg-cvslog] avutil/wchar_filename: propagate MultiByteToWideChar() and WideCharToMultiByte() failures

[James Almer]

ffmpeg | branch: release/5.1 | James Almer <jamrial at gmail.com> | Wed Apr 12 22:53:33 2023 -0300| [ded3989a65fa2c03a499677172e01ee9f0489709] | committer: James Almer

avutil/wchar_filename: propagate MultiByteToWideChar() and WideCharToMultiByte() failures

Don't return success if the string could not be converted.

Signed-off-by: James Almer <jamrial at gmail.com>
(cherry picked from commit 92885f26817b6b2515ac9fd5410c9e8be64bd0c0)

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=ded3989a65fa2c03a499677172e01ee9f0489709
---

 libavutil/wchar_filename.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libavutil/wchar_filename.h b/libavutil/wchar_filename.h
index 9a04a069f1..fbc0a55146 100644
--- a/libavutil/wchar_filename.h
+++ b/libavutil/wchar_filename.h
@@ -32,7 +32,8 @@ static inline int utf8towchar(const char *filename_utf8, wchar_t **filename_w)
     num_chars = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, filename_utf8, -1, NULL, 0);
     if (num_chars <= 0) {
         *filename_w = NULL;
-        return 0;
+        errno = EINVAL;
+        return -1;
     }
     *filename_w = (wchar_t *)av_calloc(num_chars, sizeof(wchar_t));
     if (!*filename_w) {
@@ -52,7 +53,8 @@ static inline int wchartocp(unsigned int code_page, const wchar_t *filename_w,
                                         NULL, 0, NULL, NULL);
     if (num_chars <= 0) {
         *filename = NULL;
-        return 0;
+        errno = EINVAL;
+        return -1;
     }
     *filename = (char*)av_malloc_array(num_chars, sizeof *filename);
     if (!*filename) {