From 7d8e198c33b7ad283db53315129209a2bd310f23 Mon Sep 17 00:00:00 2001
From: Mary-nyan <mary@mary.zone>
Date: Wed, 2 Nov 2022 09:26:50 +0100
Subject: [PATCH] fix: Support FFmpeg 5.1.x for decoding (#3816)

For some reason FFmpeg 5.1.x reverted part of the changes made in 5.0.x
on AVCodec.

This fix decoding issues with it.
---
 Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs         | 10 ++++++----
 Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodec.cs        |  1 +
 .../Native/{AVCodecLegacy.cs => AVCodec501.cs}         |  3 +--
 Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodecContext.cs |  2 +-
 Ryujinx.Graphics.Nvdec.FFmpeg/Native/FFCodec.cs        |  6 ++----
 5 files changed, 11 insertions(+), 11 deletions(-)
 rename Ryujinx.Graphics.Nvdec.FFmpeg/Native/{AVCodecLegacy.cs => AVCodec501.cs} (92%)

diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs
index 8c2e94c3..572ceaaa 100644
--- a/Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs
@@ -7,7 +7,9 @@ namespace Ryujinx.Graphics.Nvdec.FFmpeg
 {
     unsafe class FFmpegContext : IDisposable
     {
-        private readonly FFCodec.AVCodec_decode _decodeFrame;
+        private unsafe delegate int AVCodec_decode(AVCodecContext* avctx, void* outdata, int* got_frame_ptr, AVPacket* avpkt);
+
+        private readonly AVCodec_decode _decodeFrame;
         private static readonly FFmpegApi.av_log_set_callback_callback _logFunc;
         private readonly AVCodec* _codec;
         private AVPacket* _packet;
@@ -53,17 +55,17 @@ namespace Ryujinx.Graphics.Nvdec.FFmpeg
             // libavcodec 59.24 changed AvCodec to move its private API and also move the codec function to an union.
             if (avCodecMajorVersion > 59 || (avCodecMajorVersion == 59 && avCodecMinorVersion > 24))
             {
-                _decodeFrame = Marshal.GetDelegateForFunctionPointer<FFCodec.AVCodec_decode>(((FFCodec*)_codec)->CodecCallback);
+                _decodeFrame = Marshal.GetDelegateForFunctionPointer<AVCodec_decode>(((FFCodec<AVCodec>*)_codec)->CodecCallback);
             }
             // libavcodec 59.x changed AvCodec private API layout.
             else if (avCodecMajorVersion == 59)
             {
-                _decodeFrame = Marshal.GetDelegateForFunctionPointer<FFCodec.AVCodec_decode>(((FFCodecLegacy<AVCodec>*)_codec)->Decode);
+                _decodeFrame = Marshal.GetDelegateForFunctionPointer<AVCodec_decode>(((FFCodecLegacy<AVCodec501>*)_codec)->Decode);
             }
             // libavcodec 58.x and lower
             else
             {
-                _decodeFrame = Marshal.GetDelegateForFunctionPointer<FFCodec.AVCodec_decode>(((FFCodecLegacy<AVCodecLegacy>*)_codec)->Decode);
+                _decodeFrame = Marshal.GetDelegateForFunctionPointer<AVCodec_decode>(((FFCodecLegacy<AVCodec>*)_codec)->Decode);
             }
         }
 
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodec.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodec.cs
index 93edb788..46d3ad61 100644
--- a/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodec.cs
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodec.cs
@@ -20,6 +20,7 @@ namespace Ryujinx.Graphics.Nvdec.FFmpeg.Native
         public unsafe IntPtr PrivClass;
         public IntPtr Profiles;
         public unsafe byte* WrapperName;
+        public IntPtr ChLayouts;
 #pragma warning restore CS0649
     }
 }
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodecLegacy.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodec501.cs
similarity index 92%
rename from Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodecLegacy.cs
rename to Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodec501.cs
index 0913cbc4..47d4969a 100644
--- a/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodecLegacy.cs
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodec501.cs
@@ -2,7 +2,7 @@
 
 namespace Ryujinx.Graphics.Nvdec.FFmpeg.Native
 {
-    struct AVCodecLegacy
+    struct AVCodec501
     {
 #pragma warning disable CS0649
         public unsafe byte* Name;
@@ -20,7 +20,6 @@ namespace Ryujinx.Graphics.Nvdec.FFmpeg.Native
         public unsafe IntPtr PrivClass;
         public IntPtr Profiles;
         public unsafe byte* WrapperName;
-        public IntPtr ChLayouts;
 #pragma warning restore CS0649
     }
 }
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodecContext.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodecContext.cs
index 11bd63b0..6c9fbc89 100644
--- a/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodecContext.cs
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/Native/AVCodecContext.cs
@@ -9,7 +9,7 @@ namespace Ryujinx.Graphics.Nvdec.FFmpeg.Native
         public unsafe IntPtr AvClass;
         public int LogLevelOffset;
         public int CodecType;
-        public unsafe AVCodecLegacy* Codec;
+        public unsafe AVCodec* Codec;
         public AVCodecID CodecId;
         public uint CodecTag;
         public IntPtr PrivData;
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/Native/FFCodec.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/Native/FFCodec.cs
index 8b08c02c..4df45af4 100644
--- a/Ryujinx.Graphics.Nvdec.FFmpeg/Native/FFCodec.cs
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/Native/FFCodec.cs
@@ -2,12 +2,10 @@
 
 namespace Ryujinx.Graphics.Nvdec.FFmpeg.Native
 {
-    struct FFCodec
+    struct FFCodec<T> where T: struct
     {
-        public unsafe delegate int AVCodec_decode(AVCodecContext* avctx, void* outdata, int* got_frame_ptr, AVPacket* avpkt);
-
 #pragma warning disable CS0649
-        public AVCodec Base;
+        public T Base;
         public int CapsInternalOrCbType;
         public int PrivDataSize;
         public IntPtr UpdateThreadContext;