From f5a6f45b27d43f8314227e4a8e7f9ddd681a4277 Mon Sep 17 00:00:00 2001
From: Mary <mary@mary.zone>
Date: Sat, 1 Apr 2023 10:05:02 +0200
Subject: vulkan: Separate debug utils logic from VulkanInitialization (#4609)

* vulkan: Separate debug utils logic from VulkanInitialization

Also checks for VK_EXT_debug_utils existence instead of force enabling it and allow possible error during messenger init

* Address gdkchan's comment

* Use CreateDebugUtilsMessenger Span variant
---
 Ryujinx.Graphics.Vulkan/VulkanInitialization.cs | 115 ++----------------------
 1 file changed, 7 insertions(+), 108 deletions(-)

(limited to 'Ryujinx.Graphics.Vulkan/VulkanInitialization.cs')

diff --git a/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs b/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs
index c4e9a626..f04ab5c0 100644
--- a/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs
+++ b/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs
@@ -47,19 +47,7 @@ namespace Ryujinx.Graphics.Vulkan
             KhrSwapchain.ExtensionName
         };
 
-        private static string[] _excludedMessages = new string[]
-        {
-            // NOTE: Done on purpose right now.
-            "UNASSIGNED-CoreValidation-Shader-OutputNotConsumed",
-            // TODO: Figure out if fixable
-            "VUID-vkCmdDrawIndexed-None-04584",
-            // TODO: Might be worth looking into making this happy to possibly optimize copies.
-            "UNASSIGNED-CoreValidation-DrawState-InvalidImageLayout",
-            // TODO: Fix this, it's causing too much noise right now.
-            "VUID-VkSubpassDependency-srcSubpass-00867"
-        };
-
-        internal static Instance CreateInstance(Vk api, GraphicsDebugLevel logLevel, string[] requiredExtensions, out ExtDebugUtils debugUtils, out DebugUtilsMessengerEXT debugUtilsMessenger)
+        internal static Instance CreateInstance(Vk api, GraphicsDebugLevel logLevel, string[] requiredExtensions)
         {
             var enabledLayers = new List<string>();
 
@@ -95,7 +83,12 @@ namespace Ryujinx.Graphics.Vulkan
                 AddAvailableLayer("VK_LAYER_KHRONOS_validation");
             }
 
-            var enabledExtensions = requiredExtensions.Append(ExtDebugUtils.ExtensionName).ToArray();
+            var enabledExtensions = requiredExtensions;
+
+            if (api.IsInstanceExtensionPresent("VK_EXT_debug_utils"))
+            {
+                enabledExtensions = enabledExtensions.Append(ExtDebugUtils.ExtensionName).ToArray();
+            }
 
             var appName = Marshal.StringToHGlobalAnsi(AppName);
 
@@ -145,47 +138,9 @@ namespace Ryujinx.Graphics.Vulkan
                 Marshal.FreeHGlobal(ppEnabledLayers[i]);
             }
 
-            CreateDebugMessenger(api, logLevel, instance, out debugUtils, out debugUtilsMessenger);
-
             return instance;
         }
 
-        private unsafe static uint DebugMessenger(
-            DebugUtilsMessageSeverityFlagsEXT messageSeverity,
-            DebugUtilsMessageTypeFlagsEXT messageTypes,
-            DebugUtilsMessengerCallbackDataEXT* pCallbackData,
-            void* pUserData)
-        {
-            var msg = Marshal.PtrToStringAnsi((IntPtr)pCallbackData->PMessage);
-
-            foreach (string excludedMessagePart in _excludedMessages)
-            {
-                if (msg.Contains(excludedMessagePart))
-                {
-                    return 0;
-                }
-            }
-
-            if (messageSeverity.HasFlag(DebugUtilsMessageSeverityFlagsEXT.ErrorBitExt))
-            {
-                Logger.Error?.Print(LogClass.Gpu, msg);
-            }
-            else if (messageSeverity.HasFlag(DebugUtilsMessageSeverityFlagsEXT.WarningBitExt))
-            {
-                Logger.Warning?.Print(LogClass.Gpu, msg);
-            }
-            else if (messageSeverity.HasFlag(DebugUtilsMessageSeverityFlagsEXT.InfoBitExt))
-            {
-                Logger.Info?.Print(LogClass.Gpu, msg);
-            }
-            else // if (messageSeverity.HasFlag(DebugUtilsMessageSeverityFlagsEXT.VerboseBitExt))
-            {
-                Logger.Debug?.Print(LogClass.Gpu, msg);
-            }
-
-            return 0;
-        }
-
         internal static PhysicalDevice FindSuitablePhysicalDevice(Vk api, Instance instance, SurfaceKHR surface, string preferredGpuId)
         {
             uint physicalDeviceCount;
@@ -671,61 +626,5 @@ namespace Ryujinx.Graphics.Vulkan
 
             return extensionProperties.Select(x => Marshal.PtrToStringAnsi((IntPtr)x.ExtensionName)).ToArray();
         }
-
-        internal unsafe static void CreateDebugMessenger(
-            Vk api,
-            GraphicsDebugLevel logLevel,
-            Instance instance,
-            out ExtDebugUtils debugUtils,
-            out DebugUtilsMessengerEXT debugUtilsMessenger)
-        {
-            debugUtils = default;
-
-            if (logLevel != GraphicsDebugLevel.None)
-            {
-                if (!api.TryGetInstanceExtension(instance, out debugUtils))
-                {
-                    debugUtilsMessenger = default;
-                    return;
-                }
-
-                var filterLogType = logLevel switch
-                {
-                    GraphicsDebugLevel.Error => DebugUtilsMessageTypeFlagsEXT.ValidationBitExt,
-                    GraphicsDebugLevel.Slowdowns => DebugUtilsMessageTypeFlagsEXT.ValidationBitExt |
-                                                    DebugUtilsMessageTypeFlagsEXT.PerformanceBitExt,
-                    GraphicsDebugLevel.All => DebugUtilsMessageTypeFlagsEXT.GeneralBitExt |
-                                              DebugUtilsMessageTypeFlagsEXT.ValidationBitExt |
-                                              DebugUtilsMessageTypeFlagsEXT.PerformanceBitExt,
-                    _ => throw new ArgumentException($"Invalid log level \"{logLevel}\".")
-                };
-
-                var filterLogSeverity = logLevel switch
-                {
-                    GraphicsDebugLevel.Error => DebugUtilsMessageSeverityFlagsEXT.ErrorBitExt,
-                    GraphicsDebugLevel.Slowdowns => DebugUtilsMessageSeverityFlagsEXT.ErrorBitExt |
-                                                    DebugUtilsMessageSeverityFlagsEXT.WarningBitExt,
-                    GraphicsDebugLevel.All => DebugUtilsMessageSeverityFlagsEXT.InfoBitExt |
-                                              DebugUtilsMessageSeverityFlagsEXT.WarningBitExt |
-                                              DebugUtilsMessageSeverityFlagsEXT.VerboseBitExt |
-                                              DebugUtilsMessageSeverityFlagsEXT.ErrorBitExt,
-                    _ => throw new ArgumentException($"Invalid log level \"{logLevel}\".")
-                };
-
-                var debugUtilsMessengerCreateInfo = new DebugUtilsMessengerCreateInfoEXT()
-                {
-                    SType = StructureType.DebugUtilsMessengerCreateInfoExt,
-                    MessageType = filterLogType,
-                    MessageSeverity = filterLogSeverity,
-                    PfnUserCallback = new PfnDebugUtilsMessengerCallbackEXT(DebugMessenger)
-                };
-
-                debugUtils.CreateDebugUtilsMessenger(instance, in debugUtilsMessengerCreateInfo, null, out debugUtilsMessenger).ThrowOnError();
-            }
-            else
-            {
-                debugUtilsMessenger = default;
-            }
-        }
     }
 }
-- 
cgit v1.2.3-70-g09d2