From 3f28f88a358ff9beb4346f543811bdc82031e0fc Mon Sep 17 00:00:00 2001
From: kd-11 <karokidii@gmail.com>
Date: Sun, 29 Sep 2024 05:34:15 +0300
Subject: [PATCH] vk: Add support for explicit border colors

---
 rpcs3/Emu/RSX/VK/vkutils/sampler.cpp | 33 ++++++++++++++++++++++------
 rpcs3/Emu/RSX/VK/vkutils/sampler.h   | 19 +++++++++++-----
 2 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/rpcs3/Emu/RSX/VK/vkutils/sampler.cpp b/rpcs3/Emu/RSX/VK/vkutils/sampler.cpp
index 92aee578bc..c6bf83cdec 100644
--- a/rpcs3/Emu/RSX/VK/vkutils/sampler.cpp
+++ b/rpcs3/Emu/RSX/VK/vkutils/sampler.cpp
@@ -20,8 +20,8 @@ namespace vk
 		return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
 	}
 
-	border_color_t::border_color_t(u32 encoded_color)
-		: storage_key(0)
+	border_color_t::border_color_t(u32 encoded_color, VkFormat fmt, VkImageAspectFlags aspect)
+		: storage_key(0), format(fmt), aspect(aspect)
 	{
 		value = vk::get_border_color(encoded_color);
 
@@ -38,14 +38,33 @@ namespace vk
 			return;
 		}
 
-		storage_key = encoded_color;
+		if (aspect == VK_IMAGE_ASPECT_STENCIL_BIT)
+		{
+			// We must use INT color format for stencil
+			value = VK_BORDER_COLOR_INT_CUSTOM_EXT;
+			auto int_color = static_cast<s32>(color_value.z * 255.f);
+			color_value = color4f(std::bit_cast<f32>(int_color));
+		}
+
+		ensure(aspect <= VK_IMAGE_ASPECT_METADATA_BIT);
+		storage_key = static_cast<u64>(encoded_color)
+			| (static_cast<u64>(aspect) << 32)
+			| (static_cast<u64>(fmt) << 34);
 	}
 
+	border_color_t::border_color_t(VkBorderColor value)
+		: storage_key(0)
+		, value(value)
+		, format(VK_FORMAT_UNDEFINED)
+		, aspect(VK_IMAGE_ASPECT_COLOR_BIT)
+		, color_value(0.f)
+	{}
+
 	sampler::sampler(const vk::render_device& dev, VkSamplerAddressMode clamp_u, VkSamplerAddressMode clamp_v, VkSamplerAddressMode clamp_w,
 		VkBool32 unnormalized_coordinates, float mipLodBias, float max_anisotropy, float min_lod, float max_lod,
 		VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, const vk::border_color_t& border_color,
 		VkBool32 depth_compare, VkCompareOp depth_compare_mode)
-		: m_device(dev)
+		: m_device(dev), m_border_color(border_color)
 	{
 		info.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
 		info.addressModeU = clamp_u;
@@ -65,12 +84,12 @@ namespace vk
 		info.borderColor = border_color.value;
 
 		VkSamplerCustomBorderColorCreateInfoEXT custom_color_info;
-		if (border_color.value == VK_BORDER_COLOR_FLOAT_CUSTOM_EXT)
+		if (border_color.value >= VK_BORDER_COLOR_FLOAT_CUSTOM_EXT)
 		{
 			custom_color_info =
 			{
 				.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT,
-				.format = VK_FORMAT_UNDEFINED
+				.format = border_color.format
 			};
 
 			std::memcpy(custom_color_info.customBorderColor.float32, border_color.color_value.rgba, sizeof(float) * 4);
@@ -97,7 +116,7 @@ namespace vk
 		    info.compareEnable != depth_compare || info.unnormalizedCoordinates != unnormalized_coordinates ||
 		    !rsx::fcmp(info.maxLod, max_lod) || !rsx::fcmp(info.mipLodBias, mipLodBias) || !rsx::fcmp(info.minLod, min_lod) ||
 		    !rsx::fcmp(info.maxAnisotropy, max_anisotropy) ||
-		    info.compareOp != depth_compare_mode || info.borderColor != border_color)
+		    info.compareOp != depth_compare_mode || m_border_color != border_color)
 			return false;
 
 		return true;
diff --git a/rpcs3/Emu/RSX/VK/vkutils/sampler.h b/rpcs3/Emu/RSX/VK/vkutils/sampler.h
index ff86ffe9b4..182fdb0b27 100644
--- a/rpcs3/Emu/RSX/VK/vkutils/sampler.h
+++ b/rpcs3/Emu/RSX/VK/vkutils/sampler.h
@@ -7,11 +7,15 @@ namespace vk
 {
 	struct border_color_t
 	{
-		u32 storage_key;
+		u64 storage_key;
 		VkBorderColor value;
+		VkFormat format;
+		VkImageCreateFlags aspect;
 		color4f color_value;
 
-		border_color_t(u32 encoded_color);
+		border_color_t(u32 encoded_color, VkFormat fmt = VK_FORMAT_UNDEFINED, VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT | VK_IMAGE_ASPECT_DEPTH_BIT);
+
+		border_color_t(VkBorderColor color);
 
 		bool operator == (const border_color_t& that) const
 		{
@@ -20,12 +24,14 @@ namespace vk
 				return false;
 			}
 
-			if (this->value != VK_BORDER_COLOR_FLOAT_CUSTOM_EXT)
+			switch (this->value)
 			{
+			case VK_BORDER_COLOR_FLOAT_CUSTOM_EXT:
+			case VK_BORDER_COLOR_INT_CUSTOM_EXT:
+				return this->color_value == that.color_value;
+			default:
 				return true;
 			}
-
-			return this->color_value == that.color_value;
 		}
 	};
 
@@ -50,13 +56,14 @@ namespace vk
 		sampler(sampler&&) = delete;
 	private:
 		VkDevice m_device;
+		border_color_t m_border_color;
 	};
 
 	// Caching helpers
 	struct sampler_pool_key_t
 	{
 		u64 base_key;
-		u32 border_color_key;
+		u64 border_color_key;
 	};
 
 	struct cached_sampler_object_t : public vk::sampler, public rsx::ref_counted