rsx: Fix image scaling

- Specifically fixes a corner case where double transforms are required.
  Technically this can be made more readable using transformation matrices:
  * M1 = transform_virtual_to_physical()
  * M2 = transform_image_to_virtual()
  * M3 = M1 * M2
  * Result = Input * M3
  But we don't use a CPU-side matrix library and it is not reasonable to do this on the GPU.

rpcs3/Emu/RSX/GL/GLPresent.cpp

@@ -199,18 +199,19 @@ void GLGSRender::flip(const rsx::display_flip_info_t& info)
 	const int height = m_frame->client_height();
 
 	// Calculate blit coordinates
-	coordi aspect_ratio;
-	const sizei csize(width, height);
-	sizei new_size = csize;
-
+	areai aspect_ratio;
 	if (!g_cfg.video.stretch_to_display_area)
 	{
-		avconfig.downscale_to_aspect_ratio(aspect_ratio.x, aspect_ratio.y, new_size.width, new_size.height);
+		const sizeu csize(width, height);
+		const auto converted = avconfig.aspect_convert_region(size2u{ buffer_width, buffer_height }, csize);
+		aspect_ratio = static_cast<areai>(converted);
+	}
+	else
+	{
+		aspect_ratio = { 0, 0, width, height };
 	}
 
-	aspect_ratio.size = new_size;
-
-	if (!image_to_flip || aspect_ratio.width < csize.width || aspect_ratio.height < csize.height)
+	if (!image_to_flip || aspect_ratio.x1 || aspect_ratio.y1)
 	{
 		// Clear the window background to black
 		gl_state.clear_color(0, 0, 0, 0);
@@ -251,7 +252,7 @@ void GLGSRender::flip(const rsx::display_flip_info_t& info)
 			m_flip_fbo.color = image_to_flip;
 			m_flip_fbo.read_buffer(m_flip_fbo.color);
 			m_flip_fbo.draw_buffer(m_flip_fbo.color);
-			m_flip_fbo.blit(gl::screen, screen_area, areai(aspect_ratio).flipped_vertical(), gl::buffers::color, gl::filter::linear);
+			m_flip_fbo.blit(gl::screen, screen_area, aspect_ratio.flipped_vertical(), gl::buffers::color, gl::filter::linear);
 		}
 		else
 		{

rpcs3/Emu/RSX/VK/VKPresent.cpp

@@ -550,16 +550,16 @@ void VKGSRender::flip(const rsx::display_flip_info_t& info)
 	ensure(m_current_frame->present_image != umax);
 
 	// Calculate output dimensions. Done after swapchain acquisition in case it was recreated.
-	coordi aspect_ratio;
-	const sizei csize = static_cast<sizei>(m_swapchain_dims);
-	sizei new_size = csize;
-
+	areai aspect_ratio;
 	if (!g_cfg.video.stretch_to_display_area)
 	{
-		avconfig.downscale_to_aspect_ratio(aspect_ratio.x, aspect_ratio.y, new_size.width, new_size.height);
+		const auto converted = avconfig.aspect_convert_region({ buffer_width, buffer_height }, m_swapchain_dims);
+		aspect_ratio = static_cast<areai>(converted);
+	}
+	else
+	{
+		aspect_ratio = { 0, 0, s32(m_swapchain_dims.width), s32(m_swapchain_dims.height) };
 	}
-
-	aspect_ratio.size = new_size;
 
 	// Blit contents to screen..
 	VkImage target_image = m_swapchain->get_image(m_current_frame->present_image);
@@ -572,7 +572,7 @@ void VKGSRender::flip(const rsx::display_flip_info_t& info)
 	vk::framebuffer_holder* direct_fbo = nullptr;
 	rsx::simple_array<vk::viewable_image*> calibration_src;
 
-	if (!image_to_flip || aspect_ratio.width < csize.width || aspect_ratio.height < csize.height)
+	if (!image_to_flip || aspect_ratio.x1 || aspect_ratio.y1)
 	{
 		// Clear the window background to black
 		VkClearColorValue clear_black {};
@@ -617,7 +617,7 @@ void VKGSRender::flip(const rsx::display_flip_info_t& info)
 				request.srcOffsets[0] = { 0, 0, 0 };
 				request.srcOffsets[1] = { s32(buffer_width), s32(buffer_height), 1 };
 				request.dstOffsets[0] = { 0, 0, 0 };
-				request.dstOffsets[1] = { aspect_ratio.width, aspect_ratio.height, 1 };
+				request.dstOffsets[1] = { aspect_ratio.width(), aspect_ratio.height(), 1 };
 
 				for (unsigned i = 0; i < calibration_src.size(); ++i)
 				{
@@ -645,15 +645,13 @@ void VKGSRender::flip(const rsx::display_flip_info_t& info)
 		else
 		{
 			// Do raw transfer here as there is no image object associated with textures owned by the driver (TODO)
-			const areai dst_rect = aspect_ratio;
 			VkImageBlit rgn = {};
-
 			rgn.srcSubresource = { image_to_flip->aspect(), 0, 0, 1 };
 			rgn.dstSubresource = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1 };
 			rgn.srcOffsets[0] = { 0, 0, 0 };
 			rgn.srcOffsets[1] = { s32(buffer_width), s32(buffer_height), 1 };
-			rgn.dstOffsets[0] = { dst_rect.x1, dst_rect.y1, 0 };
-			rgn.dstOffsets[1] = { dst_rect.x2, dst_rect.y2, 1 };
+			rgn.dstOffsets[0] = { aspect_ratio.x1, aspect_ratio.y1, 0 };
+			rgn.dstOffsets[1] = { aspect_ratio.x2, aspect_ratio.y2, 1 };
 
 			if (target_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
 			{

rpcs3/Emu/RSX/rsx_utils.cpp

@@ -103,6 +103,38 @@ namespace rsx
 		}
 	}
 
+	// Fit a aspect-correct rectangle within a frame of wxh dimensions
+	template <typename T>
+	area_base<T> convert_aspect_ratio_impl(const size2_base<T>& output_dimensions, double aspect)
+	{
+		const double output_aspect = 1. * output_dimensions.width / output_dimensions.height;
+		const double convert_ratio = aspect / output_aspect;
+
+		area_base<T> result;
+		if (convert_ratio > 1.)
+		{
+			const auto height = static_cast<T>(output_dimensions.height / convert_ratio);
+			result.y1 = (output_dimensions.height - height) / 2;
+			result.y2 = result.y1 + height;
+			result.x1 = 0;
+			result.x2 = output_dimensions.width;
+		}
+		else if (convert_ratio < 1.)
+		{
+			const auto width = static_cast<T>(output_dimensions.width * convert_ratio);
+			result.x1 = (output_dimensions.width - width) / 2;
+			result.x2 = result.x1 + width;
+			result.y1 = 0;
+			result.y2 = output_dimensions.height;
+		}
+		else
+		{
+			result = { 0, 0, output_dimensions.width, output_dimensions.height };
+		}
+
+		return result;
+	}
+
 	u32 avconf::get_compatible_gcm_format() const
 	{
 		switch (format)
@@ -135,60 +167,60 @@ namespace rsx
 
 	double avconf::get_aspect_ratio() const
 	{
-		video_aspect v_aspect;
 		switch (aspect)
 		{
-		case CELL_VIDEO_OUT_ASPECT_16_9: v_aspect = video_aspect::_16_9; break;
-		case CELL_VIDEO_OUT_ASPECT_4_3: v_aspect = video_aspect::_4_3; break;
-		case CELL_VIDEO_OUT_ASPECT_AUTO:
-		default: v_aspect = g_cfg.video.aspect_ratio; break;
-		}
-
-		double aspect_ratio;
-		switch (v_aspect)
-		{
-		case video_aspect::_4_3: aspect_ratio = 4.0 / 3.0; break;
-		case video_aspect::_16_9: aspect_ratio = 16.0 / 9.0; break;
-		}
-		return aspect_ratio;
-	}
-
-	void avconf::upscale_to_aspect_ratio(int& width, int& height) const
-	{
-		if (width == 0 || height == 0) return;
-
-		const double old_aspect = 1. * width / height;
-		const double scaling_factor = get_aspect_ratio() / old_aspect;
-
-		if (scaling_factor > 1.0)
-		{
-			width = static_cast<int>(width * scaling_factor);
-		}
-		else if (scaling_factor < 1.0)
-		{
-			height = static_cast<int>(height / scaling_factor);
+		case CELL_VIDEO_OUT_ASPECT_16_9: return 16. / 9.;
+		case CELL_VIDEO_OUT_ASPECT_4_3: return 4. / 3.;
+		default: fmt::throw_exception("Invalid aspect ratio %d", aspect);
 		}
 	}
 
-	void avconf::downscale_to_aspect_ratio(int& x, int& y, int& width, int& height) const
+	size2u avconf::aspect_convert_dimensions(const size2u& image_dimensions) const
 	{
-		if (width == 0 || height == 0) return;
+		if (image_dimensions.width == 0 || image_dimensions.height == 0)
+		{
+			rsx_log.trace("Empty region passed to aspect-correct conversion routine [size]. This should never happen.");
+			return {};
+		}
 
-		const double old_aspect = 1. * width / height;
+		const double old_aspect = 1. * image_dimensions.width / image_dimensions.height;
 		const double scaling_factor = get_aspect_ratio() / old_aspect;
+		size2u result{ image_dimensions.width, image_dimensions.height };
 
 		if (scaling_factor > 1.0)
 		{
-			const int new_height = static_cast<int>(height / scaling_factor);
-			y = (height - new_height) / 2;
-			height = new_height;
+			result.width = static_cast<int>(image_dimensions.width * scaling_factor);
 		}
 		else if (scaling_factor < 1.0)
 		{
-			const int new_width = static_cast<int>(width * scaling_factor);
-			x = (width - new_width) / 2;
-			width = new_width;
+			result.height = static_cast<int>(image_dimensions.height / scaling_factor);
 		}
+
+		return result;
+	}
+
+	areau avconf::aspect_convert_region(const size2u& image_dimensions, const size2u& output_dimensions) const
+	{
+		if (const auto test = image_dimensions * output_dimensions;
+			test.width == 0 || test.height == 0)
+		{
+			rsx_log.trace("Empty region passed to aspect-correct conversion routine [region]. This should never happen.");
+			return {};
+		}
+
+		// Fit the input image into the virtual display 'window'
+		const auto source_aspect = 1. * image_dimensions.width / image_dimensions.height;
+		const auto virtual_output = size2u{ resolution_x, resolution_y };
+		const auto area1 = convert_aspect_ratio_impl(virtual_output, source_aspect);
+
+		// Fit the virtual display into the physical display
+		const auto area2 = convert_aspect_ratio_impl(output_dimensions, get_aspect_ratio());
+
+		// Merge the two regions. Since aspect ratio was conserved between both transforms, a simple scale can be used
+		const double stretch_x = 1. * area2.width() / virtual_output.width;
+		const double stretch_y = 1. * area2.height() / virtual_output.height;
+
+		return static_cast<areau>(static_cast<aread>(area1) * size2d { stretch_x, stretch_y }) + size2u{ area2.x1, area2.y1 };
 	}
 
 #ifdef TEXTURE_CACHE_DEBUG

rpcs3/Emu/RSX/rsx_utils.h

@@ -165,8 +165,8 @@ namespace rsx
 		u8 get_bpp() const;
 		double get_aspect_ratio() const;
 
-		void upscale_to_aspect_ratio(int& width, int& height) const;
-		void downscale_to_aspect_ratio(int& x, int& y, int& width, int& height) const;
+		areau aspect_convert_region(const size2u& image_dimensions, const size2u& output_dimensions) const;
+		size2u aspect_convert_dimensions(const size2u& image_dimensions) const;
 	};
 
 	struct blit_src_info

rpcs3/rpcs3qt/gs_frame.cpp

@@ -623,14 +623,12 @@ void gs_frame::take_screenshot(std::vector<u8> data, const u32 sshot_width, cons
 			QImage img(sshot_data_alpha.data(), sshot_width, sshot_height, sshot_width * 4, QImage::Format_RGBA8888);
 
 			// Scale image if necessary
-			int new_width = img.width();
-			int new_height = img.height();
-			auto& avconf = g_fxo->get<rsx::avconf>();
-			avconf.upscale_to_aspect_ratio(new_width, new_height);
+			const auto& avconf = g_fxo->get<rsx::avconf>();
+			auto new_size = avconf.aspect_convert_dimensions(size2u{ u32(img.width()), u32(img.height()) });
 
-			if (new_width != img.width() || new_height != img.height())
+			if (new_size.width != img.width() || new_size.height != img.height())
 			{
-				img = img.scaled(QSize(new_width, new_height), Qt::AspectRatioMode::IgnoreAspectRatio, Qt::TransformationMode::SmoothTransformation);
+				img = img.scaled(QSize(new_size.width, new_size.height), Qt::AspectRatioMode::IgnoreAspectRatio, Qt::TransformationMode::SmoothTransformation);
 				img.convertTo(QImage::Format_RGBA8888); // The current Qt version changes the image format during smooth scaling, so we have to change it back.
 			}
 
@@ -679,11 +677,9 @@ void gs_frame::take_screenshot(std::vector<u8> data, const u32 sshot_width, cons
 					// We need to scale the overlay if our resolution scaling causes the image to have a different size.
 
 					// Scale the resolution first (as seen before with the image)
-					new_width = avconf.resolution_x;
-					new_height = avconf.resolution_y;
-					avconf.upscale_to_aspect_ratio(new_width, new_height);
+					new_size = avconf.aspect_convert_dimensions(size2u{ avconf.resolution_x, avconf.resolution_y });
 
-					if (new_width != img.width() || new_height != img.height())
+					if (new_size.width != img.width() || new_size.height != img.height())
 					{
 						const int scale = rsx::get_resolution_scale_percent();
 						const int x = (scale * manager.overlay_offset_x) / 100;