Savestates: Implement initial RAM ventilation system

Utilities/File.cpp

@@ -2255,8 +2255,22 @@ fs::file fs::make_gather(std::vector<fs::file> files)
 	return result;
 }
 
-fs::pending_file::pending_file(std::string_view path)
+bool fs::pending_file::open(std::string_view path)
 {
+	file.close();
+
+	if (!m_path.empty())
+	{
+		fs::remove_file(m_path);
+	}
+
+	if (path.empty())
+	{
+		fs::g_tls_error = fs::error::noent;
+		m_dest.clear();
+		return false;
+	}
+
 	do
 	{
 		m_path = fmt::format(u8"%s/＄%s.%s.tmp", get_parent_dir(path), path.substr(path.find_last_of(fs::delim) + 1), fmt::base57(utils::get_unique_tsc()));
@@ -2270,6 +2284,8 @@ fs::pending_file::pending_file(std::string_view path)
 		m_path.clear();
 	}
 	while (fs::g_tls_error == fs::error::exist); // Only retry if failed due to existing file
+
+	return file.operator bool();
 }
 
 fs::pending_file::~pending_file()
@@ -2284,7 +2300,7 @@ fs::pending_file::~pending_file()
 
 bool fs::pending_file::commit(bool overwrite)
 {
-	if (!file || m_path.empty())
+	if (m_path.empty())
 	{
 		fs::g_tls_error = fs::error::noent;
 		return false;
@@ -2292,7 +2308,11 @@ bool fs::pending_file::commit(bool overwrite)
 
 	// The temporary file's contents must be on disk before rename
 #ifndef _WIN32
-	file.sync();
+	if (file)
+	{
+		file.sync();
+	}
+
 #endif
 	file.close();
 

Utilities/File.h

@@ -674,12 +674,24 @@ namespace fs
 
 		// This is meant to modify files atomically, overwriting is likely
 		bool commit(bool overwrite = true);
+		bool open(std::string_view path);
+
+		pending_file() noexcept = default;
+
+		pending_file(std::string_view path) noexcept
+		{
+			open(path);
+		}
 
-		pending_file(std::string_view path);
 		pending_file(const pending_file&) = delete;
 		pending_file& operator=(const pending_file&) = delete;
 		~pending_file();
 
+		const std::string& get_temp_path() const
+		{
+			return m_path;
+		}
+
 	private:
 		std::string m_path{}; // Pending file path
 		std::string m_dest{}; // Destination file path

rpcs3/Emu/Cell/PPUThread.cpp

@@ -3493,13 +3493,26 @@ namespace
 	// Read-only file view starting with specified offset (for MSELF)
 	struct file_view : fs::file_base
 	{
-		const fs::file m_file;
+		const fs::file m_storage;
+		const fs::file& m_file;
 		const u64 m_off;
+		const u64 m_max_size;
 		u64 m_pos;
 
-		explicit file_view(fs::file&& _file, u64 offset)
+		explicit file_view(const fs::file& _file, u64 offset, u64 max_size) noexcept
-			: m_file(std::move(_file))
+			: m_storage(fs::file())
+			, m_file(_file)
 			, m_off(offset)
+			, m_max_size(max_size)
+			, m_pos(0)
+		{
+		}
+
+		explicit file_view(fs::file&& _file, u64 offset, u64 max_size) noexcept
+			: m_storage(std::move(_file))
+			, m_file(m_storage)
+			, m_off(offset)
+			, m_max_size(max_size)
 			, m_pos(0)
 		{
 		}
@@ -3520,18 +3533,14 @@ namespace
 
 		u64 read(void* buffer, u64 size) override
 		{
-			const u64 old_pos = m_file.pos();
+			const u64 result = file_view::read_at(m_pos, buffer, size);
-			m_file.seek(m_off + m_pos);
-			const u64 result = m_file.read(buffer, size);
-			ensure(old_pos == m_file.seek(old_pos));
-
 			m_pos += result;
 			return result;
 		}
 
 		u64 read_at(u64 offset, void* buffer, u64 size) override
 		{
-			return m_file.read_at(offset + m_off, buffer, size);
+			return m_file.read_at(m_off + m_pos, buffer, std::min<u64>(size, utils::sub_saturate<u64>(m_max_size, m_pos)));
 		}
 
 		u64 write(const void*, u64) override
@@ -3563,10 +3572,17 @@ namespace
 	};
 }
 
-extern fs::file make_file_view(fs::file&& _file, u64 offset)
+extern fs::file make_file_view(const fs::file& _file, u64 offset, u64 max_size = umax)
 {
 	fs::file file;
-	file.reset(std::make_unique<file_view>(std::move(_file), offset));
+	file.reset(std::make_unique<file_view>(_file, offset, max_size));
+	return file;
+}
+
+extern fs::file make_file_view(fs::file&& _file, u64 offset, u64 max_size = umax)
+{
+	fs::file file;
+	file.reset(std::make_unique<file_view>(std::move(_file), offset, max_size));
 	return file;
 }
 
@@ -3835,7 +3851,7 @@ extern void ppu_precompile(std::vector<std::string>& dir_queue, std::vector<ppu_
 			if (u64 off = offset)
 			{
 				// Adjust offset for MSELF
-				src.reset(std::make_unique<file_view>(std::move(src), off));
+				src = make_file_view(std::move(src), offset);
 
 				// Adjust path for MSELF too
 				fmt::append(path, "_x%x", off);

rpcs3/Emu/Cell/lv2/sys_overlay.cpp

@@ -68,7 +68,7 @@ static error_code overlay_load_module(vm::ptr<u32> ovlmid, const std::string& vp
 	return CELL_OK;
 }
 
-fs::file make_file_view(fs::file&&, u64);
+fs::file make_file_view(fs::file&& file, u64 offset, u64 size);
 
 std::shared_ptr<void> lv2_overlay::load(utils::serial& ar)
 {
@@ -82,7 +82,7 @@ std::shared_ptr<void> lv2_overlay::load(utils::serial& ar)
 	if (file)
 	{
 		u128 klic = g_fxo->get<loaded_npdrm_keys>().last_key();
-		file = make_file_view(std::move(file), offset);
+		file = make_file_view(std::move(file), offset, umax);
 		ovlm = ppu_load_overlay(ppu_exec_object{ decrypt_self(std::move(file), reinterpret_cast<u8*>(&klic)) }, false, path, 0, &ar).first;
 		ensure(ovlm);
 	}

rpcs3/Emu/Cell/lv2/sys_prx.cpp

@@ -286,7 +286,7 @@ static error_code prx_load_module(const std::string& vpath, u64 flags, vm::ptr<s
 	return not_an_error(idm::last_id());
 }
 
-fs::file make_file_view(fs::file&& _file, u64 offset);
+fs::file make_file_view(fs::file&& file, u64 offset, u64 size);
 
 std::shared_ptr<void> lv2_prx::load(utils::serial& ar)
 {
@@ -319,7 +319,7 @@ std::shared_ptr<void> lv2_prx::load(utils::serial& ar)
 		if (file)
 		{
 			u128 klic = g_fxo->get<loaded_npdrm_keys>().last_key();
-			file = make_file_view(std::move(file), offset);
+			file = make_file_view(std::move(file), offset, umax);
 			prx = ppu_load_prx(ppu_prx_object{ decrypt_self(std::move(file), reinterpret_cast<u8*>(&klic)) }, false, path, 0, &ar);
 			prx->m_loaded_flags = std::move(loaded_flags);
 			prx->m_external_loaded_flags = std::move(external_flags);

rpcs3/Emu/System.cpp

@@ -40,6 +40,7 @@
 #include "../Crypto/unself.h"
 #include "util/logs.hpp"
 #include "util/serialization.hpp"
+#include "savestate_utils.hpp"
 
 #include <fstream>
 #include <memory>
@@ -87,6 +88,8 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_
 
 extern void send_close_home_menu_cmds();
 
+fs::file make_file_view(const fs::file& file, u64 offset, u64 size);
+
 fs::file g_tty;
 atomic_t<s64> g_tty_size{0};
 std::array<std::deque<std::string>, 16> g_tty_input;
@@ -711,9 +714,8 @@ bool Emulator::BootRsxCapture(const std::string& path)
 
 	std::unique_ptr<rsx::frame_capture_data> frame = std::make_unique<rsx::frame_capture_data>();
 	utils::serial load;
+	load.m_file_handler = make_uncompressed_serialization_file_handler(std::move(in_file));
 	load.set_reading_state();
-	in_file.read(load.data, in_file.size());
-	load.data.shrink_to_fit();
 
 	load(*frame);
 	in_file.close();
@@ -883,9 +885,8 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 			{
 				m_ar = std::make_shared<utils::serial>();
 				m_ar->set_reading_state();
-				save.seek(0);
+
-				save.read(m_ar->data, save.size());
+				m_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(save));
-				m_ar->data.shrink_to_fit();
 			}
 
 			m_boot_source_type = CELL_GAME_GAMETYPE_SYS;
@@ -946,22 +947,29 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 				return game_boot_result::savestate_corrupted;
 			}
 
-			if (header.LE_format != (std::endian::native == std::endian::little) || header.offset >= m_ar->data.size())
+			if (header.LE_format != (std::endian::native == std::endian::little) || header.offset >= m_ar->get_size(header.offset))
 			{
 				return game_boot_result::savestate_corrupted;
 			}
 
 			g_cfg.savestate.state_inspection_mode.set(header.state_inspection_support);
 
-			// Emulate seek operation (please avoid using in other places)
-			m_ar->pos = header.offset;
-
-			if (!is_savestate_version_compatible(m_ar->operator std::vector<std::pair<u16, u16>>(), true))
 			{
-				return game_boot_result::savestate_version_unsupported;
+				// Read data on another container to keep the existing data
-			}
+				utils::serial ar_temp;
+				ar_temp.set_reading_state();
+				ar_temp.swap_handler(*m_ar);
+				ar_temp.seek_pos(header.offset);
+				ar_temp.m_avoid_large_prefetch = true;
 
-			m_ar->pos = sizeof(file_header); // Restore position
+				if (!is_savestate_version_compatible(ar_temp.operator std::vector<std::pair<u16, u16>>(), true))
+				{
+					return game_boot_result::savestate_version_unsupported;
+				}
+
+				// Restore file handler
+				ar_temp.swap_handler(*m_ar);
+			}
 
 			argv.clear();
 			klic.clear();
@@ -1006,8 +1014,9 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 				if (size)
 				{
 					fs::remove_all(path, false);
-					ensure(tar_object(fs::file(&m_ar->data[m_ar->pos], size)).extract(path));
+					m_ar->breathe(true);
-					m_ar->pos += size;
+					ensure(tar_object(make_file_view(*static_cast<uncompressed_serialization_file_handler*>(m_ar->m_file_handler.get())->m_file, m_ar->data_offset, size)).extract(path));
+					m_ar->seek_pos(m_ar->pos + size, size >= 4096);
 				}
 			};
 
@@ -2831,10 +2840,34 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 
 		std::unique_ptr<utils::serial> to_ar;
 
+		fs::pending_file file;
+		std::string path;
+
+		while (savestate)
+		{
+			path = get_savestate_file(m_title_id, m_path, 0, 0);
+
+			if (!fs::create_path(fs::get_parent_dir(path)))
+			{
+				sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error);
+				savestate = false;
+				break;
+			}
+
+			if (!file.open(path))
+			{
+				sys_log.error("Failed to create savestate temporary file! (path='%s', %s)", file.get_temp_path(), fs::g_tls_error);
+				savestate = false;
+				break;
+			}
+
+			to_ar = std::make_unique<utils::serial>();
+			to_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(file.file));
+			break;
+		}
+
 		if (savestate)
 		{
-			to_ar = std::make_unique<utils::serial>();
-
 			// Savestate thread
 			named_thread emu_state_cap_thread("Emu State Capture Thread", [&]()
 			{
@@ -2851,12 +2884,33 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 				// Avoid duplicating TAR object memory because it can be very large
 				auto save_tar = [&](const std::string& path)
 				{
+					const usz old_data_start = ar.data_offset;
+					const usz old_pos = ar.seek_end();
+
+					ar.breathe();
+
 					ar(usz{}); // Reserve memory to be patched later with correct size
-					const usz old_size = ar.data.size();
+					tar_object::save_directory(path, ar);
-					ar.data = tar_object::save_directory(path, std::move(ar.data));
+
-					ar.seek_end();
+					const usz new_pos = ar.seek_end();
-					const usz tar_size = ar.data.size() - old_size;
+					const usz tar_size = new_pos - old_pos - sizeof(usz);
-					std::memcpy(ar.data.data() + old_size - sizeof(usz), &tar_size, sizeof(usz));
+
+					// Check if breathe() actually did something, in this case memory needs to be discarded
+					const bool was_emptied = old_data_start != ar.data_offset;
+
+					if (was_emptied)
+					{
+						ensure(ar.data_offset > old_data_start);
+
+						// Write to file directly (slower)
+						ar.m_file_handler->handle_file_op(ar, old_pos, sizeof(tar_size), &tar_size);
+					}
+					else
+					{
+						// If noty written to file, simply write to memory
+						std::memcpy(ar.data.data() + old_pos - old_data_start, &tar_size, sizeof(usz));
+					}
+
 					sys_log.success("Saved the contents of directory '%s' (size=0x%x)", path, tar_size);
 				};
 
@@ -2900,8 +2954,8 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 				ar("RPCS3SAV"_u64);
 				ar(std::endian::native == std::endian::little);
 				ar(g_cfg.savestate.state_inspection_mode.get());
-				ar(std::array<u8, 32>{}); // Reserved for future use
 				ar(usz{0}); // Offset of versioning data, to be overwritten at the end of saving
+				ar(std::array<u8, 32>{}); // Reserved for future use
 
 				if (auto dir = vfs::get("/dev_bdvd/PS3_GAME"); fs::is_dir(dir) && !fs::is_file(fs::get_parent_dir(dir) + "/PS3_DISC.SFB"))
 				{
@@ -2953,24 +3007,29 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 
 		if (savestate)
 		{
-			const std::string path = get_savestate_file(m_title_id, m_path, 0, 0);
-
-			if (!fs::create_path(fs::get_parent_dir(path)))
-			{
-				sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error);
-			}
-
-			fs::pending_file file(path);
-
 			// Identifer -> version
 			std::vector<std::pair<u16, u16>> used_serial = read_used_savestate_versions();
 
 			auto& ar = *to_ar;
+
 			const usz pos = ar.seek_end();
-			std::memcpy(&ar.data[10], &pos, 8);// Set offset
+
+			// Patch offset with a direct write
+			ar.m_file_handler->handle_file_op(ar, 10, sizeof(usz), &pos);
+
+			// Write the version data at the end
 			ar(used_serial);
 
-			if (!file.file || (file.file.write(ar.data), !file.commit()))
+			// Final file write, the file is ready to be committed
+			ar.breathe(true);
+
+#ifndef _WIN32
+			// The temporary file's contents must be on disk before rename
+			// Flush to file
+			ar.m_file_handler->handle_file_op(ar, umax, umax, nullptr);
+#endif
+
+			if (!file.commit())
 			{
 				sys_log.error("Failed to write savestate to file! (path='%s', %s)", path, fs::g_tls_error);
 				savestate = false;

rpcs3/Emu/savestate_utils.cpp

@@ -1,9 +1,11 @@
 #include "stdafx.h"
 #include "util/types.hpp"
-#include "util/serialization.hpp"
 #include "util/logs.hpp"
+#include "util/asm.hpp"
 #include "Utilities/File.h"
+#include "Utilities/StrFmt.h"
 #include "system_config.h"
+#include "savestate_utils.hpp"
 
 #include "System.h"
 
@@ -11,6 +13,14 @@
 
 LOG_CHANNEL(sys_log, "SYS");
 
+template <>
+void fmt_class_string<utils::serial>::format(std::string& out, u64 arg)
+{
+	const utils::serial& ar = get_object(arg);
+
+	fmt::append(out, "{ %s, 0x%x/0%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size());
+}
+
 struct serial_ver_t
 {
 	bool used = false;
@@ -79,7 +89,7 @@ SERIALIZATION_VER(sys_io, 23,                                   1)
 SERIALIZATION_VER(LLE, 24,                                      1)
 SERIALIZATION_VER(HLE, 25,                                      1)
 
-std::vector<std::pair<u16, u16>> get_savestate_versioning_data(const fs::file& file)
+std::vector<std::pair<u16, u16>> get_savestate_versioning_data(fs::file&& file)
 {
 	if (!file)
 	{
@@ -105,11 +115,10 @@ std::vector<std::pair<u16, u16>> get_savestate_versioning_data(const fs::file& f
 		return {};
 	}
 
-	file.seek(offs);
-
 	utils::serial ar;
 	ar.set_reading_state();
-    file.read(ar.data, fsize - offs);
+	ar.m_file_handler = make_uncompressed_serialization_file_handler(std::move(file));
+	ar.seek_pos(offs);
 	return ar;
 }
 
@@ -172,9 +181,9 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_
 	return fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT";
 }
 
-bool is_savestate_compatible(const fs::file& file)
+bool is_savestate_compatible(fs::file&& file)
 {
-	return is_savestate_version_compatible(get_savestate_versioning_data(file), false);
+	return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file)), false);
 }
 
 std::vector<std::pair<u16, u16>> read_used_savestate_versions()
@@ -199,7 +208,7 @@ bool boot_last_savestate(bool testing)
 {
 	if (!g_cfg.savestate.suspend_emu && !Emu.GetTitleID().empty() && (Emu.IsRunning() || Emu.GetStatus() == system_state::paused))
 	{
-		extern bool is_savestate_compatible(const fs::file& file);
+		extern bool is_savestate_compatible(fs::file&& file);
 
 		const std::string save_dir = fs::get_cache_dir() + "/savestates/";
 
@@ -252,3 +261,149 @@ bool boot_last_savestate(bool testing)
 
 	return false;
 }
+
+bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, usz pos, usz size, const void* data)
+{
+	if (ar.is_writing())
+	{
+		if (data)
+		{
+			m_file->seek(pos);
+			m_file->write(data, size);
+			return  true;
+		}
+
+		m_file->seek(ar.data_offset);
+		m_file->write(ar.data);
+
+		if (pos == umax && size == umax)
+		{
+			// Request to flush the file to disk
+			m_file->sync();
+		}
+
+		ar.data_offset += ar.data.size();
+		ar.data.clear();
+		ar.seek_end();
+		return true;
+	}
+
+	if (!size)
+	{
+		return true;
+	}
+
+	if (pos == 0 && size == umax)
+	{
+		// Discard loaded data until pos if profitable
+		const usz limit = ar.data_offset + ar.data.size();
+
+		if (ar.pos > ar.data_offset && ar.pos < limit)
+		{
+			const usz may_discard_bytes = ar.pos - ar.data_offset;
+			const usz moved_byte_count_on_discard = limit - ar.pos;
+
+			// Cheeck profitability (check recycled memory and std::memmove costs)
+			if (may_discard_bytes >= 0x50'0000 || (may_discard_bytes >= 0x20'0000 && moved_byte_count_on_discard / may_discard_bytes < 3))
+			{
+				ar.data_offset += may_discard_bytes;
+				ar.data.erase(ar.data.begin(), ar.data.begin() + may_discard_bytes);
+
+				if (ar.data.capacity() >= 0x200'0000)
+				{
+					// Discard memory
+					ar.data.shrink_to_fit();
+				}
+			}
+
+			return true;
+		}
+
+		// Discard all loaded data
+		ar.data_offset += ar.data.size();
+		ar.data.clear();
+
+		if (ar.data.capacity() >= 0x200'0000)
+		{
+			// Discard memory
+			ar.data.shrink_to_fit();
+		}
+
+		return true;
+	}
+
+	if (~size < pos)
+	{
+		// Overflow
+		return false;
+	}
+
+	if (ar.data.empty() && pos != ar.pos)
+	{
+		// Relocate instead oof over-fetch
+		ar.seek_pos(pos);
+	}
+
+	const usz read_pre_buffer = utils::sub_saturate<usz>(ar.data_offset, pos);
+
+	if (read_pre_buffer)
+	{
+		// Read past data
+		// Harsh operation on performance, luckily rare and not typically needed
+		// Also this may would be disallowed when moving to compressed files
+		// This may be a result of wrong usage of breathe() function
+		ar.data.resize(ar.data.size() + read_pre_buffer);
+		std::memmove(ar.data.data() + read_pre_buffer, ar.data.data(), ar.data.size() - read_pre_buffer);
+		ensure(m_file->read_at(pos, ar.data.data(), read_pre_buffer) == read_pre_buffer);
+		ar.data_offset -= read_pre_buffer;
+	}
+
+	const usz read_past_buffer = utils::sub_saturate<usz>(pos + size, ar.data_offset + ar.data.size());
+
+	if (read_past_buffer)
+	{
+		// Read proceeding data
+		// More lightweight operation, this is the common operation
+		// Allowed to fail, if memory is truly needed an assert would take place later
+		const usz old_size = ar.data.size();
+
+		// Try to prefetch data by reading more than requested
+		ar.data.resize(std::max<usz>({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} }));
+		ar.data.resize(m_file->read_at(old_size + ar.data_offset, data ? const_cast<void*>(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size);
+	}
+
+	return true;
+}
+
+
+usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const
+{
+	if (ar.is_writing())
+	{
+		return m_file->size();
+	}
+
+	const usz memory_available = ar.data_offset + ar.data.size();
+
+	if (memory_available >= recommended)
+	{
+		// Avoid calling size() if possible
+		return memory_available;
+	}
+
+	return std::max<usz>(m_file->size(), memory_available);
+}
+
+namespace stx
+{
+	extern void serial_breathe(utils::serial& ar)
+	{
+		ar.breathe();
+	}
+}
+
+// MSVC bug workaround, see above similar case
+extern void serial_breathe(utils::serial& ar)
+{
+	::stx::serial_breathe(ar);
+}

rpcs3/Emu/savestate_utils.hpp

@@ -0,0 +1,45 @@
+#include "util/serialization.hpp"
+
+namespace fs
+{
+	class file;
+}
+
+// Uncompressed file serialization handler
+struct uncompressed_serialization_file_handler : utils::serialization_file_handler
+{
+	const std::unique_ptr<fs::file> m_file_storage;
+	const std::add_pointer_t<const fs::file> m_file;
+
+	explicit uncompressed_serialization_file_handler(fs::file&& file) noexcept
+		: utils::serialization_file_handler()
+		, m_file_storage(std::make_unique<fs::file>(std::move(file)))
+		, m_file(m_file_storage.get())
+	{
+	}
+
+	explicit uncompressed_serialization_file_handler(const fs::file& file) noexcept
+		: utils::serialization_file_handler()
+		, m_file_storage(nullptr)
+		, m_file(std::addressof(file))
+	{
+	}
+
+	// Handle file read and write requests
+	bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override;
+
+	// Get available memory or file size
+	// Preferably memory size if is already greater/equal to recommended to avoid additional file ops
+	usz get_size(const utils::serial& ar, usz recommended) const override;
+};
+
+inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(fs::file&& file)
+{
+	return std::make_unique<uncompressed_serialization_file_handler>(std::move(file));
+}
+
+inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(const fs::file& file)
+{
+	return std::make_unique<uncompressed_serialization_file_handler>(file);
+}
+

rpcs3/Loader/TAR.cpp

@@ -8,6 +8,7 @@
 #include "TAR.h"
 
 #include "util/asm.hpp"
+#include "util/serialization.hpp"
 
 #include <charconv>
 
@@ -258,14 +259,14 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs)
 	return true;
 }
 
-std::vector<u8> tar_object::save_directory(const std::string& src_dir, std::vector<u8>&& init, const process_func& func, std::string full_path)
+void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func, std::string full_path)
 {
 	const std::string& target_path = full_path.empty() ? src_dir : full_path;
 
 	fs::stat_t stat{};
 	if (!fs::get_stat(target_path, stat))
 	{
-		return std::move(init);
+		return;
 	}
 
 	if (stat.is_directory)
@@ -276,13 +277,13 @@ std::vector<u8> tar_object::save_directory(const std::string& src_dir, std::vect
 		{
 			if (entry.name.find_first_not_of('.') == umax) continue;
 
-			init = save_directory(src_dir, std::move(init), func, target_path + '/' + entry.name);
+			save_directory(src_dir, ar, func, target_path + '/' + entry.name);
 			has_items = true;
 		}
 
 		if (has_items)
 		{
-			return std::move(init);
+			return;
 		}
 	}
 
@@ -304,34 +305,34 @@ std::vector<u8> tar_object::save_directory(const std::string& src_dir, std::vect
 
 	std::string saved_path{target_path.data() + src_dir.size(), target_path.size() - src_dir.size()};
 
-	const u64 old_size = init.size();
+	const u64 old_size = ar.data.size();
-	init.resize(old_size + sizeof(TARHeader));
+	ar.data.resize(old_size + sizeof(TARHeader));
 
 	if (!stat.is_directory)
 	{
 		fs::file fd(target_path);
 
-		const u64 old_size2 = init.size();
+		const u64 old_size2 = ar.data.size();
 
 		if (func)
 		{
 			// Use custom function for file saving if provided
 			// Allows for example to compress PNG files as JPEG in the TAR itself
-			if (!func(fd, saved_path, std::move(init)))
+			if (!func(fd, saved_path, ar))
 			{
 				// Revert (this entry should not be included if func returns false)
-				init.resize(old_size);
+				ar.data.resize(old_size);
-				return std::move(init);
+				return;
 			}
 		}
 		else
 		{
-			init.resize(init.size() + stat.size);
+			ar.data.resize(ar.data.size() + stat.size);
-			ensure(fd.read(init.data() + old_size2, stat.size) == stat.size);
+			ensure(fd.read(ar.data.data() + old_size2, stat.size) == stat.size);
 		}
 
 		// Align
-		init.resize(old_size2 + utils::align(init.size() - old_size2, 512));
+		ar.data.resize(old_size2 + utils::align(ar.data.size() - old_size2, 512));
 
 		fd.close();
 		fs::utime(target_path, stat.atime, stat.mtime);
@@ -352,8 +353,10 @@ std::vector<u8> tar_object::save_directory(const std::string& src_dir, std::vect
 	write_octal(header.padding, stat.atime);
 	header.filetype = stat.is_directory ? '5' : '0';
 
-	std::memcpy(init.data() + old_size, &header, sizeof(header));
+	std::memcpy(ar.data.data() + old_size, &header, sizeof(header));
-	return std::move(init);
+
+	// TAR is an old format which does not depend on previous data so memory ventilation is trivial here 
+	ar.breathe();
 }
 
 bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file)

rpcs3/Loader/TAR.h

@@ -22,6 +22,11 @@ namespace fs
 	class file;
 }
 
+namespace utils
+{
+	struct serial;
+}
+
 class tar_object
 {
 	const fs::file& m_file;
@@ -38,13 +43,13 @@ public:
 
 	fs::file get_file(const std::string& path);
 
-	using process_func = std::function<bool(const fs::file&, std::string&, std::vector<u8>&&)>;
+	using process_func = std::function<bool(const fs::file&, std::string&, utils::serial&)>;
 
 	// Extract all files in archive to destination (as VFS if is_vfs is true)
 	// Allow to optionally specify explicit mount point (which may be directory meant for extraction)
 	bool extract(std::string prefix_path = {}, bool is_vfs = false);
 
-	static std::vector<u8> save_directory(const std::string& src_dir, std::vector<u8>&& init = std::vector<u8>{}, const process_func& func = {}, std::string append_path = {});
+	static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::string append_path = {});
 };
 
 bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file = {});

rpcs3/emucore.vcxproj

@@ -607,6 +607,7 @@
     <ClInclude Include="Emu\RSX\Overlays\Shaders\shader_loading_dialog_native.h" />
     <ClInclude Include="Emu\RSX\RSXDisAsm.h" />
     <ClInclude Include="Emu\RSX\RSXZCULL.h" />
+    <ClInclude Include="Emu\savestate_utils.hpp" />
     <ClInclude Include="Emu\system_progress.hpp" />
     <ClInclude Include="Emu\system_utils.hpp" />
     <ClInclude Include="Emu\title.h" />

rpcs3/emucore.vcxproj.filters

@@ -2389,6 +2389,9 @@
     <ClInclude Include="Crypto\unzip.h">
       <Filter>Crypto</Filter>
     </ClInclude>
+    <ClInclude Include="Emu\savestate_utils.hpp">
+      <Filter>Emu</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="Emu\RSX\Program\GLSLSnippets\GPUDeswizzle.glsl">

rpcs3/rpcs3qt/game_list_frame.cpp

@@ -1078,7 +1078,7 @@ void game_list_frame::ShowContextMenu(const QPoint &pos)
 		});
 	}
 
-	extern bool is_savestate_compatible(const fs::file& file);
+	extern bool is_savestate_compatible(fs::file&& file);
 
 	if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate)))
 	{

rpcs3/util/fixed_typemap.hpp

@@ -251,6 +251,12 @@ namespace stx
 					*m_order++ = data;
 					*m_info++ = &type;
 					m_init[id] = true;
+
+					if (ar)
+					{
+						extern void serial_breathe(utils::serial& ar);
+						serial_breathe(*ar);
+					}
 				}
 			}
 
@@ -310,7 +316,8 @@ namespace stx
 			}
 		}
 
-		void save(utils::serial& ar)
+		template <typename T> requires (std::is_same_v<T&, utils::serial&>)
+		void save(T& ar)
 		{
 			if (!is_init())
 			{
@@ -332,7 +339,11 @@ namespace stx
 			// Save data in forward order
 			for (u32 i = _max; i; i--)
 			{
-				if (auto save = (*std::prev(m_info, i))->save) save(*std::prev(m_order, i), ar);
+				if (auto save = (*std::prev(m_info, i))->save)
+				{
+					save(*std::prev(m_order, i), ar);
+					ar.breathe();
+				}
 			}
 		}
 

rpcs3/util/serialization.hpp

@@ -8,13 +8,13 @@ namespace utils
 	template <typename T>
 	concept FastRandomAccess = requires (T& obj)
 	{
-		std::data(obj)[0];
+		std::data(obj)[std::size(obj)];
 	};
 
 	template <typename T>
 	concept Reservable = requires (T& obj)
 	{
-		obj.reserve(0);
+		obj.reserve(std::size(obj));
 	};
 
 	template <typename T>
@@ -32,15 +32,29 @@ namespace utils
 	template <typename T>
 	concept ListAlike = requires (T& obj) { obj.insert(obj.end(), std::declval<typename T::value_type>()); };
 
+	struct serial;
+
+	struct serialization_file_handler
+	{
+		serialization_file_handler() = default;
+		virtual ~serialization_file_handler() = default;
+
+		virtual bool handle_file_op(serial& ar, usz pos, usz size, const void* data = nullptr) = 0;
+		virtual usz get_size(const utils::serial& ar, usz recommended = umax) const = 0;
+	};
+
 	struct serial
 	{
 		std::vector<u8> data;
+		usz data_offset = 0;
 		usz pos = 0;
 		bool m_is_writing = true;
+		bool m_avoid_large_prefetch = false;
+		std::unique_ptr<serialization_file_handler> m_file_handler;
 
-		serial() = default;
+		serial() noexcept = default;
 		serial(const serial&) = delete;
-		~serial() = default;
+		~serial() noexcept = default;
 
 		// Checks if this instance is currently used for serialization
 		bool is_writing() const
@@ -58,21 +72,44 @@ namespace utils
 			}
 		}
 
-		bool raw_serialize(const void* ptr, usz size)
+		template <typename Func> requires (std::is_convertible_v<std::invoke_result_t<Func>, const void*>)
+		bool raw_serialize(Func&& memory_provider, usz size)
 		{
+			if (!size)
+			{
+				return true;
+			}
+
+			// Overflow check
+			ensure(~pos >= size);
+
 			if (is_writing())
 			{
-				data.insert(data.begin() + pos, static_cast<const u8*>(ptr), static_cast<const u8*>(ptr) + size);
+				ensure(pos >= data_offset);
+				const auto ptr = reinterpret_cast<const u8*>(memory_provider());
+				data.insert(data.begin() + pos - data_offset, ptr, ptr + size);
 				pos += size;
 				return true;
 			}
 
-			ensure(data.size() - pos >= size);
+			if (data.empty() || pos < data_offset || pos + size > data.size() + data_offset)
-			std::memcpy(const_cast<void*>(ptr), data.data() + pos, size);
+			{
+				// Load from file
+				ensure(m_file_handler);
+				ensure(m_file_handler->handle_file_op(*this, pos, size, nullptr));
+				ensure(!data.empty() && pos >= data_offset && pos + size <= data.size() + data_offset);
+			}
+
+			std::memcpy(const_cast<void*>(static_cast<const void*>(memory_provider())), data.data() + pos - data_offset, size);
 			pos += size;
 			return true;
 		}
 
+		bool raw_serialize(const void* ptr, usz size)
+		{
+			return raw_serialize(FN(ptr), size);
+		}
+
 		template <typename T> requires Integral<T>
 		bool serialize_vle(T&& value)
 		{
@@ -157,19 +194,15 @@ namespace utils
 				return true;
 			}
 
-			obj.clear();
-
 			usz size = 0;
 			if (!deserialize_vle(size))
 			{
 				return false;
 			}
 
-			obj.resize(size);
-
 			if constexpr (Bitcopy<typename T::value_type>)
 			{
-				if (!raw_serialize(obj.data(), sizeof(obj[0]) * size))
+				if (!raw_serialize([&](){ obj.resize(size); return obj.data(); }, sizeof(obj[0]) * size))
 				{
 					obj.clear();
 					return false;
@@ -177,6 +210,10 @@ namespace utils
 			}
 			else
 			{
+				// TODO: Postpone resizing to after file bounds checks
+				obj.clear();
+				obj.resize(size);
+
 				for (auto&& value : obj)
 				{
 					if (!serialize(value))
@@ -306,7 +343,9 @@ namespace utils
 			}
 
 			m_is_writing = false;
+			m_avoid_large_prefetch = false;
 			pos = 0;
+			data_offset = 0;
 		}
 
 		// Reset to empty serialization manager
@@ -315,23 +354,53 @@ namespace utils
 			data.clear();
 			m_is_writing = true;
 			pos = 0;
+			data_offset = 0;
+			m_file_handler.reset();
 		}
 
 		usz seek_end(usz backwards = 0)
 		{
-			ensure(pos >= backwards);
+			ensure(data.size() + data_offset >= backwards);
-			pos = data.size() - backwards;
+			pos = data.size() + data_offset - backwards;
 			return pos;
 		}
 
+		usz seek_pos(usz val, bool empty_data = false)
+		{
+			const usz old_pos = std::exchange(pos, val);
+
+			if (empty_data || data.empty())
+			{
+				// Relocate future data
+				data.clear();
+				data_offset = pos;
+			}
+
+			return old_pos;
+		}
+
 		usz pad_from_end(usz forwards)
 		{
 			ensure(is_writing());
-			pos = data.size();
+			pos = data.size() + data_offset;
-			data.resize(pos + forwards);
+			data.resize(data.size() + forwards);
 			return pos;
 		}
 
+		// Allow for memory saving operations: if writing, flush to file if too large. If reading, discard memory (not implemented).
+		// Execute only if past memory is known to not going be reused
+		void breathe(bool forced = false)
+		{
+			if (!forced && (!m_file_handler || (data.size() < 0x20'0000 && pos >= data_offset)))
+			{
+				// Let's not do anything if less than 32MB
+				return;
+			}
+
+			ensure(m_file_handler);
+			ensure(m_file_handler->handle_file_op(*this, 0, umax, nullptr));
+		}
+
 		template <typename T> requires (std::is_copy_constructible_v<std::remove_const_t<T>>) && (std::is_constructible_v<std::remove_const_t<T>> || Bitcopy<std::remove_const_t<T>> ||
 			std::is_constructible_v<std::remove_const_t<T>, stx::exact_t<serial&>> || TupleAlike<std::remove_const_t<T>>)
 		operator T()
@@ -364,6 +433,16 @@ namespace utils
 			}
 		}
 
+		void swap_handler(serial& ar)
+		{
+			std::swap(ar.m_file_handler, this->m_file_handler);
+		}
+
+		usz get_size(usz recommended = umax) const
+		{
+			return m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size();
+		}
+
 		template <typename T> requires (std::is_copy_constructible_v<T> && std::is_constructible_v<T> && Bitcopy<T>)
 		std::pair<bool, T> try_read()
 		{
@@ -372,10 +451,11 @@ namespace utils
 				return {};
 			}
 
-			const usz left = data.size() - pos;
+			const usz end_pos = pos + sizeof(T);
+			const usz size = get_size(end_pos);
 	 		using type = std::remove_const_t<T>;
 
-			if (left >= sizeof(type))
+			if (size >= end_pos)
 			{
 				u8 buf[sizeof(type)]{};
 				ensure(raw_serialize(buf, sizeof(buf)));
@@ -389,7 +469,8 @@ namespace utils
 		// Used when an invalid state is encountered somewhere in a place we can't check success code such as constructor)
 		bool is_valid() const
 		{
-			return pos <= data.size();
+			// TODO
+			return true;
 		}
 	};
 }