diff --git a/.gitmodules b/.gitmodules index 3801c6cc68..109aa44837 100644 --- a/.gitmodules +++ b/.gitmodules @@ -88,3 +88,7 @@ path = 3rdparty/rtmidi/rtmidi url = ../../thestk/rtmidi ignore = dirty +[submodule "3rdparty/zstd/zstd"] + path = 3rdparty/zstd/zstd + url = ../../facebook/zstd + ignore = dirty diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt index 4006a4920c..4c3877c85f 100644 --- a/3rdparty/CMakeLists.txt +++ b/3rdparty/CMakeLists.txt @@ -16,6 +16,9 @@ add_library(3rdparty_dummy_lib INTERFACE) # ZLib add_subdirectory(zlib EXCLUDE_FROM_ALL) +# ZSTD +add_subdirectory(zstd EXCLUDE_FROM_ALL) + # 7z sdk add_subdirectory(7z EXCLUDE_FROM_ALL) @@ -356,6 +359,7 @@ else() add_library(3rdparty::libusb ALIAS usb-1.0-static) endif() add_library(3rdparty::zlib ALIAS 3rdparty_zlib) +add_library(3rdparty::zstd ALIAS 3rdparty_zstd) add_library(3rdparty::7z ALIAS 3rdparty_7z) add_library(3rdparty::flatbuffers ALIAS 3rdparty_flatbuffers) add_library(3rdparty::pugixml ALIAS pugixml) diff --git a/3rdparty/rtmidi/CMakeLists.txt b/3rdparty/rtmidi/CMakeLists.txt index ed291f4044..073f362b58 100644 --- a/3rdparty/rtmidi/CMakeLists.txt +++ b/3rdparty/rtmidi/CMakeLists.txt @@ -1,2 +1,3 @@ option(RTMIDI_API_JACK "Compile with JACK support." OFF) +set(RTMIDI_TARGETNAME_UNINSTALL "uninstall-rpcs3-rtmidi") add_subdirectory(rtmidi EXCLUDE_FROM_ALL) diff --git a/3rdparty/zstd/CMakeLists.txt b/3rdparty/zstd/CMakeLists.txt new file mode 100644 index 0000000000..ad1c3e0b97 --- /dev/null +++ b/3rdparty/zstd/CMakeLists.txt @@ -0,0 +1,9 @@ + +project(3rdparty_zstd) + +add_library(3rdparty_zstd INTERFACE) + +add_subdirectory(zstd/build/cmake EXLUDE_FROM_ALL) + +target_include_directories(3rdparty_zstd INTERFACE zstd zstd/lib) + diff --git a/3rdparty/zstd/zstd b/3rdparty/zstd/zstd new file mode 160000 index 0000000000..97291fc502 --- /dev/null +++ b/3rdparty/zstd/zstd @@ -0,0 +1 @@ +Subproject commit 97291fc5020a8994019ab76cf0cda83a9824374c diff --git a/3rdparty/zstd/zstd.rc b/3rdparty/zstd/zstd.rc new file mode 100644 index 0000000000..5851ae8f33 --- /dev/null +++ b/3rdparty/zstd/zstd.rc @@ -0,0 +1,51 @@ +// Microsoft Visual C++ generated resource script. +// + +#include "zstd\lib\zstd.h" /* ZSTD_VERSION_STRING */ +#define APSTUDIO_READONLY_SYMBOLS +#include "verrsrc.h" +#undef APSTUDIO_READONLY_SYMBOLS + + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE 9, 1 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 + PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Meta Platforms, Inc." + VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" + VALUE "FileVersion", ZSTD_VERSION_STRING + VALUE "InternalName", "zstd.exe" + VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." + VALUE "OriginalFilename", "zstd.exe" + VALUE "ProductName", "Zstandard" + VALUE "ProductVersion", ZSTD_VERSION_STRING + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1200 + END +END + +#endif diff --git a/3rdparty/zstd/zstd.vcxproj b/3rdparty/zstd/zstd.vcxproj new file mode 100644 index 0000000000..a863dc5bc3 --- /dev/null +++ b/3rdparty/zstd/zstd.vcxproj @@ -0,0 +1,193 @@ + + + + + Debug + x64 + + + Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C} + Win32Proj + zstd + $(SolutionDir)lib\$(Configuration)-$(Platform)\ + $(SolutionDir)tmp\$(ProjectName)-$(Configuration)-$(Platform)\ + NotSet + 10.0 + + + + + + Application + true + MultiByte + + + StaticLibrary + false + true + MultiByte + + + + + + + + + + + + + true + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + false + $(LibraryPath); + + + false + $(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); + false + $(LibraryPath); + + + + + + Level4 + Disabled + ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + $(InstructionSet) + + + Console + true + setargv.obj;%(AdditionalDependencies) + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + MultiThreaded + /DZSTD_MULTITHREAD %(AdditionalOptions) + $(InstructionSet) + Default + + + Console + true + true + true + setargv.obj;%(AdditionalDependencies) + + + + + + \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 4aa13efff4..97775039c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,6 +114,8 @@ if(MSVC) add_compile_options(/wd4530 /utf-8) # C++ exception handler used, but unwind semantics are not enabled endif() +set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE) + add_subdirectory(3rdparty) if (DISABLE_LTO) diff --git a/rpcs3.sln b/rpcs3.sln index 2c1e851ce9..7cba3ef052 100644 --- a/rpcs3.sln +++ b/rpcs3.sln @@ -100,6 +100,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "miniupnpc_static", "3rdpart EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "rtmidi", "3rdparty\rtmidi\rtmidi.vcxproj", "{2C902C67-985C-4BE0-94A3-E0FE2EB929A3}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "3rdparty\zstd\zstd.vcxproj", "{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -202,6 +204,10 @@ Global {2C902C67-985C-4BE0-94A3-E0FE2EB929A3}.Debug|x64.Build.0 = Debug|x64 {2C902C67-985C-4BE0-94A3-E0FE2EB929A3}.Release|x64.ActiveCfg = Release|x64 {2C902C67-985C-4BE0-94A3-E0FE2EB929A3}.Release|x64.Build.0 = Release|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.ActiveCfg = Debug|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Debug|x64.Build.0 = Debug|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.ActiveCfg = Release|x64 + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -233,6 +239,7 @@ Global {A55DA1B5-CC17-4525-BE7F-1659CE17BB56} = {6C3B64A0-8F8A-4DC4-8C0B-D71EBEED7FA8} {5228F863-E0DD-4DE7-AA7B-5C52B14CD4D0} = {6C3B64A0-8F8A-4DC4-8C0B-D71EBEED7FA8} {2C902C67-985C-4BE0-94A3-E0FE2EB929A3} = {6C3B64A0-8F8A-4DC4-8C0B-D71EBEED7FA8} + {4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C} = {6C3B64A0-8F8A-4DC4-8C0B-D71EBEED7FA8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {06CC7920-E085-4B81-9582-8DE8AAD42510} diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index c196e4032c..d79a4a34cb 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -771,10 +771,19 @@ bool Emulator::BootRsxCapture(const std::string& path) std::unique_ptr frame = std::make_unique(); utils::serial load; load.set_reading_state(); + + const std::string lower = fmt::to_lower(path); - if (fmt::to_lower(path).ends_with(".gz")) + if (lower.ends_with(".SAVESTAT.gz") || lower.ends_with(".SAVESTAT.zst")) { - load.m_file_handler = make_compressed_serialization_file_handler(std::move(in_file)); + if (lower.ends_with(".SAVESTAT.gz")) + { + load.m_file_handler = make_compressed_serialization_file_handler(std::move(in_file)); + } + else + { + load.m_file_handler = make_compressed_zstd_serialization_file_handler(std::move(in_file)); + } // Forcefully read some data to check validity load.pop(); @@ -956,13 +965,29 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, { fs::file save{m_path, fs::isfile + fs::read}; - if (!m_path.ends_with(".gz") && save && save.size() >= 8 && save.read() == "RPCS3SAV"_u64) + if (m_path.ends_with(".SAVESTAT") && save && save.size() >= 8 && save.read() == "RPCS3SAV"_u64) { m_ar = std::make_shared(); m_ar->set_reading_state(); m_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(save)); } + else if (save && m_path.ends_with(".zst")) + { + m_ar = std::make_shared(); + m_ar->set_reading_state(); + + m_ar->m_file_handler = make_compressed_zstd_serialization_file_handler(std::move(save)); + + if (m_ar->try_read().second != "RPCS3SAV"_u64) + { + m_ar.reset(); + } + else + { + m_ar->pos = 0; + } + } else if (save && m_path.ends_with(".gz")) { m_ar = std::make_shared(); @@ -3138,12 +3163,11 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s path = get_savestate_file(m_title_id, m_path, 0, 0); - // The function is meant for reading files, so if there is no GZ file it would not return compressed file path + // The function is meant for reading files, so if there is no ZST file it would not return compressed file path // So this is the only place where the result is edited if need to be - if (!path.ends_with(".gz")) - { - path += ".gz"; - } + constexpr std::string_view save = ".SAVESTAT"; + path.resize(path.rfind(save) + save.size()); + path += ".zst"; if (!fs::create_path(fs::get_parent_dir(path))) { @@ -3160,7 +3184,7 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s } auto serial_ptr = stx::make_single(); - serial_ptr->m_file_handler = make_compressed_serialization_file_handler(file.file); + serial_ptr->m_file_handler = make_compressed_zstd_serialization_file_handler(file.file); *to_ar = std::move(serial_ptr); signal_system_cache_can_stay(); diff --git a/rpcs3/Emu/savestate_utils.cpp b/rpcs3/Emu/savestate_utils.cpp index cfd37ee2bf..3f2d1976ee 100644 --- a/rpcs3/Emu/savestate_utils.cpp +++ b/rpcs3/Emu/savestate_utils.cpp @@ -126,8 +126,18 @@ std::vector get_savestate_versioning_data(fs::file&& file, std::s utils::serial ar; ar.set_reading_state({}, true); - ar.m_file_handler = filepath.ends_with(".gz") ? static_cast>(make_compressed_serialization_file_handler(std::move(file))) - : make_uncompressed_serialization_file_handler(std::move(file)); + if (filepath.ends_with(".zst")) + { + ar.m_file_handler = make_compressed_zstd_serialization_file_handler(std::move(file)); + } + else if (filepath.ends_with(".gz")) + { + ar.m_file_handler = make_compressed_serialization_file_handler(std::move(file)); + } + else + { + ar.m_file_handler = make_uncompressed_serialization_file_handler(std::move(file)); + } if (u64 r = 0; ar.try_read(r) != 0 || r != "RPCS3SAV"_u64) { @@ -228,6 +238,11 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_ std::string path = fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT"; + if (std::string path_compressed = path + ".zst"; fs::is_file(path_compressed)) + { + return path_compressed; + } + if (std::string path_compressed = path + ".gz"; fs::is_file(path_compressed)) { return path_compressed; @@ -278,7 +293,12 @@ bool boot_last_savestate(bool testing) // Find the latest savestate file compatible with the game (TODO: Check app version and anything more) if (entry.name.find(Emu.GetTitleID()) != umax && mtime <= entry.mtime) { - if (std::string path = save_dir + entry.name + ".gz"; is_savestate_compatible(fs::file(path), path)) + if (std::string path = save_dir + entry.name + ".zst"; is_savestate_compatible(fs::file(path), path)) + { + savestate_path = std::move(path); + mtime = entry.mtime; + } + else if (std::string path = save_dir + entry.name + ".gz"; is_savestate_compatible(fs::file(path), path)) { savestate_path = std::move(path); mtime = entry.mtime; diff --git a/rpcs3/emucore.vcxproj b/rpcs3/emucore.vcxproj index b404f1e546..79e97817cf 100644 --- a/rpcs3/emucore.vcxproj +++ b/rpcs3/emucore.vcxproj @@ -40,10 +40,12 @@ Use - ..\3rdparty\miniupnp\miniupnp\miniupnpc\include;..\3rdparty\wolfssl\wolfssl;..\3rdparty\flatbuffers\include;..\3rdparty\libusb\libusb\libusb;..\3rdparty\yaml-cpp\yaml-cpp\include;..\3rdparty\SoundTouch\soundtouch\include;..\3rdparty\rtmidi\rtmidi;..\3rdparty\zlib\zlib;..\3rdparty\llvm\llvm\include;..\3rdparty\llvm\llvm\llvm\include;..\3rdparty\llvm\llvm_build\include;$(VULKAN_SDK)\Include + ..\3rdparty\miniupnp\miniupnp\miniupnpc\include;..\3rdparty\wolfssl\wolfssl;..\3rdparty\flatbuffers\include;..\3rdparty\libusb\libusb\libusb;..\3rdparty\yaml-cpp\yaml-cpp\include;..\3rdparty\SoundTouch\soundtouch\include;..\3rdparty\rtmidi\rtmidi;..\3rdparty\zlib\zlib;..\3rdparty\llvm\llvm\include;..\3rdparty\llvm\llvm\llvm\include;..\3rdparty\llvm\llvm_build\include;$(VULKAN_SDK)\Include;..\3rdparty\zstd\zstd\lib MaxSpeed MINIUPNP_STATICLIB;HAVE_VULKAN;HAVE_SDL2;ZLIB_CONST;%(PreprocessorDefinitions) MINIUPNP_STATICLIB;HAVE_VULKAN;HAVE_SDL2;ZLIB_CONST;%(PreprocessorDefinitions) + %(AdditionalModuleDependencies) + %(AdditionalModuleDependencies) cmd.exe /c "$(SolutionDir)\Utilities\git-version-gen.cmd" @@ -946,6 +948,12 @@ {fdc361c5-7734-493b-8cfb-037308b35122} + + {60f89955-91c6-3a36-8000-13c592fec2df} + + + {4e52a41a-f33b-4c7a-8c36-a1a6b4f4277c} + diff --git a/rpcs3/rpcs3.vcxproj b/rpcs3/rpcs3.vcxproj index 637ca2c13a..b4d2117adb 100644 --- a/rpcs3/rpcs3.vcxproj +++ b/rpcs3/rpcs3.vcxproj @@ -89,7 +89,7 @@ TurnOffAllWarnings - DbgHelp.lib;Ole32.lib;gdi32.lib;hidapi.lib;libusb-1.0.lib;winmm.lib;miniupnpc_static.lib;rtmidi.lib;imm32.lib;ksuser.lib;version.lib;OpenAL32.lib;XAudio.lib;GLGSRender.lib;shlwapi.lib;VKGSRender.lib;vulkan-1.lib;wolfssl.lib;libcurl.lib;Wldap32.lib;glslang.lib;OSDependent.lib;OGLCompiler.lib;SPIRV.lib;MachineIndependent.lib;GenericCodeGen.lib;Advapi32.lib;user32.lib;zlib.lib;libpng16.lib;asmjit.lib;yaml-cpp.lib;discord-rpc.lib;emucore.lib;dxgi.lib;shell32.lib;Qt6Core.lib;Qt6Gui.lib;Qt6Widgets.lib;Qt6Concurrent.lib;Qt6Multimedia.lib;Qt6MultimediaWidgets.lib;Qt6Svg.lib;Qt6SvgWidgets.lib;7zlib.lib;SPIRV-Tools.lib;SPIRV-Tools-opt.lib;libcubeb.lib;cubeb.lib;soundtouch.lib;Avrt.lib;SDL.lib;%(AdditionalDependencies) + DbgHelp.lib;Ole32.lib;gdi32.lib;hidapi.lib;libusb-1.0.lib;winmm.lib;miniupnpc_static.lib;rtmidi.lib;imm32.lib;ksuser.lib;version.lib;OpenAL32.lib;XAudio.lib;GLGSRender.lib;shlwapi.lib;VKGSRender.lib;vulkan-1.lib;wolfssl.lib;libcurl.lib;Wldap32.lib;glslang.lib;OSDependent.lib;OGLCompiler.lib;SPIRV.lib;MachineIndependent.lib;GenericCodeGen.lib;Advapi32.lib;user32.lib;zlib.lib;zstd.lib;libpng16.lib;asmjit.lib;yaml-cpp.lib;discord-rpc.lib;emucore.lib;dxgi.lib;shell32.lib;Qt6Core.lib;Qt6Gui.lib;Qt6Widgets.lib;Qt6Concurrent.lib;Qt6Multimedia.lib;Qt6MultimediaWidgets.lib;Qt6Svg.lib;Qt6SvgWidgets.lib;7zlib.lib;SPIRV-Tools.lib;SPIRV-Tools-opt.lib;libcubeb.lib;cubeb.lib;soundtouch.lib;Avrt.lib;SDL.lib;%(AdditionalDependencies) ..\3rdparty\OpenAL\libs\Win64;..\3rdparty\glslang\build\hlsl\Release;..\3rdparty\glslang\build\SPIRV\Release;..\3rdparty\glslang\build\OGLCompilersDLL\Release;..\3rdparty\glslang\build\glslang\OSDependent\Windows\Release;..\3rdparty\glslang\build\glslang\Release;..\3rdparty\SPIRV\build\source\Release;..\3rdparty\SPIRV\build\source\opt\Release;..\lib\$(CONFIGURATION)-$(PLATFORM);..\3rdparty\discord-rpc\lib;$(QTDIR)\lib;%(AdditionalLibraryDirectories);$(VULKAN_SDK)\Lib "/MANIFESTDEPENDENCY:type='win32' name='Microsoft.Windows.Common-Controls' version='6.0.0.0' publicKeyToken='6595b64144ccf1df' language='*' processorArchitecture='*'" %(AdditionalOptions) true @@ -140,7 +140,7 @@ $(IntDir)vc$(PlatformToolsetVersion).pdb - DbgHelp.lib;Ole32.lib;gdi32.lib;hidapi.lib;libusb-1.0.lib;winmm.lib;miniupnpc_static.lib;rtmidi.lib;ksuser.lib;OpenAL32.lib;GLGSRender.lib;shlwapi.lib;VKGSRender.lib;vulkan-1.lib;wolfssl.lib;libcurl.lib;Wldap32.lib;glslangd.lib;OSDependentd.lib;OGLCompilerd.lib;SPIRVd.lib;MachineIndependentd.lib;GenericCodeGend.lib;Advapi32.lib;user32.lib;zlib.lib;libpng16.lib;asmjit.lib;yaml-cpp.lib;discord-rpc.lib;emucore.lib;dxgi.lib;shell32.lib;Qt6Cored.lib;Qt6Guid.lib;Qt6Widgetsd.lib;Qt6Concurrentd.lib;Qt6Multimediad.lib;Qt6MultimediaWidgetsd.lib;Qt6Svgd.lib;Qt6SvgWidgetsd.lib;7zlib.lib;SPIRV-Tools.lib;SPIRV-Tools-opt.lib;libcubeb.lib;cubeb.lib;soundtouch.lib;Avrt.lib;XAudio.lib;%(AdditionalDependencies) + DbgHelp.lib;Ole32.lib;gdi32.lib;hidapi.lib;libusb-1.0.lib;winmm.lib;miniupnpc_static.lib;rtmidi.lib;ksuser.lib;OpenAL32.lib;GLGSRender.lib;shlwapi.lib;VKGSRender.lib;vulkan-1.lib;wolfssl.lib;libcurl.lib;Wldap32.lib;glslangd.lib;OSDependentd.lib;OGLCompilerd.lib;SPIRVd.lib;MachineIndependentd.lib;GenericCodeGend.lib;Advapi32.lib;user32.lib;zlib.lib;zstd.lib;libpng16.lib;asmjit.lib;yaml-cpp.lib;discord-rpc.lib;emucore.lib;dxgi.lib;shell32.lib;Qt6Cored.lib;Qt6Guid.lib;Qt6Widgetsd.lib;Qt6Concurrentd.lib;Qt6Multimediad.lib;Qt6MultimediaWidgetsd.lib;Qt6Svgd.lib;Qt6SvgWidgetsd.lib;7zlib.lib;SPIRV-Tools.lib;SPIRV-Tools-opt.lib;libcubeb.lib;cubeb.lib;soundtouch.lib;Avrt.lib;XAudio.lib;%(AdditionalDependencies) ..\3rdparty\OpenAL\libs\Win64;..\3rdparty\glslang\build\hlsl\Debug;..\3rdparty\glslang\build\SPIRV\Debug;..\3rdparty\glslang\build\OGLCompilersDLL\Debug;..\3rdparty\glslang\build\glslang\OSDependent\Windows\Debug;..\3rdparty\glslang\build\glslang\Debug;..\3rdparty\SPIRV\build\source\opt\Debug;..\3rdparty\discord-rpc\lib;..\lib\$(CONFIGURATION)-$(PLATFORM);$(QTDIR)\lib;%(AdditionalLibraryDirectories);$(VULKAN_SDK)\Lib "/MANIFESTDEPENDENCY:type='win32' name='Microsoft.Windows.Common-Controls' version='6.0.0.0' publicKeyToken='6595b64144ccf1df' language='*' processorArchitecture='*'" /VERBOSE %(AdditionalOptions) true diff --git a/rpcs3/rpcs3qt/main_window.cpp b/rpcs3/rpcs3qt/main_window.cpp index 632c683bb5..9bd7acf494 100644 --- a/rpcs3/rpcs3qt/main_window.cpp +++ b/rpcs3/rpcs3qt/main_window.cpp @@ -545,7 +545,7 @@ void main_window::BootElf() "SELF files (EBOOT.BIN *.self);;" "BOOT files (*BOOT.BIN);;" "BIN files (*.bin);;" - "All executable files (*.SAVESTAT.gz *.SAVESTAT *.sprx *.SPRX *.self *.SELF *.bin *.BIN *.prx *.PRX *.elf *.ELF *.o *.O);;" + "All executable files (*.SAVESTAT.zst *.SAVESTAT.gz *.SAVESTAT *.sprx *.SPRX *.self *.SELF *.bin *.BIN *.prx *.PRX *.elf *.ELF *.o *.O);;" "All files (*.*)"), Q_NULLPTR, QFileDialog::DontResolveSymlinks); @@ -619,7 +619,7 @@ void main_window::BootSavestate() } const QString file_path = QFileDialog::getOpenFileName(this, tr("Select Savestate To Boot"), qstr(fs::get_cache_dir() + "savestates/"), tr( - "Savestate files (*.SAVESTAT *.SAVESTAT.gz);;" + "Savestate files (*.SAVESTAT *.SAVESTAT.zst *.SAVESTAT.gz);;" "All files (*.*)"), Q_NULLPTR, QFileDialog::DontResolveSymlinks); @@ -3746,7 +3746,7 @@ main_window::drop_type main_window::IsValidFile(const QMimeData& md, QStringList type = drop_type::drop_rrc; } // The emulator allows to execute ANY filetype, just not from drag-and-drop because it is confusing to users - else if (path.toLower().endsWith(".savestat.gz") || suffix_lo == "savestat" || suffix_lo == "sprx" || suffix_lo == "self" || suffix_lo == "bin" || suffix_lo == "prx" || suffix_lo == "elf" || suffix_lo == "o") + else if (path.toLower().endsWith(".savestat.gz") || path.toLower().endsWith(".savestat.zst") || suffix_lo == "savestat" || suffix_lo == "sprx" || suffix_lo == "self" || suffix_lo == "bin" || suffix_lo == "prx" || suffix_lo == "elf" || suffix_lo == "o") { type = drop_type::drop_game; } diff --git a/rpcs3/util/serialization_ext.cpp b/rpcs3/util/serialization_ext.cpp index c5cbdabebc..7d529a9225 100644 --- a/rpcs3/util/serialization_ext.cpp +++ b/rpcs3/util/serialization_ext.cpp @@ -10,6 +10,7 @@ #include "serialization_ext.hpp" #include +#include <3rdparty/zstd/zstd/lib/zstd.h> LOG_CHANNEL(sys_log, "SYS"); @@ -38,7 +39,7 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, { m_file->seek(pos); m_file->write(data, size); - return true; + return true; } m_file->seek(ar.data_offset); @@ -218,8 +219,8 @@ void compressed_serialization_file_handler::initialize(utils::serial& ar) if (!ar.expect_little_data()) { - m_stream_data_prepare_thread = std::make_unique>>("Compressed Data Prepare Thread"sv, [this]() { this->stream_data_prepare_thread_op(); }); - m_file_writer_thread = std::make_unique>>("Compressed File Writer Thread"sv, [this]() { this->file_writer_thread_op(); }); + m_stream_data_prepare_thread = std::make_unique>>("CompressedPrepare Thread"sv, [this]() { this->stream_data_prepare_thread_op(); }); + m_file_writer_thread = std::make_unique>>("CompressedWriter Thread"sv, [this]() { this->file_writer_thread_op(); }); } } else @@ -274,6 +275,12 @@ bool compressed_serialization_file_handler::handle_file_op(utils::serial& ar, us if (ar.data.empty()) { + if (pos == umax && size == umax && *m_file) + { + // Request to flush the file to disk + m_file->sync(); + } + return true; } @@ -601,8 +608,11 @@ void compressed_serialization_file_handler::finalize(utils::serial& ar) m_stream_data = {}; ensure(deflateEnd(&m_zs) == Z_OK); + m_write_inited = false; ar.data = {}; // Deallocate and clear + + m_file->sync(); } void compressed_serialization_file_handler::stream_data_prepare_thread_op() @@ -770,6 +780,551 @@ usz compressed_serialization_file_handler::get_size(const utils::serial& ar, usz return std::max(utils::mul_saturate(m_file->size(), 6), memory_available); } +struct compressed_zstd_stream_data +{ + ZSTD_DCtx* m_zd{}; + ZSTD_DStream* m_zs{}; + lf_queue> m_queued_data_to_process; + lf_queue> m_queued_data_to_write; +}; + +void compressed_zstd_serialization_file_handler::initialize(utils::serial& ar) +{ + if (!m_stream) + { + m_stream = std::make_shared(); + } + + if (ar.is_writing()) + { + if (m_write_inited) + { + return; + } + + if (m_read_inited && m_stream->m_zd) + { + finalize(ar); + } + + m_write_inited = true; + m_errored = false; + + m_compression_threads.clear(); + m_file_writer_thread.reset(); + + // Make sure at least one thread is free + // Limit thread count in order to make sure memory limits are under control (TODO: scale with RAM size) + const usz thread_count = std::min(std::max(utils::get_thread_count(), 2) - 1, 16); + + for (usz i = 0; i < thread_count; i++) + { + m_compression_threads.emplace_back().m_thread = std::make_unique>>(fmt::format("CompressedPrepare Thread %d", i + 1), [this]() { this->stream_data_prepare_thread_op(); }); + } + + m_file_writer_thread = std::make_unique>>("CompressedWriter Thread"sv, [this]() { this->file_writer_thread_op(); }); + } + else + { + if (m_read_inited) + { + return; + } + + if (m_write_inited) + { + finalize(ar); + } + + auto& m_zd = m_stream->m_zd; + m_zd = ZSTD_createDCtx(); + m_stream->m_zs = ZSTD_createDStream(); + m_read_inited = true; + m_errored = false; + } +} + +bool compressed_zstd_serialization_file_handler::handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) +{ + if (ar.is_writing()) + { + initialize(ar); + + if (m_errored) + { + return false; + } + + auto& manager = *m_stream; + + if (data) + { + ensure(false); + } + + // Writing not at the end is forbidden + ensure(ar.pos == ar.data_offset + ar.data.size()); + + if (ar.data.empty()) + { + if (pos == umax && size == umax && *m_file) + { + // Request to flush the file to disk + m_file->sync(); + } + + return true; + } + + ar.seek_end(); + + const usz buffer_idx = m_input_buffer_index++ % m_compression_threads.size(); + auto& input = m_compression_threads[buffer_idx].m_input; + + while (input) + { + // No waiting support on non-null pointer + thread_ctrl::wait_for(2'000); + } + + input.store(stx::make_single_value(std::move(ar.data))); + input.notify_all(); + + ar.data_offset = ar.pos; + ar.data.clear(); + + if (pos == umax && size == umax && *m_file) + { + // Request to flush the file to disk + m_file->sync(); + } + + return true; + } + + initialize(ar); + + if (m_errored) + { + return false; + } + + if (!size) + { + return true; + } + + if (pos == 0 && size == umax) + { + // Discard loaded data until pos if profitable + const usz limit = ar.data_offset + ar.data.size(); + + if (ar.pos > ar.data_offset && ar.pos < limit) + { + const usz may_discard_bytes = ar.pos - ar.data_offset; + const usz moved_byte_count_on_discard = limit - ar.pos; + + // Cheeck profitability (check recycled memory and std::memmove costs) + if (may_discard_bytes >= 0x50'0000 || (may_discard_bytes >= 0x20'0000 && moved_byte_count_on_discard / may_discard_bytes < 3)) + { + ar.data_offset += may_discard_bytes; + ar.data.erase(ar.data.begin(), ar.data.begin() + may_discard_bytes); + + if (ar.data.capacity() >= 0x200'0000) + { + // Discard memory + ar.data.shrink_to_fit(); + } + } + + return true; + } + + // Discard all loaded data + ar.data_offset += ar.data.size(); + ensure(ar.pos >= ar.data_offset); + ar.data.clear(); + + if (ar.data.capacity() >= 0x200'0000) + { + // Discard memory + ar.data.shrink_to_fit(); + } + + return true; + } + + if (~pos < size - 1) + { + // Overflow + return false; + } + + // TODO: Investigate if this optimization is worth an implementation for compressed stream + // if (ar.data.empty() && pos != ar.pos) + // { + // // Relocate instead of over-fetch + // ar.seek_pos(pos); + // } + + const usz read_pre_buffer = utils::sub_saturate(ar.data_offset, pos); + + if (read_pre_buffer) + { + // Not allowed with compressed data for now + // Unless someone implements mechanism for it + ensure(false); + } + + // Adjustment to prevent overflow + const usz subtrahend = ar.data.empty() ? 0 : 1; + const usz read_past_buffer = utils::sub_saturate(pos + (size - subtrahend), ar.data_offset + (ar.data.size() - subtrahend)); + const usz read_limit = utils::sub_saturate(ar.m_max_data, ar.data_offset); + + if (read_past_buffer) + { + // Read proceeding data + // More lightweight operation, this is the common operation + // Allowed to fail, if memory is truly needed an assert would take place later + const usz old_size = ar.data.size(); + + // Try to prefetch data by reading more than requested + const usz new_size = std::min(read_limit, std::max({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.expect_little_data() ? usz{4096} : usz{0x10'0000} })); + + if (new_size < old_size) + { + // Read limit forbids further reads at this point + return true; + } + + ar.data.resize(new_size); + ar.data.resize(this->read_at(ar, old_size + ar.data_offset, data ? const_cast(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size); + } + + return true; +} + +usz compressed_zstd_serialization_file_handler::read_at(utils::serial& ar, usz read_pos, void* data, usz size) +{ + ensure(read_pos == ar.data.size() + ar.data_offset - size); + + if (!size || m_errored) + { + return 0; + } + + initialize(ar); + + auto& m_zd = m_stream->m_zd; + + const usz total_to_read = size; + usz read_size = 0; + u8* out_data = static_cast(data); + + for (; read_size < total_to_read;) + { + // Drain extracted memory stash (also before first file read) + out_data = static_cast(data) + read_size; + + auto avail_in = m_stream_data.size() - m_stream_data_index; + auto next_in = reinterpret_cast(m_stream_data.data() + m_stream_data_index); + auto next_out = out_data; + auto avail_out = size - read_size; + + for (bool is_first = true; read_size < total_to_read && avail_in; is_first = false) + { + bool need_more_file_memory = false; + + ZSTD_outBuffer outs{next_out, avail_out, 0}; + + // Try to extract previously held data first + ZSTD_inBuffer ins{next_in, is_first ? 0 : avail_in, 0}; + + const usz res = ZSTD_decompressStream(m_zd, &outs, &ins); + + if (ZSTD_isError(res)) + { + need_more_file_memory = true; + // finalize(ar); + // m_errored = true; + // sys_log.error("Failure of compressed data reading. (res=%d, read_size=0x%x, avail_in=0x%x, avail_out=0x%x, ar=%s)", res, read_size, avail_in, avail_out, ar); + // return read_size; + } + + read_size += outs.pos; + next_out += outs.pos; + avail_out -= outs.pos; + next_in += ins.pos; + avail_in -= ins.pos; + + m_stream_data_index = next_in - m_stream_data.data(); + + if (need_more_file_memory) + { + break; + } + } + + if (read_size >= total_to_read) + { + break; + } + + const usz add_size = ar.expect_little_data() ? 0x1'0000 : 0x10'0000; + const usz old_file_buf_size = m_stream_data.size(); + + m_stream_data.resize(old_file_buf_size + add_size); + m_stream_data.resize(old_file_buf_size + m_file->read_at(m_file_read_index, m_stream_data.data() + old_file_buf_size, add_size)); + + if (m_stream_data.size() == old_file_buf_size) + { + // EOF + //ensure(read_size == total_to_read); + break; + } + + m_file_read_index += m_stream_data.size() - old_file_buf_size; + } + + if (m_stream_data.size() - m_stream_data_index <= m_stream_data_index / 5) + { + // Shrink to required memory size + m_stream_data.erase(m_stream_data.begin(), m_stream_data.begin() + m_stream_data_index); + + if (m_stream_data.capacity() >= 0x200'0000) + { + // Discard memory + m_stream_data.shrink_to_fit(); + } + + m_stream_data_index = 0; + } + + return read_size; +} + +void compressed_zstd_serialization_file_handler::skip_until(utils::serial& ar) +{ + ensure(!ar.is_writing() && ar.pos >= ar.data_offset); + + if (ar.pos > ar.data_offset) + { + handle_file_op(ar, ar.data_offset, ar.pos - ar.data_offset, nullptr); + } +} + +void compressed_zstd_serialization_file_handler::finalize(utils::serial& ar) +{ + handle_file_op(ar, 0, umax, nullptr); + + if (!m_stream) + { + return; + } + + auto& m_zd = m_stream->m_zd; + + if (m_read_inited) + { + //ZSTD_decompressEnd(m_stream->m_zd); + ensure(ZSTD_freeDCtx(m_zd)); + m_read_inited = false; + return; + } + + const stx::shared_ptr> empty_data = stx::make_single>(); + const stx::shared_ptr> null_ptr = stx::null_ptr; + + for (auto& context : m_compression_threads) + { + // Try to notify all on the first iteration + if (context.m_input.compare_and_swap_test(null_ptr, empty_data)) + { + context.notified = true; + context.m_input.notify_one(); + } + } + + for (auto& context : m_compression_threads) + { + // Notify to abort + while (!context.notified) + { + const auto data = context.m_input.compare_and_swap(null_ptr, empty_data); + + if (!data) + { + context.notified = true; + context.m_input.notify_one(); + break; + } + + // Wait until valid input is processed + thread_ctrl::wait_for(1000); + } + } + + for (auto& context : m_compression_threads) + { + // Wait for notification to be consumed + while (context.m_input) + { + thread_ctrl::wait_for(1000); + } + } + + for (auto& context : m_compression_threads) + { + // Wait for data to be writen to be read by the thread + while (context.m_output) + { + thread_ctrl::wait_for(1000); + } + } + + for (usz idx = m_output_buffer_index;;) + { + auto& out_cur = m_compression_threads[idx % m_compression_threads.size()].m_output; + auto& out_next = m_compression_threads[(idx + 1) % m_compression_threads.size()].m_output; + + out_cur.compare_and_swap_test(null_ptr, empty_data); + out_next.compare_and_swap_test(null_ptr, empty_data); + + if (usz new_val = m_output_buffer_index; idx != new_val) + { + // Index was changed inbetween, retry on the next index + idx = new_val; + continue; + } + + // Must be waiting on either of the two + out_cur.notify_all(); + + // Check for single thread + if (&out_next != &out_cur) + { + out_next.notify_all(); + } + + break; + } + + if (m_file_writer_thread) + { + // Join here to avoid log messages in the destructor + (*m_file_writer_thread)(); + } + + m_compression_threads.clear(); + m_file_writer_thread.reset(); + + m_stream_data = {}; + m_write_inited = false; + ar.data = {}; // Deallocate and clear + + m_file->sync(); +} + +void compressed_zstd_serialization_file_handler::stream_data_prepare_thread_op() +{ + ZSTD_CCtx* m_zc = ZSTD_createCCtx(); + + std::vector stream_data; + + const stx::shared_ptr> null_ptr = stx::null_ptr; + const usz thread_index = m_thread_buffer_index++; + + while (true) + { + auto& input = m_compression_threads[thread_index].m_input; + auto& output = m_compression_threads[thread_index].m_output; + + while (!input) + { + input.wait(nullptr); + } + + auto data = input.exchange(stx::null_ptr); + input.notify_all(); + + if (data->empty()) + { + // Abort is requested, flush data and exit + break; + } + + usz buffer_offset = 0; + stream_data.resize(::ZSTD_compressBound(data->size())); + const usz out_size = ZSTD_compressCCtx(m_zc, stream_data.data(), stream_data.size(), data->data(), data->size(), ZSTD_btultra); + + ensure(!ZSTD_isError(out_size) && out_size); + + if (m_errored) + { + break; + } + + stream_data.resize(out_size); + + const stx::shared_ptr> data_ptr = make_single>(std::move(stream_data)); + + while (output || !output.compare_and_swap_test(null_ptr, data_ptr)) + { + thread_ctrl::wait_for(1000); + } + + //if (m_output_buffer_index % m_compression_threads.size() == thread_index) + { + output.notify_all(); + } + } + + ZSTD_freeCCtx(m_zc); +} + +void compressed_zstd_serialization_file_handler::file_writer_thread_op() +{ + for (m_output_buffer_index = 0;; m_output_buffer_index++) + { + auto& output = m_compression_threads[m_output_buffer_index % m_compression_threads.size()].m_output; + + while (!output) + { + output.wait(nullptr); + } + + auto data = output.exchange(stx::null_ptr); + output.notify_all(); + + if (data->empty()) + { + break; + } + + const usz last_size = data->size(); + m_file->write(*data); + } +} + +usz compressed_zstd_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const +{ + if (ar.is_writing()) + { + return *m_file ? m_file->size() : 0; + } + + const usz memory_available = ar.data_offset + ar.data.size(); + + if (memory_available >= recommended || !*m_file) + { + // Avoid calling size() if possible + return memory_available; + } + + return recommended; + //return std::max(utils::mul_saturate(ZSTD_decompressBound(m_file->size()), 2), memory_available); +} + bool null_serialization_file_handler::handle_file_op(utils::serial&, usz, usz, const void*) { return true; diff --git a/rpcs3/util/serialization_ext.hpp b/rpcs3/util/serialization_ext.hpp index bfa3c37cd6..75b6212890 100644 --- a/rpcs3/util/serialization_ext.hpp +++ b/rpcs3/util/serialization_ext.hpp @@ -1,9 +1,12 @@ #pragma once #include "util/serialization.hpp" +#include "util/shared_ptr.hpp" #include "Utilities/Thread.h" +#include + namespace fs { class file; @@ -115,6 +118,85 @@ inline std::unique_ptr make_compressed_se return std::make_unique(std::forward(file)); } +struct compressed_zstd_stream_data; + +// Compressed file serialization handler +struct compressed_zstd_serialization_file_handler : utils::serialization_file_handler +{ + explicit compressed_zstd_serialization_file_handler(fs::file&& file) noexcept + : utils::serialization_file_handler() + , m_file_storage(std::make_unique(std::move(file))) + , m_file(m_file_storage.get()) + { + } + + explicit compressed_zstd_serialization_file_handler(const fs::file& file) noexcept + : utils::serialization_file_handler() + , m_file_storage(nullptr) + , m_file(std::addressof(file)) + { + } + + compressed_zstd_serialization_file_handler(const compressed_zstd_serialization_file_handler&) = delete; + compressed_zstd_serialization_file_handler& operator=(const compressed_zstd_serialization_file_handler&) = delete; + + // Handle file read and write requests + bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override; + + // Get available memory or file size + // Preferably memory size if is already greater/equal to recommended to avoid additional file ops + usz get_size(const utils::serial& ar, usz recommended) const override; + void skip_until(utils::serial& ar) override; + + bool is_valid() const override + { + return !m_errored; + } + + void finalize(utils::serial& ar) override; + +private: + const std::unique_ptr m_file_storage; + const std::add_pointer_t m_file; + std::vector m_stream_data; + usz m_stream_data_index = 0; + usz m_file_read_index = 0; + atomic_t m_pending_bytes = 0; + atomic_t m_pending_signal = false; + bool m_write_inited = false; + bool m_read_inited = false; + atomic_t m_errored = false; + + usz m_input_buffer_index = 0; + atomic_t m_output_buffer_index = 0; + atomic_t m_thread_buffer_index = 0; + + struct compression_thread_context_t + { + atomic_ptr> m_input; + atomic_ptr> m_output; + bool notified = false; + std::unique_ptr>> m_thread; + }; + + std::deque m_compression_threads; + std::shared_ptr m_stream; + std::unique_ptr>> m_file_writer_thread; + + usz read_at(utils::serial& ar, usz read_pos, void* data, usz size); + void initialize(utils::serial& ar); + void stream_data_prepare_thread_op(); + void file_writer_thread_op(); + void blocked_compressed_write(const std::vector& data); +}; + +template requires (std::is_same_v, fs::file>) +inline std::unique_ptr make_compressed_zstd_serialization_file_handler(File&& file) +{ + ensure(file); + return std::make_unique(std::forward(file)); +} + // Null file serialization handler struct null_serialization_file_handler : utils::serialization_file_handler { diff --git a/rpcs3/util/shared_ptr.hpp b/rpcs3/util/shared_ptr.hpp index 138599d107..70bac278e3 100644 --- a/rpcs3/util/shared_ptr.hpp +++ b/rpcs3/util/shared_ptr.hpp @@ -867,7 +867,7 @@ namespace stx if (exch.m_ptr) { - exch.d().refs += c_ref_mask; + exch.d()->refs += c_ref_mask; } atomic_ptr old; @@ -903,7 +903,7 @@ namespace stx old_exch.m_val.raw() = reinterpret_cast(std::exchange(exch.m_ptr, nullptr)) << c_ref_size; // Set to reset old cmp_and_old value - old.m_val.raw() = (cmp_and_old.m_ptr << c_ref_size) | c_ref_mask; + old.m_val.raw() = (reinterpret_cast(cmp_and_old.m_ptr) << c_ref_size) | c_ref_mask; if (!_val) { @@ -953,7 +953,7 @@ namespace stx if (exch.m_ptr) { - exch.d().refs += c_ref_mask; + exch.d()->refs += c_ref_mask; } atomic_ptr old;