From f60bdbaece594e5334943f287ebeeaf566e6bc30 Mon Sep 17 00:00:00 2001 From: Eladash <18193363+elad335@users.noreply.github.com> Date: Wed, 15 Nov 2023 21:07:42 +0200 Subject: [PATCH] Savestates: Compressed state files --- Utilities/File.cpp | 14 + rpcs3/Crypto/unzip.h | 16 +- rpcs3/Emu/Cell/Modules/sceNpTrophy.cpp | 4 +- rpcs3/Emu/Cell/Modules/sys_io_.cpp | 3 + rpcs3/Emu/Cell/PPUModule.cpp | 15 +- rpcs3/Emu/Cell/PPUThread.cpp | 31 +- rpcs3/Emu/Cell/SPUThread.cpp | 8 +- rpcs3/Emu/Cell/lv2/sys_event.cpp | 2 +- rpcs3/Emu/Cell/lv2/sys_fs.cpp | 2 +- rpcs3/Emu/Cell/lv2/sys_lwcond.cpp | 4 +- rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp | 4 +- rpcs3/Emu/Cell/lv2/sys_mmapper.cpp | 4 +- rpcs3/Emu/Cell/lv2/sys_overlay.cpp | 2 +- rpcs3/Emu/Cell/lv2/sys_prx.cpp | 2 +- rpcs3/Emu/Cell/lv2/sys_spu.cpp | 12 +- rpcs3/Emu/Cell/lv2/sys_usbd.cpp | 2 +- rpcs3/Emu/IdManager.h | 6 +- rpcs3/Emu/Memory/vm.cpp | 81 ++-- rpcs3/Emu/RSX/RSXThread.cpp | 51 ++- rpcs3/Emu/System.cpp | 152 ++++--- rpcs3/Emu/savestate_utils.cpp | 496 ++++++++++++++++++++-- rpcs3/Emu/savestate_utils.hpp | 101 ++++- rpcs3/Loader/TAR.cpp | 547 ++++++++++++++++++------- rpcs3/Loader/TAR.h | 14 +- rpcs3/rpcs3qt/game_list_frame.cpp | 4 +- rpcs3/rpcs3qt/main_window.cpp | 18 +- rpcs3/util/serialization.hpp | 152 ++++++- 27 files changed, 1377 insertions(+), 370 deletions(-) diff --git a/Utilities/File.cpp b/Utilities/File.cpp index c47c1c503e..2e21d0e9fd 100644 --- a/Utilities/File.cpp +++ b/Utilities/File.cpp @@ -2277,6 +2277,12 @@ bool fs::pending_file::open(std::string_view path) if (file.open(m_path, fs::create + fs::write + fs::read + fs::excl)) { +#ifdef _WIN32 + // Auto-delete pending log file + FILE_DISPOSITION_INFO disp; + disp.DeleteFileW = true; + SetFileInformationByHandle(file.get_handle(), FileDispositionInfo, &disp, sizeof(disp)); +#endif m_dest = path; break; } @@ -2314,6 +2320,14 @@ bool fs::pending_file::commit(bool overwrite) } #endif + +#ifdef _WIN32 + // Disable auto-delete + FILE_DISPOSITION_INFO disp; + disp.DeleteFileW = false; + SetFileInformationByHandle(file.get_handle(), FileDispositionInfo, &disp, sizeof(disp)); +#endif + file.close(); #ifdef _WIN32 diff --git a/rpcs3/Crypto/unzip.h b/rpcs3/Crypto/unzip.h index dc7701da3e..80025ce65c 100644 --- a/rpcs3/Crypto/unzip.h +++ b/rpcs3/Crypto/unzip.h @@ -2,32 +2,32 @@ std::vector unzip(const void* src, usz size); -template -std::vector unzip(const T& src) +template +inline std::vector unzip(const T& src) { return unzip(src.data(), src.size()); } bool unzip(const void* src, usz size, fs::file& out); -template -bool unzip(const std::vector& src, fs::file& out) +template +inline bool unzip(const std::vector& src, fs::file& out) { return unzip(src.data(), src.size(), out); } std::vector zip(const void* src, usz size); -template -std::vector zip(const T& src) +template +inline std::vector zip(const T& src) { return zip(src.data(), src.size()); } bool zip(const void* src, usz size, fs::file& out); -template -bool zip(const T& src, fs::file& out) +template +inline bool zip(const T& src, fs::file& out) { return zip(src.data(), src.size(), out); } diff --git a/rpcs3/Emu/Cell/Modules/sceNpTrophy.cpp b/rpcs3/Emu/Cell/Modules/sceNpTrophy.cpp index 8852df6955..7494ea49af 100644 --- a/rpcs3/Emu/Cell/Modules/sceNpTrophy.cpp +++ b/rpcs3/Emu/Cell/Modules/sceNpTrophy.cpp @@ -43,7 +43,7 @@ struct trophy_context_t trophy_context_t() = default; trophy_context_t(utils::serial& ar) - : trp_name(ar.operator std::string()) + : trp_name(ar.pop()) { std::string trophy_path = vfs::get(Emu.GetDir() + "TROPDIR/" + trp_name + "/TROPHY.TRP"); fs::file trp_stream(trophy_path); @@ -55,7 +55,7 @@ struct trophy_context_t trp_stream.open(trophy_path); } - if (!ar.operator bool()) + if (!ar.pop()) { ar(read_only); return; diff --git a/rpcs3/Emu/Cell/Modules/sys_io_.cpp b/rpcs3/Emu/Cell/Modules/sys_io_.cpp index fd69113e43..ec43e5bc43 100644 --- a/rpcs3/Emu/Cell/Modules/sys_io_.cpp +++ b/rpcs3/Emu/Cell/Modules/sys_io_.cpp @@ -52,6 +52,7 @@ void config_event_entry(ppu_thread& ppu) { if (ppu.is_stopped()) { + ppu.state += cpu_flag::again; return; } @@ -73,6 +74,8 @@ void config_event_entry(ppu_thread& ppu) } } + sys_io.notice("config_event_entry(): Exited with the following error code: %s", CellError{static_cast(ppu.gpr[3])}); + ppu_execute<&sys_ppu_thread_exit>(ppu, 0); } diff --git a/rpcs3/Emu/Cell/PPUModule.cpp b/rpcs3/Emu/Cell/PPUModule.cpp index adc4c23dc1..7d01594c0e 100644 --- a/rpcs3/Emu/Cell/PPUModule.cpp +++ b/rpcs3/Emu/Cell/PPUModule.cpp @@ -358,7 +358,7 @@ static void ppu_initialize_modules(ppu_linkage_info* link, utils::serial* ar = n while (true) { - const std::string name = ar.operator std::string(); + const std::string name = ar.pop(); if (name.empty()) { @@ -370,10 +370,10 @@ static void ppu_initialize_modules(ppu_linkage_info* link, utils::serial* ar = n auto& variable = _module->variables; - for (u32 i = 0, end = ar.operator usz(); i < end; i++) + for (u32 i = 0, end = ar.pop(); i < end; i++) { - auto* ptr = &::at32(variable, ar.operator u32()); - ptr->addr = ar.operator u32(); + auto* ptr = &::at32(variable, ar.pop()); + ptr->addr = ar.pop(); ensure(!!ptr->var); } } @@ -1052,7 +1052,12 @@ void init_ppu_functions(utils::serial* ar, bool full = false) if (ar) { - ensure(vm::check_addr(g_fxo->init(*ar)->addr)); + const u32 addr = g_fxo->init(*ar)->addr; + + if (addr % 0x1000 || !vm::check_addr(addr)) + { + fmt::throw_exception("init_ppu_functions(): Failure to initialize function manager. (addr=0x%x, %s)", addr, *ar); + } } else g_fxo->init(); diff --git a/rpcs3/Emu/Cell/PPUThread.cpp b/rpcs3/Emu/Cell/PPUThread.cpp index 26cf2812b7..b44c281175 100644 --- a/rpcs3/Emu/Cell/PPUThread.cpp +++ b/rpcs3/Emu/Cell/PPUThread.cpp @@ -2352,6 +2352,16 @@ void ppu_thread::serialize_common(utils::serial& ar) ar(gpr, fpr, cr, fpscr.bits, lr, ctr, vrsave, cia, xer, sat, nj, prio.raw().all); + if (cia % 4 || !vm::check_addr(cia)) + { + fmt::throw_exception("Failed to serialize PPU thread ID=0x%x (cia=0x%x, ar=%s)", this->id, cia, ar); + } + + if (ar.is_writing()) + { + ppu_log.notice("Saving PPU Thread [0x%x: %s]: cia=0x%x, state=%s", id, *ppu_tname.load(), cia, +state); + } + ar(optional_savestate_state, vr); if (optional_savestate_state->data.empty()) @@ -2364,7 +2374,7 @@ ppu_thread::ppu_thread(utils::serial& ar) : cpu_thread(idm::last_id()) // last_id() is showed to constructor on serialization , stack_size(ar) , stack_addr(ar) - , joiner(ar.operator ppu_join_status()) + , joiner(ar.pop()) , entry_func(std::bit_cast(ar)) , is_interrupt_thread(ar) { @@ -2397,7 +2407,7 @@ ppu_thread::ppu_thread(utils::serial& ar) } }; - switch (const u32 status = ar.operator u32()) + switch (const u32 status = ar.pop()) { case PPU_THREAD_STATUS_IDLE: { @@ -2490,7 +2500,9 @@ ppu_thread::ppu_thread(utils::serial& ar) state += cpu_flag::memory; } - ppu_tname = make_single(ar.operator std::string()); + ppu_tname = make_single(ar.pop()); + + ppu_log.notice("Loading PPU Thread [0x%x: %s]: cia=0x%x, state=%s", id, *ppu_tname.load(), cia, +state); } void ppu_thread::save(utils::serial& ar) @@ -2506,12 +2518,6 @@ void ppu_thread::save(utils::serial& ar) _joiner = ppu_join_status::joinable; } - if (state & cpu_flag::again) - { - std::memcpy(&gpr[3], syscall_args, sizeof(syscall_args)); - cia -= 4; - } - ar(stack_size, stack_addr, _joiner, entry, is_interrupt_thread); serialize_common(ar); @@ -2685,6 +2691,13 @@ void ppu_thread::fast_call(u32 addr, u64 rtoc, bool is_thread_entry) std::memcpy(syscall_args, &gpr[3], sizeof(syscall_args)); } + if (!old_cia && state & cpu_flag::again) + { + // Fixup argument registers and CIA for reloading + std::memcpy(&gpr[3], syscall_args, sizeof(syscall_args)); + cia -= 4; + } + current_function = old_func; g_tls_log_prefix = old_fmt; state -= cpu_flag::ret; diff --git a/rpcs3/Emu/Cell/SPUThread.cpp b/rpcs3/Emu/Cell/SPUThread.cpp index db97696d69..097695ecfd 100644 --- a/rpcs3/Emu/Cell/SPUThread.cpp +++ b/rpcs3/Emu/Cell/SPUThread.cpp @@ -1981,7 +1981,7 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group) : cpu_thread(idm::last_id()) , group(group) , index(ar) - , thread_type(group ? spu_type::threaded : ar.operator u8() ? spu_type::isolated : spu_type::raw) + , thread_type(group ? spu_type::threaded : ar.pop() ? spu_type::isolated : spu_type::raw) , shm(ensure(vm::get(vm::spu)->peek(vm_offset()).second)) , ls(map_ls(*this->shm)) , option(ar) @@ -2029,12 +2029,12 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group) for (auto& pair : spuq) { ar(pair.first); - pair.second = idm::get_unlocked(ar.operator u32()); + pair.second = idm::get_unlocked(ar.pop()); } for (auto& q : spup) { - q = idm::get_unlocked(ar.operator u32()); + q = idm::get_unlocked(ar.pop()); } } else @@ -2042,7 +2042,7 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group) for (spu_int_ctrl_t& ctrl : int_ctrl) { ar(ctrl.mask, ctrl.stat); - ctrl.tag = idm::get_unlocked(ar.operator u32()); + ctrl.tag = idm::get_unlocked(ar.pop()); } g_raw_spu_ctr++; diff --git a/rpcs3/Emu/Cell/lv2/sys_event.cpp b/rpcs3/Emu/Cell/lv2/sys_event.cpp index 9e41d952cb..56bfee417d 100644 --- a/rpcs3/Emu/Cell/lv2/sys_event.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_event.cpp @@ -59,7 +59,7 @@ void lv2_event_queue::save_ptr(utils::serial& ar, lv2_event_queue* q) std::shared_ptr lv2_event_queue::load_ptr(utils::serial& ar, std::shared_ptr& queue, std::string_view msg) { - const u32 id = ar.operator u32(); + const u32 id = ar.pop(); if (!id) { diff --git a/rpcs3/Emu/Cell/lv2/sys_fs.cpp b/rpcs3/Emu/Cell/lv2/sys_fs.cpp index 88f2553a7c..763ef702ae 100644 --- a/rpcs3/Emu/Cell/lv2/sys_fs.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_fs.cpp @@ -445,7 +445,7 @@ lv2_file::lv2_file(utils::serial& ar) g_fxo->get().npdrm_fds.raw() += type != lv2_file_type::regular; - if (ar.operator bool()) // see lv2_file::save in_mem + if (ar.pop()) // see lv2_file::save in_mem { const fs::stat_t stat = ar; diff --git a/rpcs3/Emu/Cell/lv2/sys_lwcond.cpp b/rpcs3/Emu/Cell/lv2/sys_lwcond.cpp index a96db29a91..541d8f27ad 100644 --- a/rpcs3/Emu/Cell/lv2/sys_lwcond.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_lwcond.cpp @@ -12,10 +12,10 @@ LOG_CHANNEL(sys_lwcond); lv2_lwcond::lv2_lwcond(utils::serial& ar) - : name(ar.operator be_t()) + : name(ar.pop>()) , lwid(ar) , protocol(ar) - , control(ar.operator decltype(control)()) + , control(ar.pop()) { } diff --git a/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp b/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp index 5be53a2513..d1d1738070 100644 --- a/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp @@ -12,8 +12,8 @@ LOG_CHANNEL(sys_lwmutex); lv2_lwmutex::lv2_lwmutex(utils::serial& ar) : protocol(ar) - , control(ar.operator decltype(control)()) - , name(ar.operator be_t()) + , control(ar.pop()) + , name(ar.pop>()) { ar(lv2_control.raw().signaled); } diff --git a/rpcs3/Emu/Cell/lv2/sys_mmapper.cpp b/rpcs3/Emu/Cell/lv2/sys_mmapper.cpp index 87a5d89d86..4e3a410fd3 100644 --- a/rpcs3/Emu/Cell/lv2/sys_mmapper.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_mmapper.cpp @@ -50,7 +50,7 @@ lv2_memory::lv2_memory(utils::serial& ar) , flags(ar) , key(ar) , pshared(ar) - , ct(lv2_memory_container::search(ar.operator u32())) + , ct(lv2_memory_container::search(ar.pop())) , shm([&](u32 addr) { if (addr) @@ -61,7 +61,7 @@ lv2_memory::lv2_memory(utils::serial& ar) const auto _shm = std::make_shared(size, 1); ar(std::span(_shm->map_self(), size)); return _shm; - }(ar.operator u32())) + }(ar.pop())) , counter(ar) { #ifndef _WIN32 diff --git a/rpcs3/Emu/Cell/lv2/sys_overlay.cpp b/rpcs3/Emu/Cell/lv2/sys_overlay.cpp index f987905fa8..fc831e3bf6 100644 --- a/rpcs3/Emu/Cell/lv2/sys_overlay.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_overlay.cpp @@ -72,7 +72,7 @@ fs::file make_file_view(fs::file&& file, u64 offset, u64 size); std::shared_ptr lv2_overlay::load(utils::serial& ar) { - const std::string path = vfs::get(ar.operator std::string()); + const std::string path = vfs::get(ar.pop()); const s64 offset = ar; std::shared_ptr ovlm; diff --git a/rpcs3/Emu/Cell/lv2/sys_prx.cpp b/rpcs3/Emu/Cell/lv2/sys_prx.cpp index d16c617e34..35bd37d4f5 100644 --- a/rpcs3/Emu/Cell/lv2/sys_prx.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_prx.cpp @@ -292,7 +292,7 @@ std::shared_ptr lv2_prx::load(utils::serial& ar) { [[maybe_unused]] const s32 version = GET_SERIALIZATION_VERSION(lv2_prx_overlay); - const std::string path = vfs::get(ar.operator std::string()); + const std::string path = vfs::get(ar.pop()); const s64 offset = ar; const u32 state = ar; diff --git a/rpcs3/Emu/Cell/lv2/sys_spu.cpp b/rpcs3/Emu/Cell/lv2/sys_spu.cpp index d69d20c2b7..14fb9cc677 100644 --- a/rpcs3/Emu/Cell/lv2/sys_spu.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_spu.cpp @@ -202,13 +202,13 @@ void sys_spu_image::deploy(u8* loc, std::span segs, bool } lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept - : name(ar.operator std::string()) + : name(ar.pop()) , id(idm::last_id()) , max_num(ar) , mem_size(ar) , type(ar) // SPU Thread Group Type , ct(lv2_memory_container::search(ar)) - , has_scheduler_context(ar.operator u8()) + , has_scheduler_context(ar.pop()) , max_run(ar) , init(ar) , prio([&ar]() @@ -219,12 +219,12 @@ lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept return prio; }()) - , run_state(ar.operator spu_group_status()) + , run_state(ar.pop()) , exit_status(ar) { for (auto& thread : threads) { - if (ar.operator u8()) + if (ar.pop()) { ar(id_manager::g_id); thread = std::make_shared>(ar, this); @@ -239,7 +239,7 @@ lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept for (auto ep : {&ep_run, &ep_exception, &ep_sysmodule}) { - *ep = idm::get_unlocked(ar.operator u32()); + *ep = idm::get_unlocked(ar.pop()); } waiter_spu_index = -1; @@ -328,7 +328,7 @@ void lv2_spu_group::save(utils::serial& ar) lv2_spu_image::lv2_spu_image(utils::serial& ar) : e_entry(ar) - , segs(ar.operator decltype(segs)()) + , segs(ar.pop()) , nsegs(ar) { } diff --git a/rpcs3/Emu/Cell/lv2/sys_usbd.cpp b/rpcs3/Emu/Cell/lv2/sys_usbd.cpp index b3b8b59413..65e183000b 100644 --- a/rpcs3/Emu/Cell/lv2/sys_usbd.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_usbd.cpp @@ -70,7 +70,7 @@ public: usb_handler_thread(utils::serial& ar) : usb_handler_thread() { - is_init = !!ar.operator u8(); + is_init = !!ar.pop(); } void save(utils::serial& ar) diff --git a/rpcs3/Emu/IdManager.h b/rpcs3/Emu/IdManager.h index 36163b258d..6322cd7e41 100644 --- a/rpcs3/Emu/IdManager.h +++ b/rpcs3/Emu/IdManager.h @@ -270,7 +270,7 @@ namespace id_manager { vec.resize(T::id_count); - u32 i = ar.operator u32(); + u32 i = ar.pop(); ensure(i <= T::id_count); @@ -309,7 +309,7 @@ namespace id_manager void save(utils::serial& ar) requires IdmSavable { u32 obj_count = 0; - usz obj_count_offs = ar.data.size(); + usz obj_count_offs = ar.pos; // To be patched at the end of the function ar(obj_count); @@ -340,7 +340,7 @@ namespace id_manager } // Patch object count - std::memcpy(ar.data.data() + obj_count_offs, &obj_count, sizeof(obj_count)); + ar.patch_raw_data(obj_count_offs, &obj_count, sizeof(obj_count)); } id_map& operator=(thread_state state) noexcept requires (std::is_assignable_v) diff --git a/rpcs3/Emu/Memory/vm.cpp b/rpcs3/Emu/Memory/vm.cpp index 79680c5c41..769a31a494 100644 --- a/rpcs3/Emu/Memory/vm.cpp +++ b/rpcs3/Emu/Memory/vm.cpp @@ -1620,45 +1620,52 @@ namespace vm return _7 == v128{}; } - static void save_memory_bytes(utils::serial& ar, const u8* ptr, usz size) + static void serialize_memory_bytes(utils::serial& ar, u8* ptr, usz size) { - AUDIT(ar.is_writing() && !(size % 1024)); + ensure((size % 4096) == 0); - for (; size; ptr += 128 * 8, size -= 128 * 8) + for (; size; ptr += 128 * 8) { - ar(u8{}); // bitmap of 1024 bytes (bit is 128-byte) - u8 bitmap = 0, count = 0; + const usz process_size = std::min(size, 128 * 8); + size -= process_size; - for (usz i = 0, end = std::min(size, 128 * 8); i < end; i += 128) + u8 bitmap = 0; + + if (ar.is_writing()) { - if (!check_cache_line_zero(ptr + i)) + for (usz i = 0; i < process_size; i += 128) { - bitmap |= 1u << (i / 128); - count++; - ar(std::span(ptr + i, 128)); + if (!check_cache_line_zero(ptr + i)) + { + bitmap |= 1u << (i / 128); + } } } - // Patch bitmap with correct value - *std::prev(&ar.data.back(), count * 128) = bitmap; - } - } + // bitmap of 1024 bytes (bit is 128-byte) + ar(bitmap); - static void load_memory_bytes(utils::serial& ar, u8* ptr, usz size) - { - AUDIT(!ar.is_writing() && !(size % 128)); - - for (; size; ptr += 128 * 8, size -= 128 * 8) - { - const u8 bitmap{ar}; - - for (usz i = 0, end = std::min(size, 128 * 8); i < end; i += 128) + for (usz i = 0; i < process_size;) { - if (bitmap & (1u << (i / 128))) + usz block_count = 0; + + for (usz bit = i / 128; bit < sizeof(bitmap) * 8 && (bitmap & (1u << bit)) != 0;) { - ar(std::span(ptr + i, 128)); + bit++; + block_count++; } + + if (!block_count) + { + i += 128; + continue; + } + + ar(std::span(ptr + i, block_count * 128)); + i += block_count * 128; } + + ar.breathe(); } } @@ -1682,14 +1689,15 @@ namespace vm if (is_memory_compatible_for_copy_from_executable_optimization(addr, shm.first)) { // Revert changes - ar.data.resize(ar.seek_end(sizeof(u32) * 2 + sizeof(memory_page))); + ar.data.resize(ar.data.size() - (sizeof(u32) * 2 + sizeof(memory_page))); + ar.seek_end(); vm_log.success("Removed memory block matching the memory of the executable from savestate. (addr=0x%x, size=0x%x)", addr, shm.first); continue; } // Save raw binary image const u32 guard_size = flags & stack_guarded ? 0x1000 : 0; - save_memory_bytes(ar, vm::get_super_ptr(addr + guard_size), shm.first - guard_size * 2); + serialize_memory_bytes(ar, vm::get_super_ptr(addr + guard_size), shm.first - guard_size * 2); } else { @@ -1758,13 +1766,13 @@ namespace vm // Map the memory through the same method as alloc() and falloc() // Copy the shared handle unconditionally - ensure(try_alloc(addr0, pflags, size0, ::as_rvalue(flags & preallocated ? null_shm : shared[ar.operator usz()]))); + ensure(try_alloc(addr0, pflags, size0, ::as_rvalue(flags & preallocated ? null_shm : shared[ar.pop()]))); if (flags & preallocated) { // Load binary image const u32 guard_size = flags & stack_guarded ? 0x1000 : 0; - load_memory_bytes(ar, vm::get_super_ptr(addr0 + guard_size), size0 - guard_size * 2); + serialize_memory_bytes(ar, vm::get_super_ptr(addr0 + guard_size), size0 - guard_size * 2); } } } @@ -2224,9 +2232,8 @@ namespace vm // Save shared memory ar(shm->flags()); - // TODO: string_view serialization (even with load function, so the loaded address points to a position of the stream's buffer) ar(shm->size()); - save_memory_bytes(ar, vm::get_super_ptr(addr), shm->size()); + serialize_memory_bytes(ar, vm::get_super_ptr(addr), shm->size()); } // TODO: Serialize std::vector direcly @@ -2249,19 +2256,19 @@ namespace vm void load(utils::serial& ar) { std::vector> shared; - shared.resize(ar.operator usz()); + shared.resize(ar.pop()); for (auto& shm : shared) { // Load shared memory - const u32 flags = ar; - const u64 size = ar; + const u32 flags = ar.pop(); + const u64 size = ar.pop(); shm = std::make_shared(size, flags); // Load binary image // elad335: I'm not proud about it as well.. (ideal situation is to not call map_self()) - load_memory_bytes(ar, shm->map_self(), shm->size()); + serialize_memory_bytes(ar, shm->map_self(), shm->size()); } for (auto& block : g_locations) @@ -2270,11 +2277,11 @@ namespace vm } g_locations.clear(); - g_locations.resize(ar.operator usz()); + g_locations.resize(ar.pop()); for (auto& loc : g_locations) { - const u8 has = ar; + const u8 has = ar.pop(); if (has) { diff --git a/rpcs3/Emu/RSX/RSXThread.cpp b/rpcs3/Emu/RSX/RSXThread.cpp index 2df6e0cb60..244ad670b1 100644 --- a/rpcs3/Emu/RSX/RSXThread.cpp +++ b/rpcs3/Emu/RSX/RSXThread.cpp @@ -4,6 +4,7 @@ #include "Emu/Cell/PPUCallback.h" #include "Emu/Cell/SPUThread.h" #include "Emu/Cell/timers.hpp" +#include "Emu/savestate_utils.hpp" #include "Capture/rsx_capture.h" #include "Common/BufferUtils.h" @@ -19,6 +20,7 @@ #include "Emu/Cell/lv2/sys_event.h" #include "Emu/Cell/lv2/sys_time.h" #include "Emu/Cell/Modules/cellGcmSys.h" +#include "Emu/savestate_utils.hpp" #include "Overlays/overlay_perf_metrics.h" #include "Overlays/overlay_message.h" #include "Program/GLSLCommon.h" @@ -26,7 +28,6 @@ #include "Utilities/StrUtil.h" #include "Crypto/unzip.h" -#include "util/serialization.hpp" #include "util/asm.hpp" #include @@ -3574,6 +3575,8 @@ namespace rsx void thread::on_frame_end(u32 buffer, bool forced) { + bool pause_emulator = false; + // Marks the end of a frame scope GPU-side if (g_user_asked_for_frame_capture.exchange(false) && !capture_current_frame) { @@ -3594,36 +3597,34 @@ namespace rsx { capture_current_frame = false; - std::string file_path = fs::get_config_dir() + "captures/" + Emu.GetTitleID() + "_" + date_time::current_time_narrow() + "_capture.rrc"; - - utils::serial save_manager; - save_manager.reserve(0x800'0000); // 128MB - - save_manager(frame_capture); - - if (std::vector zipped = zip(save_manager.data); !zipped.empty()) - { - file_path += ".gz"; - save_manager.data = std::move(zipped); - } - else - { - rsx_log.error("Failed to compress capture"); - } + std::string file_path = fs::get_config_dir() + "captures/" + Emu.GetTitleID() + "_" + date_time::current_time_narrow() + "_capture.rrc.gz"; fs::pending_file temp(file_path); - if (temp.file && (temp.file.write(save_manager.data), temp.commit(false))) + utils::serial save_manager; + + if (temp.file) { - rsx_log.success("Capture successful: %s", file_path); + save_manager.m_file_handler = make_compressed_serialization_file_handler(temp.file); + save_manager(frame_capture); + + save_manager.m_file_handler->finalize(save_manager); + + if (temp.commit(false)) + { + rsx_log.success("Capture successful: %s", file_path); + frame_capture.reset(); + pause_emulator = true; + } + else + { + rsx_log.error("Capture failed: %s (%s)", file_path, fs::g_tls_error); + } } else { rsx_log.fatal("Capture failed: %s (%s)", file_path, fs::g_tls_error); } - - frame_capture.reset(); - Emu.Pause(); } if (zcull_ctrl->has_pending()) @@ -3673,6 +3674,12 @@ namespace rsx } } + if (pause_emulator) + { + Emu.Pause(); + thread_ctrl::wait_for(30'000); + } + // Reset current stats m_frame_stats = {}; m_profiler.enabled = !!g_cfg.video.overlay; diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index 1ce805d858..d3bb9aa9c2 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -83,9 +83,6 @@ extern void ppu_unload_prx(const lv2_prx&); extern std::shared_ptr ppu_load_prx(const ppu_prx_object&, bool virtual_load, const std::string&, s64 = 0, utils::serial* = nullptr); extern std::pair, CellError> ppu_load_overlay(const ppu_exec_object&, bool virtual_load, const std::string& path, s64 = 0, utils::serial* = nullptr); extern bool ppu_load_rel_exec(const ppu_rel_object&); -extern bool is_savestate_version_compatible(const std::vector>& data, bool is_boot_check); -extern std::vector> read_used_savestate_versions(); -std::string get_savestate_file(std::string_view title_id, std::string_view boot_path, s64 abs_id, s64 rel_id); extern void send_close_home_menu_cmds(); @@ -219,7 +216,7 @@ void init_fxo_for_exec(utils::serial* ar, bool full = false) const usz advance = (Emu.m_savestate_extension_flags1 & Emulator::SaveStateExtentionFlags1::SupportsMenuOpenResume ? 32 : 31); - ar->pos += advance; // Reserved area + load_and_check_reserved(*ar, advance); // Reserved area } } @@ -719,8 +716,11 @@ bool Emulator::BootRsxCapture(const std::string& path) if (fmt::to_lower(path).ends_with(".gz")) { - load.data = unzip(in_file.to_vector()); - in_file.close(); + load.m_file_handler = make_compressed_serialization_file_handler(std::move(in_file)); + + // Forcefully read some data to check validity + load.pop(); + load.pos -= sizeof(uchar); if (load.data.empty()) { @@ -896,13 +896,31 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, } else { - if (fs::file save{m_path, fs::isfile + fs::read}; save && save.size() >= 8 && save.read() == "RPCS3SAV"_u64) + fs::file save{m_path, fs::isfile + fs::read}; + + if (!m_path.ends_with(".gz") && save && save.size() >= 8 && save.read() == "RPCS3SAV"_u64) { m_ar = std::make_shared(); m_ar->set_reading_state(); m_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(save)); } + else if (save && m_path.ends_with(".gz")) + { + m_ar = std::make_shared(); + m_ar->set_reading_state(); + + m_ar->m_file_handler = make_compressed_serialization_file_handler(std::move(save)); + + if (m_ar->try_read().second != "RPCS3SAV"_u64) + { + m_ar.reset(); + } + else + { + m_ar->pos = 0; + } + } m_boot_source_type = CELL_GAME_GAMETYPE_SYS; } @@ -952,7 +970,7 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, bool LE_format; bool state_inspection_support; nse_t offset; - std::array reserved; + b8 flag_versions_is_following_data; }; const auto header = m_ar->try_read().second; @@ -969,15 +987,24 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, g_cfg.savestate.state_inspection_mode.set(header.state_inspection_support); + if (header.flag_versions_is_following_data) + { + ensure(header.offset == m_ar->pos); + + if (!is_savestate_version_compatible(m_ar->pop>(), true)) + { + return game_boot_result::savestate_version_unsupported; + } + } + else { // Read data on another container to keep the existing data utils::serial ar_temp; - ar_temp.set_reading_state(); + ar_temp.set_reading_state({}, true); ar_temp.swap_handler(*m_ar); ar_temp.seek_pos(header.offset); - ar_temp.m_avoid_large_prefetch = true; - if (!is_savestate_version_compatible(ar_temp.operator std::vector>(), true)) + if (!is_savestate_version_compatible(ar_temp.pop>(), true)) { return game_boot_result::savestate_version_unsupported; } @@ -986,11 +1013,16 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, ar_temp.swap_handler(*m_ar); } + if (!load_and_check_reserved(*m_ar, header.flag_versions_is_following_data ? 32 : 31)) + { + return game_boot_result::savestate_version_unsupported; + } + argv.clear(); klic.clear(); std::string disc_info; - (*m_ar)(argv.emplace_back(), disc_info, klic.emplace_back(), m_game_dir, hdd1); + m_ar->serialize(argv.emplace_back(), disc_info, klic.emplace_back(), m_game_dir, hdd1); if (!klic[0]) { @@ -1026,12 +1058,16 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, { const usz size = *m_ar; + fs::remove_all(path, size == 0); + if (size) { - fs::remove_all(path, false); m_ar->breathe(true); - ensure(tar_object(make_file_view(*static_cast(m_ar->m_file_handler.get())->m_file, m_ar->data_offset, size)).extract(path)); - m_ar->seek_pos(m_ar->pos + size, size >= 4096); + m_ar->m_max_data = m_ar->pos + size; + ensure(tar_object(*m_ar).extract(path)); + m_ar->seek_pos(m_ar->m_max_data, size >= 4096); + m_ar->m_max_data = umax; + m_ar->breathe(); } }; @@ -1053,7 +1089,11 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, load_tar(hdd0_game + game_data); } - m_ar->pos += 32; // Reserved area + // Reserved area + if (!load_and_check_reserved(*m_ar, 32)) + { + return game_boot_result::savestate_version_unsupported; + } if (disc_info.starts_with("/"sv)) { @@ -2261,7 +2301,7 @@ void Emulator::FixGuestTime() { if (m_ar) { - initialize_timebased_time(m_ar->operator u64()); + initialize_timebased_time(m_ar->pop()); g_cfg.savestate.state_inspection_mode.set(m_state_inspection_savestate); @@ -2862,6 +2902,13 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s { path = get_savestate_file(m_title_id, m_path, 0, 0); + // The function is meant for reading files, so if there is no GZ file it would not return compressed file path + // So this is the only place where the result is edited if need to be + if (!path.ends_with(".gz")) + { + path += ".gz"; + } + if (!fs::create_path(fs::get_parent_dir(path))) { sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error); @@ -2877,7 +2924,7 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s } to_ar = std::make_unique(); - to_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(file.file)); + to_ar->m_file_handler = make_compressed_serialization_file_handler(file.file); break; } @@ -2899,31 +2946,31 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s // Avoid duplicating TAR object memory because it can be very large auto save_tar = [&](const std::string& path) { - const usz old_data_start = ar.data_offset; - const usz old_pos = ar.seek_end(); + if (!fs::is_dir(path)) + { + ar(usz{}); + return; + } + // Cached file list from the first call + std::vector dir_entries; + + // Calculate memory requirements + utils::serial ar_null; + ar_null.m_file_handler = make_null_serialization_file_handler(); + tar_object::save_directory(path, ar_null, {}, std::move(dir_entries), false); + ar(ar_null.pos); ar.breathe(); - ar(usz{}); // Reserve memory to be patched later with correct size - tar_object::save_directory(path, ar); - + const usz old_pos = ar.seek_end(); + tar_object::save_directory(path, ar, {}, std::move(dir_entries), true); const usz new_pos = ar.seek_end(); - const usz tar_size = new_pos - old_pos - sizeof(usz); - // Check if breathe() actually did something, in this case memory needs to be discarded - const bool was_emptied = old_data_start != ar.data_offset; + const usz tar_size = new_pos - old_pos; - if (was_emptied) + if (tar_size != ar_null.pos) { - ensure(ar.data_offset > old_data_start); - - // Write to file directly (slower) - ar.m_file_handler->handle_file_op(ar, old_pos, sizeof(tar_size), &tar_size); - } - else - { - // If noty written to file, simply write to memory - std::memcpy(ar.data.data() + old_pos - old_data_start, &tar_size, sizeof(usz)); + fmt::throw_exception("Unexpected TAR entry size (size=0x%x, expected=0x%x, entries=0x%x)", tar_size, ar_null.pos, dir_entries.size()); } sys_log.success("Saved the contents of directory '%s' (size=0x%x)", path, tar_size); @@ -2969,7 +3016,18 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s ar("RPCS3SAV"_u64); ar(std::endian::native == std::endian::little); ar(g_cfg.savestate.state_inspection_mode.get()); - ar(usz{0}); // Offset of versioning data, to be overwritten at the end of saving + + ar(usz{10 + sizeof(usz) + sizeof(u8)}); // Offset of versioning data (fixed to the following data) + + { + // Gather versions because with compressed format going back and patching offset is not efficient + utils::serial ar_temp; + ar_temp.m_file_handler = make_null_serialization_file_handler(); + g_fxo->save(ar_temp); + ar(u8{1}); + ar(read_used_savestate_versions()); + } + ar(std::array{}); // Reserved for future use if (auto dir = vfs::get("/dev_bdvd/PS3_GAME"); fs::is_dir(dir) && !fs::is_file(fs::get_parent_dir(dir) + "/PS3_DISC.SFB")) @@ -3022,27 +3080,11 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s if (savestate) { - // Identifer -> version - std::vector> used_serial = read_used_savestate_versions(); - auto& ar = *to_ar; - const usz pos = ar.seek_end(); - - // Patch offset with a direct write - ar.m_file_handler->handle_file_op(ar, 10, sizeof(usz), &pos); - - // Write the version data at the end - ar(used_serial); - // Final file write, the file is ready to be committed - ar.breathe(true); - -#ifndef _WIN32 - // The temporary file's contents must be on disk before rename - // Flush to file - ar.m_file_handler->handle_file_op(ar, umax, umax, nullptr); -#endif + ar.seek_end(); + ar.m_file_handler->finalize(ar); if (!file.commit()) { diff --git a/rpcs3/Emu/savestate_utils.cpp b/rpcs3/Emu/savestate_utils.cpp index 65e2b642ed..fa95603435 100644 --- a/rpcs3/Emu/savestate_utils.cpp +++ b/rpcs3/Emu/savestate_utils.cpp @@ -2,6 +2,8 @@ #include "util/types.hpp" #include "util/logs.hpp" #include "util/asm.hpp" +#include "util/v128.hpp" +#include "util/simd.hpp" #include "Utilities/File.h" #include "Utilities/StrFmt.h" #include "system_config.h" @@ -10,6 +12,9 @@ #include "System.h" #include +#include + +#include LOG_CHANNEL(sys_log, "SYS"); @@ -18,14 +23,14 @@ void fmt_class_string::format(std::string& out, u64 arg) { const utils::serial& ar = get_object(arg); - fmt::append(out, "{ %s, 0x%x/0%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size()); + fmt::append(out, "{ %s, 0x%x/0x%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size()); } struct serial_ver_t { bool used = false; - s32 current_version = 0; - std::set compatible_versions; + u16 current_version = 0; + std::set compatible_versions; }; static std::array s_serial_versions; @@ -89,7 +94,7 @@ SERIALIZATION_VER(sys_io, 23, 1) SERIALIZATION_VER(LLE, 24, 1) SERIALIZATION_VER(HLE, 25, 1) -std::vector> get_savestate_versioning_data(fs::file&& file) +std::vector get_savestate_versioning_data(fs::file&& file, std::string_view filepath) { if (!file) { @@ -98,31 +103,36 @@ std::vector> get_savestate_versioning_data(fs::file&& file) file.seek(0); - if (u64 r = 0; !file.read(r) || r != "RPCS3SAV"_u64) + utils::serial ar; + ar.set_reading_state(); + + ar.m_file_handler = filepath.ends_with(".gz") ? static_cast>(make_compressed_serialization_file_handler(std::move(file))) + : make_uncompressed_serialization_file_handler(std::move(file)); + + if (u64 r = 0; ar.try_read(r) != 0 || r != "RPCS3SAV"_u64) { return {}; } - file.seek(10); + ar.pos = 10; - u64 offs = 0; - file.read(offs); + u64 offs = ar.try_read().second; - const usz fsize = file.size(); + const usz fsize = ar.get_size(offs); if (!offs || fsize <= offs) { return {}; } - utils::serial ar; - ar.set_reading_state(); - ar.m_file_handler = make_uncompressed_serialization_file_handler(std::move(file)); ar.seek_pos(offs); - return ar; + ar.breathe(true); + + std::vector ver_data = ar.pop>(); + return std::move(ver_data); } -bool is_savestate_version_compatible(const std::vector>& data, bool is_boot_check) +bool is_savestate_version_compatible(const std::vector& data, bool is_boot_check) { if (data.empty()) { @@ -178,24 +188,31 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_ // While not needing to keep a 59 chars long suffix at all times for this purpose const char prefix = ::at32("0123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"sv, save_id.size()); - return fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT"; + std::string path = fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT"; + + if (std::string path_compressed = path + ".gz"; fs::is_file(path_compressed)) + { + return std::move(path_compressed); + } + + return std::move(path); } -bool is_savestate_compatible(fs::file&& file) +bool is_savestate_compatible(fs::file&& file, std::string_view filepath) { - return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file)), false); + return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file), filepath), false); } -std::vector> read_used_savestate_versions() +std::vector read_used_savestate_versions() { - std::vector> used_serial; + std::vector used_serial; used_serial.reserve(s_serial_versions.size()); for (serial_ver_t& ver : s_serial_versions) { if (std::exchange(ver.used, false)) { - used_serial.emplace_back(&ver - s_serial_versions.data(), *ver.compatible_versions.rbegin()); + used_serial.push_back(version_entry{static_cast(&ver - s_serial_versions.data()), *ver.compatible_versions.rbegin()}); } ver.current_version = 0; @@ -208,8 +225,6 @@ bool boot_last_savestate(bool testing) { if (!g_cfg.savestate.suspend_emu && !Emu.GetTitleID().empty() && (Emu.IsRunning() || Emu.GetStatus() == system_state::paused)) { - extern bool is_savestate_compatible(fs::file&& file); - const std::string save_dir = fs::get_cache_dir() + "/savestates/"; std::string savestate_path; @@ -225,7 +240,12 @@ bool boot_last_savestate(bool testing) // Find the latest savestate file compatible with the game (TODO: Check app version and anything more) if (entry.name.find(Emu.GetTitleID()) != umax && mtime <= entry.mtime) { - if (std::string path = save_dir + entry.name; is_savestate_compatible(fs::file(path))) + if (std::string path = save_dir + entry.name + ".gz"; is_savestate_compatible(fs::file(path), path)) + { + savestate_path = std::move(path); + mtime = entry.mtime; + } + else if (std::string path = save_dir + entry.name; is_savestate_compatible(fs::file(path), path)) { savestate_path = std::move(path); mtime = entry.mtime; @@ -282,9 +302,9 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, m_file->sync(); } - ar.data_offset += ar.data.size(); - ar.data.clear(); ar.seek_end(); + ar.data_offset = ar.pos; + ar.data.clear(); return true; } @@ -320,7 +340,7 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, } // Discard all loaded data - ar.data_offset += ar.data.size(); + ar.data_offset = ar.pos; ar.data.clear(); if (ar.data.capacity() >= 0x200'0000) @@ -332,7 +352,7 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, return true; } - if (~size < pos) + if (~pos < size - 1) { // Overflow return false; @@ -340,11 +360,11 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, if (ar.data.empty() && pos != ar.pos) { - // Relocate instead oof over-fetch + // Relocate instead of over-fetch ar.seek_pos(pos); } - const usz read_pre_buffer = utils::sub_saturate(ar.data_offset, pos); + const usz read_pre_buffer = ar.data.empty() ? 0 : utils::sub_saturate(ar.data_offset, pos); if (read_pre_buffer) { @@ -358,7 +378,10 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, ar.data_offset -= read_pre_buffer; } - const usz read_past_buffer = utils::sub_saturate(pos + size, ar.data_offset + ar.data.size()); + // Adjustment to prevent overflow + const usz subtrahend = ar.data.empty() ? 0 : 1; + const usz read_past_buffer = utils::sub_saturate(pos + (size - subtrahend), ar.data_offset + (ar.data.size() - subtrahend)); + const usz read_limit = utils::sub_saturate(ar.m_max_data, ar.data_offset); if (read_past_buffer) { @@ -368,14 +391,13 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, const usz old_size = ar.data.size(); // Try to prefetch data by reading more than requested - ar.data.resize(std::max({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} })); + ar.data.resize(std::min(read_limit, std::max({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} }))); ar.data.resize(m_file->read_at(old_size + ar.data_offset, data ? const_cast(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size); } return true; } - usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const { if (ar.is_writing()) @@ -394,6 +416,416 @@ usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, u return std::max(m_file->size(), memory_available); } +void uncompressed_serialization_file_handler::finalize(utils::serial& ar) +{ + ar.seek_end(); + handle_file_op(ar, 0, umax, nullptr); + ar.data = {}; // Deallocate and clear +} + +void compressed_serialization_file_handler::initialize(utils::serial& ar) +{ + if (!m_stream.has_value()) + { + m_stream.emplace(); + } + + z_stream& m_zs = std::any_cast(m_stream); + + if (ar.is_writing()) + { + if (m_write_inited) + { + return; + } + +#ifndef _MSC_VER +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#endif + if (m_read_inited) + { + finalize(ar); + } + + m_zs = {}; + ensure(deflateInit2(&m_zs, 9, Z_DEFLATED, 16 + 15, 9, Z_DEFAULT_STRATEGY) == Z_OK); +#ifndef _MSC_VER +#pragma GCC diagnostic pop +#endif + m_write_inited = true; + } + else + { + if (m_read_inited) + { + return; + } + + if (m_write_inited) + { + finalize(ar); + } + + m_zs.avail_in = 0; + m_zs.avail_out = 0; + m_zs.next_in = nullptr; + m_zs.next_out = nullptr; + #ifndef _MSC_VER + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wold-style-cast" + #endif + ensure(inflateInit2(&m_zs, 16 + 15) == Z_OK); + m_read_inited = true; + } +} + +bool compressed_serialization_file_handler::handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) +{ + if (ar.is_writing()) + { + initialize(ar); + + z_stream& m_zs = std::any_cast(m_stream); + + if (data) + { + ensure(false); + } + + // Writing not at the end is forbidden + ensure(ar.pos == ar.data_offset + ar.data.size()); + + m_zs.avail_in = static_cast(ar.data.size()); + m_zs.next_in = ar.data.data(); + + do + { + m_stream_data.resize(std::max(m_stream_data.size(), ::compressBound(m_zs.avail_in))); + m_zs.avail_out = static_cast(m_stream_data.size()); + m_zs.next_out = m_stream_data.data(); + + if (deflate(&m_zs, Z_NO_FLUSH) == Z_STREAM_ERROR || m_file->write(m_stream_data.data(), m_stream_data.size() - m_zs.avail_out) != m_stream_data.size() - m_zs.avail_out) + { + deflateEnd(&m_zs); + //m_file->close(); + break; + } + } + while (m_zs.avail_out == 0); + + ar.seek_end(); + ar.data_offset = ar.pos; + ar.data.clear(); + + if (pos == umax && size == umax && *m_file) + { + // Request to flush the file to disk + m_file->sync(); + } + + return true; + } + + initialize(ar); + + if (!size) + { + return true; + } + + if (pos == 0 && size == umax) + { + // Discard loaded data until pos if profitable + const usz limit = ar.data_offset + ar.data.size(); + + if (ar.pos > ar.data_offset && ar.pos < limit) + { + const usz may_discard_bytes = ar.pos - ar.data_offset; + const usz moved_byte_count_on_discard = limit - ar.pos; + + // Cheeck profitability (check recycled memory and std::memmove costs) + if (may_discard_bytes >= 0x50'0000 || (may_discard_bytes >= 0x20'0000 && moved_byte_count_on_discard / may_discard_bytes < 3)) + { + ar.data_offset += may_discard_bytes; + ar.data.erase(ar.data.begin(), ar.data.begin() + may_discard_bytes); + + if (ar.data.capacity() >= 0x200'0000) + { + // Discard memory + ar.data.shrink_to_fit(); + } + } + + return true; + } + + // Discard all loaded data + ar.data_offset += ar.data.size(); + ensure(ar.pos >= ar.data_offset); + ar.data.clear(); + + if (ar.data.capacity() >= 0x200'0000) + { + // Discard memory + ar.data.shrink_to_fit(); + } + + return true; + } + + if (~pos < size - 1) + { + // Overflow + return false; + } + + // TODO: Investigate if this optimization is worth an implementation for compressed stream + // if (ar.data.empty() && pos != ar.pos) + // { + // // Relocate instead of over-fetch + // ar.seek_pos(pos); + // } + + const usz read_pre_buffer = utils::sub_saturate(ar.data_offset, pos); + + if (read_pre_buffer) + { + // Not allowed with compressed data for now + // Unless someone implements mechanism for it + ensure(false); + } + + // Adjustment to prevent overflow + const usz subtrahend = ar.data.empty() ? 0 : 1; + const usz read_past_buffer = utils::sub_saturate(pos + (size - subtrahend), ar.data_offset + (ar.data.size() - subtrahend)); + const usz read_limit = utils::sub_saturate(ar.m_max_data, ar.data_offset); + + if (read_past_buffer) + { + // Read proceeding data + // More lightweight operation, this is the common operation + // Allowed to fail, if memory is truly needed an assert would take place later + const usz old_size = ar.data.size(); + + // Try to prefetch data by reading more than requested + ar.data.resize(std::min(read_limit, std::max({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} }))); + ar.data.resize(this->read_at(ar, old_size + ar.data_offset, data ? const_cast(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size); + } + + return true; +} + +usz compressed_serialization_file_handler::read_at(utils::serial& ar, usz read_pos, void* data, usz size) +{ + ensure(read_pos == ar.data.size() + ar.data_offset - size); + + if (!size) + { + return 0; + } + + initialize(ar); + + z_stream& m_zs = std::any_cast(m_stream); + + const usz total_to_read = size; + usz read_size = 0; + u8* out_data = static_cast(data); + + auto adjust_for_uint = [](usz size) + { + return static_cast(std::min(uInt{umax}, size)); + }; + + for (; read_size < total_to_read;) + { + // Drain extracted memory stash (also before first file read) + out_data = static_cast(data) + read_size; + m_zs.avail_in = adjust_for_uint(m_stream_data.size() - m_stream_data_index); + m_zs.next_in = reinterpret_cast(m_stream_data.data() + m_stream_data_index); + m_zs.next_out = out_data; + m_zs.avail_out = adjust_for_uint(size - read_size); + + while (read_size < total_to_read && m_zs.avail_in) + { + const int res = inflate(&m_zs, Z_BLOCK); + + bool need_more_file_memory = false; + + switch (res) + { + case Z_OK: + case Z_STREAM_END: + break; + case Z_BUF_ERROR: + { + if (m_zs.avail_in) + { + need_more_file_memory = true; + break; + } + + [[fallthrough]]; + } + default: + inflateEnd(&m_zs); + m_read_inited = false; + return read_size; + } + + read_size = m_zs.next_out - static_cast(data); + m_stream_data_index = m_zs.avail_in ? m_zs.next_in - m_stream_data.data() : m_stream_data.size(); + + // Adjust again in case the values simply did not fit into uInt + m_zs.avail_out = adjust_for_uint(utils::sub_saturate(total_to_read, read_size)); + m_zs.avail_in = adjust_for_uint(m_stream_data.size() - m_stream_data_index); + + if (need_more_file_memory) + { + break; + } + } + + if (read_size >= total_to_read) + { + break; + } + + const usz add_size = ar.m_avoid_large_prefetch ? 0x1'0000 : 0x10'0000; + const usz old_file_buf_size = m_stream_data.size(); + + m_stream_data.resize(old_file_buf_size + add_size); + m_stream_data.resize(old_file_buf_size + m_file->read_at(m_file_read_index, m_stream_data.data() + old_file_buf_size, add_size)); + + if (m_stream_data.size() == old_file_buf_size) + { + // EOF + break; + } + + m_file_read_index += m_stream_data.size() - old_file_buf_size; + } + + if (m_stream_data.size() - m_stream_data_index <= m_stream_data_index / 5) + { + // Shrink to required memory size + m_stream_data.erase(m_stream_data.begin(), m_stream_data.begin() + m_stream_data_index); + + if (m_stream_data.capacity() >= 0x200'0000) + { + // Discard memory + m_stream_data.shrink_to_fit(); + } + + m_stream_data_index = 0; + } + + return read_size; +} + +void compressed_serialization_file_handler::skip_until(utils::serial& ar) +{ + ensure(!ar.is_writing() && ar.pos >= ar.data_offset); + + if (ar.pos > ar.data_offset) + { + handle_file_op(ar, ar.data_offset, ar.pos - ar.data_offset, nullptr); + } +} + +void compressed_serialization_file_handler::finalize(utils::serial& ar) +{ + handle_file_op(ar, 0, umax, nullptr); + + if (!m_stream.has_value()) + { + return; + } + + z_stream& m_zs = std::any_cast(m_stream); + + if (m_read_inited) + { + m_read_inited = false; + ensure(inflateEnd(&m_zs) == Z_OK); + return; + } + + m_write_inited = false; + + m_zs.avail_in = 0; + m_zs.next_in = nullptr; + + m_stream_data.resize(m_zs.avail_out); + + do + { + m_zs.avail_out = static_cast(m_stream_data.size()); + m_zs.next_out = m_stream_data.data(); + + if (deflate(&m_zs, Z_FINISH) == Z_STREAM_ERROR) + { + break; + } + + m_file->write(m_stream_data.data(), m_stream_data.size() - m_zs.avail_out); + } + while (m_zs.avail_out == 0); + + m_stream_data = {}; + ensure(deflateEnd(&m_zs) == Z_OK); + ar.data = {}; // Deallocate and clear +} + +usz compressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const +{ + if (ar.is_writing()) + { + return m_file->size(); + } + + const usz memory_available = ar.data_offset + ar.data.size(); + + if (memory_available >= recommended) + { + // Avoid calling size() if possible + return memory_available; + } + + return std::max(utils::mul_saturate(m_file->size(), 6), memory_available); +} + +bool null_serialization_file_handler::handle_file_op(utils::serial&, usz, usz, const void*) +{ + return true; +} + +void null_serialization_file_handler::finalize(utils::serial&) +{ +} + +bool load_and_check_reserved(utils::serial& ar, usz size) +{ + u8 bytes[4096]; + std::memset(&bytes[size & (0 - sizeof(v128))], 0, sizeof(v128)); + ensure(size <= std::size(bytes)); + + const usz old_pos = ar.pos; + ar(std::span(bytes, size)); + + // Check if all are 0 + for (usz i = 0; i < size; i += sizeof(v128)) + { + if (v128::loadu(&bytes[i]) != v128{}) + { + return false; + } + } + + return old_pos + size == ar.pos; +} + namespace stx { extern void serial_breathe(utils::serial& ar) diff --git a/rpcs3/Emu/savestate_utils.hpp b/rpcs3/Emu/savestate_utils.hpp index c3aa8d10d0..a33915d815 100644 --- a/rpcs3/Emu/savestate_utils.hpp +++ b/rpcs3/Emu/savestate_utils.hpp @@ -1,10 +1,22 @@ +#pragma once + #include "util/serialization.hpp" +#include + namespace fs { class file; } +struct version_entry +{ + u16 type; + u16 version; + + ENABLE_BITWISE_SERIALIZATION; +}; + // Uncompressed file serialization handler struct uncompressed_serialization_file_handler : utils::serialization_file_handler { @@ -25,21 +37,102 @@ struct uncompressed_serialization_file_handler : utils::serialization_file_handl { } + uncompressed_serialization_file_handler(const uncompressed_serialization_file_handler&) = delete; + // Handle file read and write requests bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override; // Get available memory or file size // Preferably memory size if is already greater/equal to recommended to avoid additional file ops usz get_size(const utils::serial& ar, usz recommended) const override; + + void finalize(utils::serial& ar) override; }; -inline std::unique_ptr make_uncompressed_serialization_file_handler(fs::file&& file) +template requires (std::is_same_v, fs::file>) +inline std::unique_ptr make_uncompressed_serialization_file_handler(File&& file) { - return std::make_unique(std::move(file)); + ensure(file); + return std::make_unique(std::forward(file)); } -inline std::unique_ptr make_uncompressed_serialization_file_handler(const fs::file& file) +// Compressed file serialization handler +struct compressed_serialization_file_handler : utils::serialization_file_handler { - return std::make_unique(file); + const std::unique_ptr m_file_storage; + const std::add_pointer_t m_file; + std::vector m_stream_data; + usz m_stream_data_index = 0; + usz m_file_read_index = 0; + bool m_write_inited = false; + bool m_read_inited = false; + std::any m_stream; + + explicit compressed_serialization_file_handler(fs::file&& file) noexcept + : utils::serialization_file_handler() + , m_file_storage(std::make_unique(std::move(file))) + , m_file(m_file_storage.get()) + { + } + + explicit compressed_serialization_file_handler(const fs::file& file) noexcept + : utils::serialization_file_handler() + , m_file_storage(nullptr) + , m_file(std::addressof(file)) + { + } + + compressed_serialization_file_handler(const compressed_serialization_file_handler&) = delete; + + // Handle file read and write requests + bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override; + + // Get available memory or file size + // Preferably memory size if is already greater/equal to recommended to avoid additional file ops + usz get_size(const utils::serial& ar, usz recommended) const override; + void skip_until(utils::serial& ar) override; + + void finalize(utils::serial& ar) override; + +private: + usz read_at(utils::serial& ar, usz read_pos, void* data, usz size); + void initialize(utils::serial& ar); +}; + +template requires (std::is_same_v, fs::file>) +inline std::unique_ptr make_compressed_serialization_file_handler(File&& file) +{ + ensure(file); + return std::make_unique(std::forward(file)); } +// Null file serialization handler +struct null_serialization_file_handler : utils::serialization_file_handler +{ + explicit null_serialization_file_handler() noexcept + { + } + + // Handle file read and write requests + bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override; + + void finalize(utils::serial& ar) override; + + bool is_null() const override + { + return true; + } +}; + +inline std::unique_ptr make_null_serialization_file_handler() +{ + return std::make_unique(); +} + +bool load_and_check_reserved(utils::serial& ar, usz size); +bool is_savestate_version_compatible(const std::vector& data, bool is_boot_check); +std::vector get_savestate_versioning_data(fs::file&& file, std::string_view filepath); +bool is_savestate_compatible(fs::file&& file, std::string_view filepath); +std::vector read_used_savestate_versions(); +std::string get_savestate_file(std::string_view title_id, std::string_view boot_path, s64 abs_id, s64 rel_id); + diff --git a/rpcs3/Loader/TAR.cpp b/rpcs3/Loader/TAR.cpp index 74148260eb..4734061ad3 100644 --- a/rpcs3/Loader/TAR.cpp +++ b/rpcs3/Loader/TAR.cpp @@ -8,14 +8,30 @@ #include "TAR.h" #include "util/asm.hpp" -#include "util/serialization.hpp" + +#include "Emu/savestate_utils.hpp" #include +#include LOG_CHANNEL(tar_log, "TAR"); +fs::file make_file_view(const fs::file& file, u64 offset, u64 size); + +// File constructor tar_object::tar_object(const fs::file& file) - : m_file(file) + : m_file(std::addressof(file)) + , m_ar(nullptr) + , m_ar_tar_start(umax) +{ + ensure(*m_file); +} + +// Stream (pipe-like) constructor +tar_object::tar_object(utils::serial& ar) + : m_file(nullptr) + , m_ar(std::addressof(ar)) + , m_ar_tar_start(ar.pos) { } @@ -23,12 +39,14 @@ TARHeader tar_object::read_header(u64 offset) const { TARHeader header{}; - if (m_file.seek(offset) != offset) + if (m_ar) { + ensure(m_ar->pos == m_ar_tar_start + offset); + m_ar->serialize(header); return header; } - if (!m_file.read(header)) + if (m_file->read_at(offset, &header, sizeof(header)) != sizeof(header)) { std::memset(&header, 0, sizeof(header)); } @@ -38,13 +56,13 @@ TARHeader tar_object::read_header(u64 offset) const u64 octal_text_to_u64(std::string_view sv) { - u64 i = -1; + u64 i = umax; const auto ptr = std::from_chars(sv.data(), sv.data() + sv.size(), i, 8).ptr; // Range must be terminated with either NUL or space if (ptr == sv.data() + sv.size() || (*ptr && *ptr != ' ')) { - i = -1; + i = umax; } return i; @@ -53,114 +71,166 @@ u64 octal_text_to_u64(std::string_view sv) std::vector tar_object::get_filenames() { std::vector vec; + get_file(""); + for (auto it = m_map.cbegin(); it != m_map.cend(); ++it) { vec.push_back(it->first); } + return vec; } -fs::file tar_object::get_file(const std::string& path) +std::unique_ptr tar_object::get_file(const std::string& path, std::string* new_file_path) { - if (!m_file) return fs::file(); + std::unique_ptr m_out; + + auto emplace_single_entry = [&](usz& offset, const usz max_size) -> std::pair + { + if (offset >= max_size) + { + return {}; + } + + TARHeader header = read_header(offset); + offset += 512; + + u64 size = umax; + + std::string filename; + + if (std::memcmp(header.magic, "ustar", 5) == 0) + { + const std::string_view size_sv{header.size, std::size(header.size)}; + + size = octal_text_to_u64(size_sv); + + // Check for overflows and if surpasses file size + if ((header.name[0] || header.prefix[0]) && ~size >= 512 && max_size >= size && max_size - size >= offset) + { + // Cache size in native u64 format + static_assert(sizeof(size) < sizeof(header.size)); + std::memcpy(header.size, &size, 8); + + std::string_view prefix_name{header.prefix, std::size(header.prefix)}; + std::string_view name{header.name, std::size(header.name)}; + + prefix_name = prefix_name.substr(0, prefix_name.find_first_of('\0')); + name = name.substr(0, name.find_first_of('\0')); + + filename += prefix_name; + filename += name; + + // Save header and offset + m_map.insert_or_assign(filename, std::make_pair(offset, header)); + + if (new_file_path) + { + *new_file_path = filename; + } + + return { size, std::move(filename) }; + } + else + { + tar_log.error("tar_object::get_file() failed to convert header.size=%s, filesize=0x%x", size_sv, max_size); + } + } + else + { + tar_log.trace("tar_object::get_file() failed to parse header: offset=0x%x, filesize=0x%x", largest_offset, max_size); + } + + return { size, {} }; + }; if (auto it = m_map.find(path); it != m_map.end()) { u64 size = 0; std::memcpy(&size, it->second.second.size, sizeof(size)); - std::vector buf(size); - m_file.seek(it->second.first); - m_file.read(buf, size); - return fs::make_stream(std::move(buf)); + + if (m_file) + { + m_out = std::make_unique(); + m_out->set_reading_state(); + m_out->m_file_handler = make_uncompressed_serialization_file_handler(make_file_view(*m_file, it->second.first, size)); + } + else + { + m_out = std::make_unique(); + *m_out = std::move(*m_ar); + m_out->m_max_data = m_ar_tar_start + it->second.first + size; + } + + return m_out; } - else //continue scanning from last file entered + else if (m_ar && path.empty()) { - const u64 max_size = m_file.size(); + const u64 size = emplace_single_entry(largest_offset, m_ar->get_size(umax) - m_ar_tar_start).first; + + // Advance offset to next block + largest_offset += utils::align(size, 512); + } + // Continue scanning from last file entered + else if (m_file) + { + const u64 max_size = m_file->size(); while (largest_offset < max_size) { - TARHeader header = read_header(largest_offset); - - u64 size = -1; - - std::string filename; - - if (std::memcmp(header.magic, "ustar", 5) == 0) - { - const std::string_view size_sv{header.size, std::size(header.size)}; - - size = octal_text_to_u64(size_sv); - - // Check for overflows and if surpasses file size - if ((header.name[0] || header.prefix[0]) && size + 512 > size && max_size >= size + 512 && max_size - size - 512 >= largest_offset) - { - // Cache size in native u64 format - static_assert(sizeof(size) < sizeof(header.size)); - std::memcpy(header.size, &size, 8); - - std::string_view prefix_name{header.prefix, std::size(header.prefix)}; - std::string_view name{header.name, std::size(header.name)}; - - prefix_name = prefix_name.substr(0, prefix_name.find_first_of('\0')); - name = name.substr(0, name.find_first_of('\0')); - - filename += prefix_name; - filename += name; - - // Save header and offset - m_map.insert_or_assign(filename, std::make_pair(largest_offset + 512, header)); - } - else - { - // Invalid - size = -1; - tar_log.error("tar_object::get_file() failed to convert header.size=%s, filesize=0x%x", size_sv, max_size); - } - } - else - { - tar_log.trace("tar_object::get_file() failed to parse header: offset=0x%x, filesize=0x%x", largest_offset, max_size); - } + const auto [size, filename] = emplace_single_entry(largest_offset, max_size); if (size == umax) { - largest_offset += 512; continue; } // Advance offset to next block - largest_offset += utils::align(size, 512) + 512; + largest_offset += utils::align(size, 512); if (!path.empty() && path == filename) { - // Path is equal, read file and advance offset to start of next block - std::vector buf(size); - - if (m_file.read(buf, size)) - { - return fs::make_stream(std::move(buf)); - } - - tar_log.error("tar_object::get_file() failed to read file entry %s (size=0x%x)", filename, size); - largest_offset -= utils::align(size, 512); + // Path is equal, return handle to the file data + return get_file(path); } } - - return fs::file(); } + + return m_out; } bool tar_object::extract(std::string prefix_path, bool is_vfs) { - if (!m_file) return false; + std::vector filedata_buffer(0x80'0000); + std::span filedata_span{filedata_buffer.data(), filedata_buffer.size()}; - get_file(""); // Make sure we have scanned all files + auto iter = m_map.begin(); - for (auto& iter : m_map) + auto get_next = [&](bool is_first) { - const TARHeader& header = iter.second.second; - const std::string& name = iter.first; + if (m_ar) + { + ensure(!is_first || m_map.empty()); // Must be empty on first call + std::string name_iter; + get_file("", &name_iter); // Get next entry + return m_map.find(name_iter); + } + else if (is_first) + { + get_file(""); // Scan entries + return m_map.begin(); + } + else + { + return std::next(iter); + } + }; + + for (iter = get_next(true); iter != m_map.end(); iter = get_next(false)) + { + const TARHeader& header = iter->second.second; + const std::string& name = iter->first; std::string result = name; @@ -210,15 +280,53 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs) return false; } - auto data = get_file(name).release(); + // For restoring m_ar->m_max_data + usz restore_limit = umax; + + if (!m_file) + { + // Restore m_ar (remove limit) + restore_limit = m_ar->m_max_data; + } + + std::unique_ptr file_data = get_file(name); fs::file file(result, fs::rewrite); - if (file) + if (file && file_data) { - file.write(static_cast>*>(data.get())->obj); + while (true) + { + const usz unread_size = file_data->try_read(filedata_span); + + if (unread_size == 0) + { + file.write(filedata_span.data(), filedata_span.size()); + continue; + } + + // Tail data + + if (usz read_size = filedata_span.size() - unread_size) + { + ensure(file_data->try_read(filedata_span.first(read_size)) == 0); + file.write(filedata_span.data(), read_size); + } + + break; + } + file.close(); + file_data->seek_pos(m_ar_tar_start + largest_offset, true); + + if (!m_file) + { + // Restore m_ar + *m_ar = std::move(*file_data); + m_ar->m_max_data = restore_limit; + } + if (mtime != umax && !fs::utime(result, atime, mtime)) { tar_log.error("TAR Loader: fs::utime failed on %s (%s)", result, fs::g_tls_error); @@ -229,6 +337,13 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs) break; } + if (!m_file) + { + // Restore m_ar + *m_ar = std::move(*file_data); + m_ar->m_max_data = restore_limit; + } + const auto old_error = fs::g_tls_error; tar_log.error("TAR Loader: failed to write file %s (%s) (fs::exists=%s)", name, old_error, fs::exists(result)); return false; @@ -259,32 +374,14 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs) return true; } -void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func, std::string full_path) +void tar_object::save_directory(const std::string& target_path, utils::serial& ar, const process_func& func, std::vector&& entries, bool has_evaluated_results, usz src_dir_pos) { - const std::string& target_path = full_path.empty() ? src_dir : full_path; + const bool is_null = ar.m_file_handler && ar.m_file_handler->is_null(); + const bool reuse_entries = !is_null || has_evaluated_results; - fs::stat_t stat{}; - if (!fs::get_stat(target_path, stat)) + if (reuse_entries) { - return; - } - - if (stat.is_directory) - { - bool has_items = false; - - for (auto& entry : fs::dir(target_path)) - { - if (entry.name.find_first_not_of('.') == umax) continue; - - save_directory(src_dir, ar, func, target_path + '/' + entry.name); - has_items = true; - } - - if (has_items) - { - return; - } + ensure(!entries.empty()); } auto write_octal = [](char* ptr, u64 i) @@ -303,60 +400,224 @@ void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, c } }; - std::string saved_path{target_path.data() + src_dir.size(), target_path.size() - src_dir.size()}; - - const u64 old_size = ar.data.size(); - ar.data.resize(old_size + sizeof(TARHeader)); - - if (!stat.is_directory) + auto save_file = [&](const fs::stat_t& file_stat, const std::string& file_name) { - fs::file fd(target_path); - - const u64 old_size2 = ar.data.size(); - - if (func) + if (!file_stat.size) { - // Use custom function for file saving if provided - // Allows for example to compress PNG files as JPEG in the TAR itself - if (!func(fd, saved_path, ar)) + return; + } + + if (is_null && !func) + { + ar.pos += utils::align(file_stat.size, 512); + return; + } + + if (fs::file fd{file_name}) + { + const u64 old_pos = ar.pos; + const usz old_size = ar.data.size(); + + if (func) + { + std::string saved_path{&::at32(file_name, src_dir_pos), file_name.size() - src_dir_pos}; + + // Use custom function for file saving if provided + // Allows for example to compress PNG files as JPEG in the TAR itself + if (!func(fd, saved_path, ar)) + { + // Revert (this entry should not be included if func returns false) + + if (is_null) + { + ar.pos = old_pos; + return; + } + + ar.data.resize(old_size); + ar.seek_end(); + return; + } + + if (is_null) + { + // Align + ar.pos += utils::align(ar.pos - old_pos, 512); + return; + } + } + else + { + constexpr usz transfer_block_size = 0x100'0000; + + for (usz read_index = 0; read_index < file_stat.size; read_index += transfer_block_size) + { + const usz read_size = std::min(transfer_block_size, file_stat.size - read_index); + + // Read file data + ar.data.resize(ar.data.size() + read_size); + ensure(fd.read_at(read_index, ar.data.data() + old_size, read_size) == read_size); + + // Set position to the end of data, so breathe() would work correctly + ar.seek_end(); + + // Allow flushing to file if needed + ar.breathe(); + } + } + + // Align + const usz diff = ar.pos - old_pos; + ar.data.resize(ar.data.size() + utils::align(diff, 512) - diff); + ar.seek_end(); + + fd.close(); + ensure(fs::utime(file_name, file_stat.atime, file_stat.mtime)); + } + else + { + ensure(false); + } + }; + + auto save_header = [&](const fs::stat_t& stat, const std::string& name) + { + static_assert(sizeof(TARHeader) == 512); + + std::string_view saved_path{name.size() == src_dir_pos ? name.c_str() : &::at32(name, src_dir_pos), name.size() - src_dir_pos}; + + if (is_null) + { + ar.pos += sizeof(TARHeader); + return; + } + + if (usz pos = saved_path.find_first_not_of(fs::delim); pos != umax) + { + saved_path = saved_path.substr(pos, saved_path.size()); + } + else + { + // Target the destination directory, I do not know if this is compliant with TAR format + saved_path = "/"sv; + } + + TARHeader header{}; + std::memcpy(header.magic, "ustar ", 6); + + // Prefer saving to name field as much as we can + // If it doesn't fit, save 100 characters at name and 155 characters preceding to it at max + const u64 prefix_size = std::clamp(saved_path.size(), 100, 255) - 100; + std::memcpy(header.prefix, saved_path.data(), prefix_size); + const u64 name_size = std::min(saved_path.size(), 255) - prefix_size; + std::memcpy(header.name, saved_path.data() + prefix_size, name_size); + + write_octal(header.size, stat.is_directory ? 0 : stat.size); + write_octal(header.mtime, stat.mtime); + write_octal(header.padding, stat.atime); + header.filetype = stat.is_directory ? '5' : '0'; + + ar(header); + ar.breathe(); + }; + + fs::stat_t stat{}; + + if (src_dir_pos == umax) + { + // First call, get source directory string size so it can be cut from entry paths + src_dir_pos = target_path.size(); + } + + if (has_evaluated_results) + { + // Save from cached data by previous call + for (auto&& entry : entries) + { + ensure(entry.name.starts_with(target_path)); + save_header(entry, entry.name); + + if (!entry.is_directory) + { + save_file(entry, entry.name); + } + } + } + else + { + if (entries.empty()) + { + if (!fs::get_stat(target_path, stat)) { - // Revert (this entry should not be included if func returns false) - ar.data.resize(old_size); return; } + + save_header(stat, target_path); + + // Optimization: avoid saving to list if this is not an evaluation call + if (is_null) + { + static_cast(entries.emplace_back()) = stat; + entries.back().name = target_path; + } } else { - ar.data.resize(ar.data.size() + stat.size); - ensure(fd.read(ar.data.data() + old_size2, stat.size) == stat.size); + stat = entries.back(); + save_header(stat, entries.back().name); } - // Align - ar.data.resize(old_size2 + utils::align(ar.data.size() - old_size2, 512)); + if (stat.is_directory) + { + bool exists = false; - fd.close(); - fs::utime(target_path, stat.atime, stat.mtime); + for (auto&& entry : fs::dir(target_path)) + { + exists = true; + + if (entry.name.find_first_not_of('.') == umax) + { + continue; + } + + entry.name = target_path.ends_with('/') ? target_path + entry.name : target_path + '/' + entry.name; + + if (!entry.is_directory) + { + save_header(entry, entry.name); + save_file(entry, entry.name); + + // TAR is an old format which does not depend on previous data so memory ventilation is trivial here + ar.breathe(); + + entries.emplace_back(std::move(entry)); + } + else + { + if (!is_null) + { + // Optimization: avoid saving to list if this is not an evaluation call + entries.clear(); + } + + entries.emplace_back(std::move(entry)); + save_directory(::as_rvalue(entries.back().name), ar, func, std::move(entries), false, src_dir_pos); + } + } + + ensure(exists); + } + else + { + fs::dir_entry entry{}; + entry.name = target_path; + static_cast(entry) = stat; + + save_file(entry, entry.name); + } + + ar.breathe(); } - - TARHeader header{}; - std::memcpy(header.magic, "ustar ", 6); - - // Prefer saving to name field as much as we can - // If it doesn't fit, save 100 characters at name and 155 characters preceding to it at max - const u64 prefix_size = std::clamp(saved_path.size(), 100, 255) - 100; - std::memcpy(header.prefix, saved_path.data(), prefix_size); - const u64 name_size = std::min(saved_path.size(), 255) - prefix_size; - std::memcpy(header.name, saved_path.data() + prefix_size, name_size); - - write_octal(header.size, stat.is_directory ? 0 : stat.size); - write_octal(header.mtime, stat.mtime); - write_octal(header.padding, stat.atime); - header.filetype = stat.is_directory ? '5' : '0'; - - std::memcpy(ar.data.data() + old_size, &header, sizeof(header)); - - // TAR is an old format which does not depend on previous data so memory ventilation is trivial here - ar.breathe(); } bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file) diff --git a/rpcs3/Loader/TAR.h b/rpcs3/Loader/TAR.h index 352220d497..ba69b783b8 100644 --- a/rpcs3/Loader/TAR.h +++ b/rpcs3/Loader/TAR.h @@ -1,5 +1,7 @@ #pragma once +#include "util/types.hpp" + #include struct TARHeader @@ -15,11 +17,14 @@ struct TARHeader char dontcare2[82]; char prefix[155]; char padding[12]; // atime for RPCS3 + + ENABLE_BITWISE_SERIALIZATION; }; namespace fs { class file; + struct dir_entry; } namespace utils @@ -29,7 +34,9 @@ namespace utils class tar_object { - const fs::file& m_file; + const fs::file* m_file; + utils::serial* m_ar; + const usz m_ar_tar_start; usz largest_offset = 0; // We store the largest offset so we can continue to scan from there. std::map> m_map{}; // Maps path to offset of file data and its header @@ -38,10 +45,11 @@ class tar_object public: tar_object(const fs::file& file); + tar_object(utils::serial& ar); std::vector get_filenames(); - fs::file get_file(const std::string& path); + std::unique_ptr get_file(const std::string& path, std::string* new_file_path = nullptr); using process_func = std::function; @@ -49,7 +57,7 @@ public: // Allow to optionally specify explicit mount point (which may be directory meant for extraction) bool extract(std::string prefix_path = {}, bool is_vfs = false); - static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::string append_path = {}); + static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::vector&& = std::vector{}, bool has_evaluated_results = false, usz src_dir_pos = umax); }; bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file = {}); diff --git a/rpcs3/rpcs3qt/game_list_frame.cpp b/rpcs3/rpcs3qt/game_list_frame.cpp index 6dd6c05514..56f9e600b6 100644 --- a/rpcs3/rpcs3qt/game_list_frame.cpp +++ b/rpcs3/rpcs3qt/game_list_frame.cpp @@ -1078,9 +1078,9 @@ void game_list_frame::ShowContextMenu(const QPoint &pos) }); } - extern bool is_savestate_compatible(fs::file&& file); + extern bool is_savestate_compatible(fs::file&& file, std::string_view filepath); - if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate))) + if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate), sstate)) { QAction* boot_state = menu.addAction(is_current_running_game ? tr("&Reboot with savestate") diff --git a/rpcs3/rpcs3qt/main_window.cpp b/rpcs3/rpcs3qt/main_window.cpp index 0c8bfad0c3..9a1abe763a 100644 --- a/rpcs3/rpcs3qt/main_window.cpp +++ b/rpcs3/rpcs3qt/main_window.cpp @@ -56,6 +56,7 @@ #include "Emu/vfs_config.h" #include "Emu/System.h" #include "Emu/system_utils.hpp" +#include "Emu/savestate_utils.hpp" #include "Crypto/unpkg.h" #include "Crypto/unself.h" @@ -589,7 +590,7 @@ void main_window::BootSavestate() } const QString file_path = QFileDialog::getOpenFileName(this, tr("Select Savestate To Boot"), qstr(fs::get_cache_dir() + "/savestates/"), tr( - "Savestate files (*.SAVESTAT);;" + "Savestate files (*.SAVESTAT *.SAVESTAT.gz);;" "All files (*.*)"), Q_NULLPTR, QFileDialog::DontResolveSymlinks); @@ -1529,7 +1530,18 @@ void main_window::HandlePupInstallation(const QString& file_path, const QString& { for (const auto& update_filename : update_filenames) { - fs::file update_file = update_files.get_file(update_filename); + auto update_file_stream = update_files.get_file(update_filename); + + if (update_file_stream->m_file_handler) + { + // Forcefully read all the data + update_file_stream->pop(); + update_file_stream->pos = umax; + update_file_stream->pos /= 2; // Avoid internal overflows + update_file_stream->m_file_handler->handle_file_op(*update_file_stream, update_file_stream->pos, 1, nullptr); + } + + fs::file update_file = fs::make_stream(std::move(update_file_stream->data)); SCEDecrypter self_dec(update_file); self_dec.LoadHeaders(); @@ -3558,7 +3570,7 @@ main_window::drop_type main_window::IsValidFile(const QMimeData& md, QStringList type = drop_type::drop_rrc; } // The emulator allows to execute ANY filetype, just not from drag-and-drop because it is confusing to users - else if (suffix_lo == "savestat" || suffix_lo == "sprx" || suffix_lo == "self" || suffix_lo == "bin" || suffix_lo == "prx" || suffix_lo == "elf" || suffix_lo == "o") + else if (path.toLower().endsWith(".savestat.gz") || suffix_lo == "savestat" || suffix_lo == "sprx" || suffix_lo == "self" || suffix_lo == "bin" || suffix_lo == "prx" || suffix_lo == "elf" || suffix_lo == "o") { type = drop_type::drop_game; } diff --git a/rpcs3/util/serialization.hpp b/rpcs3/util/serialization.hpp index f52e3a17ba..29019d830c 100644 --- a/rpcs3/util/serialization.hpp +++ b/rpcs3/util/serialization.hpp @@ -39,8 +39,31 @@ namespace utils serialization_file_handler() = default; virtual ~serialization_file_handler() = default; + // Handle read/write operations virtual bool handle_file_op(serial& ar, usz pos, usz size, const void* data = nullptr) = 0; - virtual usz get_size(const utils::serial& ar, usz recommended = umax) const = 0; + + // Obtain data size (targets to be only higher than 'recommended' and thus may not be accurate) + virtual usz get_size(const utils::serial& /*ar*/, usz /*recommended*/) const + { + return 0; + } + + // Skip reading some (compressed) data + virtual void skip_until(utils::serial& /*ar*/) + { + } + + // Detect empty stream (TODO: Clean this, instead perhaps use a magic static representing empty stream) + virtual bool is_null() const + { + return false; + } + + virtual void finalize_block(utils::serial& /*ar*/) + { + } + + virtual void finalize(utils::serial&) = 0; }; struct serial @@ -48,12 +71,16 @@ namespace utils std::vector data; usz data_offset = 0; usz pos = 0; + usz m_max_data = umax; bool m_is_writing = true; bool m_avoid_large_prefetch = false; std::unique_ptr m_file_handler; serial() noexcept = default; serial(const serial&) = delete; + serial& operator=(const serial&) = delete; + explicit serial(serial&&) noexcept = default; + serial& operator=(serial&&) noexcept = default; ~serial() noexcept = default; // Checks if this instance is currently used for serialization @@ -80,27 +107,34 @@ namespace utils return true; } + if (m_file_handler && m_file_handler->is_null()) + { + // Instead of doing nothing at all, increase pos so it would be possible to estimate memory requirements + pos += size; + return true; + } + // Overflow check - ensure(~pos >= size); + ensure(~pos >= size - 1); if (is_writing()) { ensure(pos >= data_offset); const auto ptr = reinterpret_cast(memory_provider()); - data.insert(data.begin() + pos - data_offset, ptr, ptr + size); + data.insert(data.begin() + (pos - data_offset), ptr, ptr + size); pos += size; return true; } - if (data.empty() || pos < data_offset || pos + size > data.size() + data_offset) + if (data.empty() || pos < data_offset || pos + (size - 1) > (data.size() - 1) + data_offset) { // Load from file ensure(m_file_handler); ensure(m_file_handler->handle_file_op(*this, pos, size, nullptr)); - ensure(!data.empty() && pos >= data_offset && pos + size <= data.size() + data_offset); + ensure(!data.empty() && pos >= data_offset && pos + (size - 1) <= (data.size() - 1) + data_offset); } - std::memcpy(const_cast(static_cast(memory_provider())), data.data() + pos - data_offset, size); + std::memcpy(const_cast(static_cast(memory_provider())), data.data() + (pos - data_offset), size); pos += size; return true; } @@ -169,7 +203,8 @@ namespace utils } // std::vector, std::basic_string - template requires FastRandomAccess && ListAlike + // Discourage using std::pair/tuple with vectors because it eliminates the possibility of bitwise optimization + template requires FastRandomAccess && ListAlike && (!TupleAlike) bool serialize(T& obj) { if (is_writing()) @@ -194,6 +229,13 @@ namespace utils return true; } + obj.clear(); + + if (m_file_handler && m_file_handler->is_null()) + { + return true; + } + usz size = 0; if (!deserialize_vle(size)) { @@ -211,7 +253,6 @@ namespace utils else { // TODO: Postpone resizing to after file bounds checks - obj.clear(); obj.resize(size); for (auto&& value : obj) @@ -270,6 +311,11 @@ namespace utils obj.clear(); + if (m_file_handler && m_file_handler->is_null()) + { + return true; + } + usz size = 0; if (!deserialize_vle(size)) { @@ -326,16 +372,23 @@ namespace utils } // Wrapper for serialize(T&), allows to pass multiple objects at once - template - bool operator()(Args&&... args) + template requires (sizeof...(Args) != 0) + bool operator()(Args&&... args) noexcept { return ((AUDIT(!std::is_const_v> || is_writing()) , serialize(const_cast&>(static_cast(args)))), ...); } + // Code style utility, for when utils::serial is a pointer for example + template requires (sizeof...(Args) > 1 || !(std::is_convertible_v && ...)) + bool serialize(Args&&... args) + { + return this->operator()(std::forward(args)...); + } + // Convert serialization manager to deserializion manager // If no arg is provided reuse saved buffer - void set_reading_state(std::vector&& _data = std::vector{}) + void set_reading_state(std::vector&& _data = std::vector{}, bool avoid_large_prefetch = false) { if (!_data.empty()) { @@ -343,7 +396,7 @@ namespace utils } m_is_writing = false; - m_avoid_large_prefetch = false; + m_avoid_large_prefetch = avoid_large_prefetch; pos = 0; data_offset = 0; } @@ -365,15 +418,19 @@ namespace utils return pos; } - usz seek_pos(usz val, bool empty_data = false) + usz seek_pos(usz val, bool cleanup = false) { const usz old_pos = std::exchange(pos, val); - if (empty_data || data.empty()) + if (cleanup || data.empty()) { // Relocate future data - data.clear(); - data_offset = pos; + if (m_file_handler) + { + m_file_handler->skip_until(*this); + } + + breathe(); } return old_pos; @@ -403,7 +460,7 @@ namespace utils template requires (std::is_copy_constructible_v>) && (std::is_constructible_v> || Bitcopy> || std::is_constructible_v, stx::exact_t> || TupleAlike>) - operator T() + operator T() noexcept { AUDIT(!is_writing()); @@ -433,6 +490,13 @@ namespace utils } } + // Code style utility wrapper for operator T() + template + T pop() + { + return this->operator T(); + } + void swap_handler(serial& ar) { std::swap(ar.m_file_handler, this->m_file_handler); @@ -440,7 +504,40 @@ namespace utils usz get_size(usz recommended = umax) const { - return m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size(); + recommended = std::min(recommended, m_max_data); + return std::min(m_max_data, m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size()); + } + + template requires (Bitcopy) + usz predict_object_size(const T&) + { + return sizeof(T); + } + + template requires FastRandomAccess && (!ListAlike) && (!Bitcopy) + usz predict_object_size(const T& obj) + { + return std::size(obj) * sizeof(obj[0]); + } + + template requires (std::is_copy_constructible_v> && std::is_constructible_v>) + usz try_read(T&& obj) + { + if (is_writing()) + { + return 0; + } + + const usz end_pos = pos + predict_object_size(std::forward(obj)); + const usz size = get_size(end_pos); + + if (size >= end_pos) + { + serialize(std::forward(obj)); + return 0; + } + + return end_pos - size; } template requires (std::is_copy_constructible_v && std::is_constructible_v && Bitcopy) @@ -457,14 +554,27 @@ namespace utils if (size >= end_pos) { - u8 buf[sizeof(type)]{}; - ensure(raw_serialize(buf, sizeof(buf))); - return {true, std::bit_cast(buf)}; + return {true, this->operator T()}; } return {}; } + void patch_raw_data(usz pos, const void* data, usz size) + { + if (m_file_handler && m_file_handler->is_null()) + { + return; + } + + if (!size) + { + return; + } + + std::memcpy(&::at32(this->data, pos - data_offset + size - 1) - (size - 1), data, size); + } + // Returns true if valid, can be invalidated by setting pos to umax // Used when an invalid state is encountered somewhere in a place we can't check success code such as constructor) bool is_valid() const