RPCS3/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2024-11-22 02:32:36 +01:00
Code Issues Releases Wiki Activity

Savestates: Compressed state files

Browse Source
This commit is contained in:
Eladash 2023-11-15 21:07:42 +02:00 committed by Elad Ashkenazi
parent 91dbd92193
commit f60bdbaece
27 changed files with 1377 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Utilities/File.cpp
View File

@ -2277,6 +2277,12 @@ bool fs::pending_file::open(std::string_view path)
if (file.open(m_path, fs::create + fs::write + fs::read + fs::excl))
{
#ifdef _WIN32
// Auto-delete pending log file
FILE_DISPOSITION_INFO disp;
disp.DeleteFileW = true;
SetFileInformationByHandle(file.get_handle(), FileDispositionInfo, &disp, sizeof(disp));
#endif
m_dest = path;
break;
}
@ -2314,6 +2320,14 @@ bool fs::pending_file::commit(bool overwrite)
}
#endif
#ifdef _WIN32
// Disable auto-delete
FILE_DISPOSITION_INFO disp;
disp.DeleteFileW = false;
SetFileInformationByHandle(file.get_handle(), FileDispositionInfo, &disp, sizeof(disp));
#endif
file.close();
#ifdef _WIN32

16
rpcs3/Crypto/unzip.h
View File

@ -2,32 +2,32 @@
std::vector<u8> unzip(const void* src, usz size);
template<typename T>
std::vector<u8> unzip(const T& src)
template <typename T>
inline std::vector<u8> unzip(const T& src)
{
return unzip(src.data(), src.size());
}
bool unzip(const void* src, usz size, fs::file& out);
template<typename T>
bool unzip(const std::vector<u8>& src, fs::file& out)
template <typename T>
inline bool unzip(const std::vector<u8>& src, fs::file& out)
{
return unzip(src.data(), src.size(), out);
}
std::vector<u8> zip(const void* src, usz size);
template<typename T>
std::vector<u8> zip(const T& src)
template <typename T>
inline std::vector<u8> zip(const T& src)
{
return zip(src.data(), src.size());
}
bool zip(const void* src, usz size, fs::file& out);
template<typename T>
bool zip(const T& src, fs::file& out)
template <typename T>
inline bool zip(const T& src, fs::file& out)
{
return zip(src.data(), src.size(), out);
}

4
rpcs3/Emu/Cell/Modules/sceNpTrophy.cpp
View File

@ -43,7 +43,7 @@ struct trophy_context_t
trophy_context_t() = default;
trophy_context_t(utils::serial& ar)
: trp_name(ar.operator std::string())
: trp_name(ar.pop<std::string>())
{
std::string trophy_path = vfs::get(Emu.GetDir() + "TROPDIR/" + trp_name + "/TROPHY.TRP");
fs::file trp_stream(trophy_path);
@ -55,7 +55,7 @@ struct trophy_context_t
trp_stream.open(trophy_path);
}
if (!ar.operator bool())
if (!ar.pop<bool>())
{
ar(read_only);
return;

3
rpcs3/Emu/Cell/Modules/sys_io_.cpp
View File

@ -52,6 +52,7 @@ void config_event_entry(ppu_thread& ppu)
{
if (ppu.is_stopped())
{
ppu.state += cpu_flag::again;
return;
}
@ -73,6 +74,8 @@ void config_event_entry(ppu_thread& ppu)
}
}
sys_io.notice("config_event_entry(): Exited with the following error code: %s", CellError{static_cast<u32>(ppu.gpr[3])});
ppu_execute<&sys_ppu_thread_exit>(ppu, 0);
}

15
rpcs3/Emu/Cell/PPUModule.cpp
View File

@ -358,7 +358,7 @@ static void ppu_initialize_modules(ppu_linkage_info* link, utils::serial* ar = n
while (true)
{
const std::string name = ar.operator std::string();
const std::string name = ar.pop<std::string>();
if (name.empty())
{
@ -370,10 +370,10 @@ static void ppu_initialize_modules(ppu_linkage_info* link, utils::serial* ar = n
auto& variable = _module->variables;
for (u32 i = 0, end = ar.operator usz(); i < end; i++)
for (u32 i = 0, end = ar.pop<usz>(); i < end; i++)
{
auto* ptr = &::at32(variable, ar.operator u32());
ptr->addr = ar.operator u32();
auto* ptr = &::at32(variable, ar.pop<u32>());
ptr->addr = ar.pop<u32>();
ensure(!!ptr->var);
}
}
@ -1052,7 +1052,12 @@ void init_ppu_functions(utils::serial* ar, bool full = false)
if (ar)
{
ensure(vm::check_addr(g_fxo->init<ppu_function_manager>(*ar)->addr));
const u32 addr = g_fxo->init<ppu_function_manager>(*ar)->addr;
if (addr % 0x1000 || !vm::check_addr(addr))
{
fmt::throw_exception("init_ppu_functions(): Failure to initialize function manager. (addr=0x%x, %s)", addr, *ar);
}
}
else
g_fxo->init<ppu_function_manager>();

31
rpcs3/Emu/Cell/PPUThread.cpp
View File

@ -2352,6 +2352,16 @@ void ppu_thread::serialize_common(utils::serial& ar)
ar(gpr, fpr, cr, fpscr.bits, lr, ctr, vrsave, cia, xer, sat, nj, prio.raw().all);
if (cia % 4 || !vm::check_addr(cia))
{
fmt::throw_exception("Failed to serialize PPU thread ID=0x%x (cia=0x%x, ar=%s)", this->id, cia, ar);
}
if (ar.is_writing())
{
ppu_log.notice("Saving PPU Thread [0x%x: %s]: cia=0x%x, state=%s", id, *ppu_tname.load(), cia, +state);
}
ar(optional_savestate_state, vr);
if (optional_savestate_state->data.empty())
@ -2364,7 +2374,7 @@ ppu_thread::ppu_thread(utils::serial& ar)
: cpu_thread(idm::last_id()) // last_id() is showed to constructor on serialization
, stack_size(ar)
, stack_addr(ar)
, joiner(ar.operator ppu_join_status())
, joiner(ar.pop<ppu_join_status>())
, entry_func(std::bit_cast<ppu_func_opd_t, u64>(ar))
, is_interrupt_thread(ar)
{
@ -2397,7 +2407,7 @@ ppu_thread::ppu_thread(utils::serial& ar)
}
};
switch (const u32 status = ar.operator u32())
switch (const u32 status = ar.pop<u32>())
{
case PPU_THREAD_STATUS_IDLE:
{
@ -2490,7 +2500,9 @@ ppu_thread::ppu_thread(utils::serial& ar)
state += cpu_flag::memory;
}
ppu_tname = make_single<std::string>(ar.operator std::string());
ppu_tname = make_single<std::string>(ar.pop<std::string>());
ppu_log.notice("Loading PPU Thread [0x%x: %s]: cia=0x%x, state=%s", id, *ppu_tname.load(), cia, +state);
}
void ppu_thread::save(utils::serial& ar)
@ -2506,12 +2518,6 @@ void ppu_thread::save(utils::serial& ar)
_joiner = ppu_join_status::joinable;
}
if (state & cpu_flag::again)
{
std::memcpy(&gpr[3], syscall_args, sizeof(syscall_args));
cia -= 4;
}
ar(stack_size, stack_addr, _joiner, entry, is_interrupt_thread);
serialize_common(ar);
@ -2685,6 +2691,13 @@ void ppu_thread::fast_call(u32 addr, u64 rtoc, bool is_thread_entry)
std::memcpy(syscall_args, &gpr[3], sizeof(syscall_args));
}
if (!old_cia && state & cpu_flag::again)
{
// Fixup argument registers and CIA for reloading
std::memcpy(&gpr[3], syscall_args, sizeof(syscall_args));
cia -= 4;
}
current_function = old_func;
g_tls_log_prefix = old_fmt;
state -= cpu_flag::ret;

8
rpcs3/Emu/Cell/SPUThread.cpp
View File

@ -1981,7 +1981,7 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group)
: cpu_thread(idm::last_id())
, group(group)
, index(ar)
, thread_type(group ? spu_type::threaded : ar.operator u8() ? spu_type::isolated : spu_type::raw)
, thread_type(group ? spu_type::threaded : ar.pop<u8>() ? spu_type::isolated : spu_type::raw)
, shm(ensure(vm::get(vm::spu)->peek(vm_offset()).second))
, ls(map_ls(*this->shm))
, option(ar)
@ -2029,12 +2029,12 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group)
for (auto& pair : spuq)
{
ar(pair.first);
pair.second = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.operator u32());
pair.second = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.pop<u32>());
}
for (auto& q : spup)
{
q = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.operator u32());
q = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.pop<u32>());
}
}
else
@ -2042,7 +2042,7 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group)
for (spu_int_ctrl_t& ctrl : int_ctrl)
{
ar(ctrl.mask, ctrl.stat);
ctrl.tag = idm::get_unlocked<lv2_obj, lv2_int_tag>(ar.operator u32());
ctrl.tag = idm::get_unlocked<lv2_obj, lv2_int_tag>(ar.pop<u32>());
}
g_raw_spu_ctr++;

2
rpcs3/Emu/Cell/lv2/sys_event.cpp
View File

@ -59,7 +59,7 @@ void lv2_event_queue::save_ptr(utils::serial& ar, lv2_event_queue* q)
std::shared_ptr<lv2_event_queue> lv2_event_queue::load_ptr(utils::serial& ar, std::shared_ptr<lv2_event_queue>& queue, std::string_view msg)
{
const u32 id = ar.operator u32();
const u32 id = ar.pop<u32>();
if (!id)
{

2
rpcs3/Emu/Cell/lv2/sys_fs.cpp
View File

@ -445,7 +445,7 @@ lv2_file::lv2_file(utils::serial& ar)
g_fxo->get<loaded_npdrm_keys>().npdrm_fds.raw() += type != lv2_file_type::regular;
if (ar.operator bool()) // see lv2_file::save in_mem
if (ar.pop<bool>()) // see lv2_file::save in_mem
{
const fs::stat_t stat = ar;

4
rpcs3/Emu/Cell/lv2/sys_lwcond.cpp
View File

@ -12,10 +12,10 @@
LOG_CHANNEL(sys_lwcond);
lv2_lwcond::lv2_lwcond(utils::serial& ar)
: name(ar.operator be_t<u64>())
: name(ar.pop<be_t<u64>>())
, lwid(ar)
, protocol(ar)
, control(ar.operator decltype(control)())
, control(ar.pop<decltype(control)>())
{
}

4
rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp
View File

@ -12,8 +12,8 @@ LOG_CHANNEL(sys_lwmutex);
lv2_lwmutex::lv2_lwmutex(utils::serial& ar)
: protocol(ar)
, control(ar.operator decltype(control)())
, name(ar.operator be_t<u64>())
, control(ar.pop<decltype(control)>())
, name(ar.pop<be_t<u64>>())
{
ar(lv2_control.raw().signaled);
}

4
rpcs3/Emu/Cell/lv2/sys_mmapper.cpp
View File

@ -50,7 +50,7 @@ lv2_memory::lv2_memory(utils::serial& ar)
, flags(ar)
, key(ar)
, pshared(ar)
, ct(lv2_memory_container::search(ar.operator u32()))
, ct(lv2_memory_container::search(ar.pop<u32>()))
, shm([&](u32 addr)
{
if (addr)
@ -61,7 +61,7 @@ lv2_memory::lv2_memory(utils::serial& ar)
const auto _shm = std::make_shared<utils::shm>(size, 1);
ar(std::span(_shm->map_self(), size));
return _shm;
}(ar.operator u32()))
}(ar.pop<u32>()))
, counter(ar)
{
#ifndef _WIN32

2
rpcs3/Emu/Cell/lv2/sys_overlay.cpp
View File

@ -72,7 +72,7 @@ fs::file make_file_view(fs::file&& file, u64 offset, u64 size);
std::shared_ptr<void> lv2_overlay::load(utils::serial& ar)
{
const std::string path = vfs::get(ar.operator std::string());
const std::string path = vfs::get(ar.pop<std::string>());
const s64 offset = ar;
std::shared_ptr<lv2_overlay> ovlm;

2
rpcs3/Emu/Cell/lv2/sys_prx.cpp
View File

@ -292,7 +292,7 @@ std::shared_ptr<void> lv2_prx::load(utils::serial& ar)
{
[[maybe_unused]] const s32 version = GET_SERIALIZATION_VERSION(lv2_prx_overlay);
const std::string path = vfs::get(ar.operator std::string());
const std::string path = vfs::get(ar.pop<std::string>());
const s64 offset = ar;
const u32 state = ar;

12
rpcs3/Emu/Cell/lv2/sys_spu.cpp
View File

@ -202,13 +202,13 @@ void sys_spu_image::deploy(u8* loc, std::span<const sys_spu_segment> segs, bool
}
lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept
: name(ar.operator std::string())
: name(ar.pop<std::string>())
, id(idm::last_id())
, max_num(ar)
, mem_size(ar)
, type(ar) // SPU Thread Group Type
, ct(lv2_memory_container::search(ar))
, has_scheduler_context(ar.operator u8())
, has_scheduler_context(ar.pop<u8>())
, max_run(ar)
, init(ar)
, prio([&ar]()
@ -219,12 +219,12 @@ lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept
return prio;
}())
, run_state(ar.operator spu_group_status())
, run_state(ar.pop<spu_group_status>())
, exit_status(ar)
{
for (auto& thread : threads)
{
if (ar.operator u8())
if (ar.pop<bool>())
{
ar(id_manager::g_id);
thread = std::make_shared<named_thread<spu_thread>>(ar, this);
@ -239,7 +239,7 @@ lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept
for (auto ep : {&ep_run, &ep_exception, &ep_sysmodule})
{
*ep = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.operator u32());
*ep = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.pop<u32>());
}
waiter_spu_index = -1;
@ -328,7 +328,7 @@ void lv2_spu_group::save(utils::serial& ar)
lv2_spu_image::lv2_spu_image(utils::serial& ar)
: e_entry(ar)
, segs(ar.operator decltype(segs)())
, segs(ar.pop<decltype(segs)>())
, nsegs(ar)
{
}

2
rpcs3/Emu/Cell/lv2/sys_usbd.cpp
View File

@ -70,7 +70,7 @@ public:
usb_handler_thread(utils::serial& ar) : usb_handler_thread()
{
is_init = !!ar.operator u8();
is_init = !!ar.pop<u8>();
}
void save(utils::serial& ar)

6
rpcs3/Emu/IdManager.h
View File

@ -270,7 +270,7 @@ namespace id_manager
{
vec.resize(T::id_count);
u32 i = ar.operator u32();
u32 i = ar.pop<u32>();
ensure(i <= T::id_count);
@ -309,7 +309,7 @@ namespace id_manager
void save(utils::serial& ar) requires IdmSavable<T>
{
u32 obj_count = 0;
usz obj_count_offs = ar.data.size();
usz obj_count_offs = ar.pos;
// To be patched at the end of the function
ar(obj_count);
@ -340,7 +340,7 @@ namespace id_manager
}
// Patch object count
std::memcpy(ar.data.data() + obj_count_offs, &obj_count, sizeof(obj_count));
ar.patch_raw_data(obj_count_offs, &obj_count, sizeof(obj_count));
}
id_map& operator=(thread_state state) noexcept requires (std::is_assignable_v<T&, thread_state>)

81
rpcs3/Emu/Memory/vm.cpp
View File

@ -1620,45 +1620,52 @@ namespace vm
return _7 == v128{};
}
static void save_memory_bytes(utils::serial& ar, const u8* ptr, usz size)
static void serialize_memory_bytes(utils::serial& ar, u8* ptr, usz size)
{
AUDIT(ar.is_writing() && !(size % 1024));
ensure((size % 4096) == 0);
for (; size; ptr += 128 * 8, size -= 128 * 8)
for (; size; ptr += 128 * 8)
{
ar(u8{}); // bitmap of 1024 bytes (bit is 128-byte)
u8 bitmap = 0, count = 0;
const usz process_size = std::min<usz>(size, 128 * 8);
size -= process_size;
for (usz i = 0, end = std::min<usz>(size, 128 * 8); i < end; i += 128)
u8 bitmap = 0;
if (ar.is_writing())
{
if (!check_cache_line_zero(ptr + i))
for (usz i = 0; i < process_size; i += 128)
{
bitmap |= 1u << (i / 128);
count++;
ar(std::span(ptr + i, 128));
if (!check_cache_line_zero(ptr + i))
{
bitmap |= 1u << (i / 128);
}
}
}
// Patch bitmap with correct value
*std::prev(&ar.data.back(), count * 128) = bitmap;
}
}
// bitmap of 1024 bytes (bit is 128-byte)
ar(bitmap);
static void load_memory_bytes(utils::serial& ar, u8* ptr, usz size)
{
AUDIT(!ar.is_writing() && !(size % 128));
for (; size; ptr += 128 * 8, size -= 128 * 8)
{
const u8 bitmap{ar};
for (usz i = 0, end = std::min<usz>(size, 128 * 8); i < end; i += 128)
for (usz i = 0; i < process_size;)
{
if (bitmap & (1u << (i / 128)))
usz block_count = 0;
for (usz bit = i / 128; bit < sizeof(bitmap) * 8 && (bitmap & (1u << bit)) != 0;)
{
ar(std::span(ptr + i, 128));
bit++;
block_count++;
}
if (!block_count)
{
i += 128;
continue;
}
ar(std::span<u8>(ptr + i, block_count * 128));
i += block_count * 128;
}
ar.breathe();
}
}
@ -1682,14 +1689,15 @@ namespace vm
if (is_memory_compatible_for_copy_from_executable_optimization(addr, shm.first))
{
// Revert changes
ar.data.resize(ar.seek_end(sizeof(u32) * 2 + sizeof(memory_page)));
ar.data.resize(ar.data.size() - (sizeof(u32) * 2 + sizeof(memory_page)));
ar.seek_end();
vm_log.success("Removed memory block matching the memory of the executable from savestate. (addr=0x%x, size=0x%x)", addr, shm.first);
continue;
}
// Save raw binary image
const u32 guard_size = flags & stack_guarded ? 0x1000 : 0;
save_memory_bytes(ar, vm::get_super_ptr<const u8>(addr + guard_size), shm.first - guard_size * 2);
serialize_memory_bytes(ar, vm::get_super_ptr<u8>(addr + guard_size), shm.first - guard_size * 2);
}
else
{
@ -1758,13 +1766,13 @@ namespace vm
// Map the memory through the same method as alloc() and falloc()
// Copy the shared handle unconditionally
ensure(try_alloc(addr0, pflags, size0, ::as_rvalue(flags & preallocated ? null_shm : shared[ar.operator usz()])));
ensure(try_alloc(addr0, pflags, size0, ::as_rvalue(flags & preallocated ? null_shm : shared[ar.pop<usz>()])));
if (flags & preallocated)
{
// Load binary image
const u32 guard_size = flags & stack_guarded ? 0x1000 : 0;
load_memory_bytes(ar, vm::get_super_ptr<u8>(addr0 + guard_size), size0 - guard_size * 2);
serialize_memory_bytes(ar, vm::get_super_ptr<u8>(addr0 + guard_size), size0 - guard_size * 2);
}
}
}
@ -2224,9 +2232,8 @@ namespace vm
// Save shared memory
ar(shm->flags());
// TODO: string_view serialization (even with load function, so the loaded address points to a position of the stream's buffer)
ar(shm->size());
save_memory_bytes(ar, vm::get_super_ptr<u8>(addr), shm->size());
serialize_memory_bytes(ar, vm::get_super_ptr<u8>(addr), shm->size());
}
// TODO: Serialize std::vector direcly
@ -2249,19 +2256,19 @@ namespace vm
void load(utils::serial& ar)
{
std::vector<std::shared_ptr<utils::shm>> shared;
shared.resize(ar.operator usz());
shared.resize(ar.pop<usz>());
for (auto& shm : shared)
{
// Load shared memory
const u32 flags = ar;
const u64 size = ar;
const u32 flags = ar.pop<u32>();
const u64 size = ar.pop<u64>();
shm = std::make_shared<utils::shm>(size, flags);
// Load binary image
// elad335: I'm not proud about it as well.. (ideal situation is to not call map_self())
load_memory_bytes(ar, shm->map_self(), shm->size());
serialize_memory_bytes(ar, shm->map_self(), shm->size());
}
for (auto& block : g_locations)
@ -2270,11 +2277,11 @@ namespace vm
}
g_locations.clear();
g_locations.resize(ar.operator usz());
g_locations.resize(ar.pop<usz>());
for (auto& loc : g_locations)
{
const u8 has = ar;
const u8 has = ar.pop<u8>();
if (has)
{

51
rpcs3/Emu/RSX/RSXThread.cpp
View File

@ -4,6 +4,7 @@
#include "Emu/Cell/PPUCallback.h"
#include "Emu/Cell/SPUThread.h"
#include "Emu/Cell/timers.hpp"
#include "Emu/savestate_utils.hpp"
#include "Capture/rsx_capture.h"
#include "Common/BufferUtils.h"
@ -19,6 +20,7 @@
#include "Emu/Cell/lv2/sys_event.h"
#include "Emu/Cell/lv2/sys_time.h"
#include "Emu/Cell/Modules/cellGcmSys.h"
#include "Emu/savestate_utils.hpp"
#include "Overlays/overlay_perf_metrics.h"
#include "Overlays/overlay_message.h"
#include "Program/GLSLCommon.h"
@ -26,7 +28,6 @@
#include "Utilities/StrUtil.h"
#include "Crypto/unzip.h"
#include "util/serialization.hpp"
#include "util/asm.hpp"
#include <span>
@ -3574,6 +3575,8 @@ namespace rsx
void thread::on_frame_end(u32 buffer, bool forced)
{
bool pause_emulator = false;
// Marks the end of a frame scope GPU-side
if (g_user_asked_for_frame_capture.exchange(false) && !capture_current_frame)
{
@ -3594,36 +3597,34 @@ namespace rsx
{
capture_current_frame = false;
std::string file_path = fs::get_config_dir() + "captures/" + Emu.GetTitleID() + "_" + date_time::current_time_narrow() + "_capture.rrc";
utils::serial save_manager;
save_manager.reserve(0x800'0000); // 128MB
save_manager(frame_capture);
if (std::vector<u8> zipped = zip(save_manager.data); !zipped.empty())
{
file_path += ".gz";
save_manager.data = std::move(zipped);
}
else
{
rsx_log.error("Failed to compress capture");
}
std::string file_path = fs::get_config_dir() + "captures/" + Emu.GetTitleID() + "_" + date_time::current_time_narrow() + "_capture.rrc.gz";
fs::pending_file temp(file_path);
if (temp.file && (temp.file.write(save_manager.data), temp.commit(false)))
utils::serial save_manager;
if (temp.file)
{
rsx_log.success("Capture successful: %s", file_path);
save_manager.m_file_handler = make_compressed_serialization_file_handler(temp.file);
save_manager(frame_capture);
save_manager.m_file_handler->finalize(save_manager);
if (temp.commit(false))
{
rsx_log.success("Capture successful: %s", file_path);
frame_capture.reset();
pause_emulator = true;
}
else
{
rsx_log.error("Capture failed: %s (%s)", file_path, fs::g_tls_error);
}
}
else
{
rsx_log.fatal("Capture failed: %s (%s)", file_path, fs::g_tls_error);
}
frame_capture.reset();
Emu.Pause();
}
if (zcull_ctrl->has_pending())
@ -3673,6 +3674,12 @@ namespace rsx
}
}
if (pause_emulator)
{
Emu.Pause();
thread_ctrl::wait_for(30'000);
}
// Reset current stats
m_frame_stats = {};
m_profiler.enabled = !!g_cfg.video.overlay;

152
rpcs3/Emu/System.cpp
View File

@ -83,9 +83,6 @@ extern void ppu_unload_prx(const lv2_prx&);
extern std::shared_ptr<lv2_prx> ppu_load_prx(const ppu_prx_object&, bool virtual_load, const std::string&, s64 = 0, utils::serial* = nullptr);
extern std::pair<std::shared_ptr<lv2_overlay>, CellError> ppu_load_overlay(const ppu_exec_object&, bool virtual_load, const std::string& path, s64 = 0, utils::serial* = nullptr);
extern bool ppu_load_rel_exec(const ppu_rel_object&);
extern bool is_savestate_version_compatible(const std::vector<std::pair<u16, u16>>& data, bool is_boot_check);
extern std::vector<std::pair<u16, u16>> read_used_savestate_versions();
std::string get_savestate_file(std::string_view title_id, std::string_view boot_path, s64 abs_id, s64 rel_id);
extern void send_close_home_menu_cmds();
@ -219,7 +216,7 @@ void init_fxo_for_exec(utils::serial* ar, bool full = false)
const usz advance = (Emu.m_savestate_extension_flags1 & Emulator::SaveStateExtentionFlags1::SupportsMenuOpenResume ? 32 : 31);
ar->pos += advance; // Reserved area
load_and_check_reserved(*ar, advance); // Reserved area
}
}
@ -719,8 +716,11 @@ bool Emulator::BootRsxCapture(const std::string& path)
if (fmt::to_lower(path).ends_with(".gz"))
{
load.data = unzip(in_file.to_vector<u8>());
in_file.close();
load.m_file_handler = make_compressed_serialization_file_handler(std::move(in_file));
// Forcefully read some data to check validity
load.pop<uchar>();
load.pos -= sizeof(uchar);
if (load.data.empty())
{
@ -896,13 +896,31 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
}
else
{
if (fs::file save{m_path, fs::isfile + fs::read}; save && save.size() >= 8 && save.read<u64>() == "RPCS3SAV"_u64)
fs::file save{m_path, fs::isfile + fs::read};
if (!m_path.ends_with(".gz") && save && save.size() >= 8 && save.read<u64>() == "RPCS3SAV"_u64)
{
m_ar = std::make_shared<utils::serial>();
m_ar->set_reading_state();
m_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(save));
}
else if (save && m_path.ends_with(".gz"))
{
m_ar = std::make_shared<utils::serial>();
m_ar->set_reading_state();
m_ar->m_file_handler = make_compressed_serialization_file_handler(std::move(save));
if (m_ar->try_read<u64>().second != "RPCS3SAV"_u64)
{
m_ar.reset();
}
else
{
m_ar->pos = 0;
}
}
m_boot_source_type = CELL_GAME_GAMETYPE_SYS;
}
@ -952,7 +970,7 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
bool LE_format;
bool state_inspection_support;
nse_t<u64, 1> offset;
std::array<u8, 32> reserved;
b8 flag_versions_is_following_data;
};
const auto header = m_ar->try_read<file_header>().second;
@ -969,15 +987,24 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
g_cfg.savestate.state_inspection_mode.set(header.state_inspection_support);
if (header.flag_versions_is_following_data)
{
ensure(header.offset == m_ar->pos);
if (!is_savestate_version_compatible(m_ar->pop<std::vector<version_entry>>(), true))
{
return game_boot_result::savestate_version_unsupported;
}
}
else
{
// Read data on another container to keep the existing data
utils::serial ar_temp;
ar_temp.set_reading_state();
ar_temp.set_reading_state({}, true);
ar_temp.swap_handler(*m_ar);
ar_temp.seek_pos(header.offset);
ar_temp.m_avoid_large_prefetch = true;
if (!is_savestate_version_compatible(ar_temp.operator std::vector<std::pair<u16, u16>>(), true))
if (!is_savestate_version_compatible(ar_temp.pop<std::vector<version_entry>>(), true))
{
return game_boot_result::savestate_version_unsupported;
}
@ -986,11 +1013,16 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
ar_temp.swap_handler(*m_ar);
}
if (!load_and_check_reserved(*m_ar, header.flag_versions_is_following_data ? 32 : 31))
{
return game_boot_result::savestate_version_unsupported;
}
argv.clear();
klic.clear();
std::string disc_info;
(*m_ar)(argv.emplace_back(), disc_info, klic.emplace_back(), m_game_dir, hdd1);
m_ar->serialize(argv.emplace_back(), disc_info, klic.emplace_back(), m_game_dir, hdd1);
if (!klic[0])
{
@ -1026,12 +1058,16 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
{
const usz size = *m_ar;
fs::remove_all(path, size == 0);
if (size)
{
fs::remove_all(path, false);
m_ar->breathe(true);
ensure(tar_object(make_file_view(*static_cast<uncompressed_serialization_file_handler*>(m_ar->m_file_handler.get())->m_file, m_ar->data_offset, size)).extract(path));
m_ar->seek_pos(m_ar->pos + size, size >= 4096);
m_ar->m_max_data = m_ar->pos + size;
ensure(tar_object(*m_ar).extract(path));
m_ar->seek_pos(m_ar->m_max_data, size >= 4096);
m_ar->m_max_data = umax;
m_ar->breathe();
}
};
@ -1053,7 +1089,11 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
load_tar(hdd0_game + game_data);
}
m_ar->pos += 32; // Reserved area
// Reserved area
if (!load_and_check_reserved(*m_ar, 32))
{
return game_boot_result::savestate_version_unsupported;
}
if (disc_info.starts_with("/"sv))
{
@ -2261,7 +2301,7 @@ void Emulator::FixGuestTime()
{
if (m_ar)
{
initialize_timebased_time(m_ar->operator u64());
initialize_timebased_time(m_ar->pop<u64>());
g_cfg.savestate.state_inspection_mode.set(m_state_inspection_savestate);
@ -2862,6 +2902,13 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
{
path = get_savestate_file(m_title_id, m_path, 0, 0);
// The function is meant for reading files, so if there is no GZ file it would not return compressed file path
// So this is the only place where the result is edited if need to be
if (!path.ends_with(".gz"))
{
path += ".gz";
}
if (!fs::create_path(fs::get_parent_dir(path)))
{
sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error);
@ -2877,7 +2924,7 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
}
to_ar = std::make_unique<utils::serial>();
to_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(file.file));
to_ar->m_file_handler = make_compressed_serialization_file_handler(file.file);
break;
}
@ -2899,31 +2946,31 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
// Avoid duplicating TAR object memory because it can be very large
auto save_tar = [&](const std::string& path)
{
const usz old_data_start = ar.data_offset;
const usz old_pos = ar.seek_end();
if (!fs::is_dir(path))
{
ar(usz{});
return;
}
// Cached file list from the first call
std::vector<fs::dir_entry> dir_entries;
// Calculate memory requirements
utils::serial ar_null;
ar_null.m_file_handler = make_null_serialization_file_handler();
tar_object::save_directory(path, ar_null, {}, std::move(dir_entries), false);
ar(ar_null.pos);
ar.breathe();
ar(usz{}); // Reserve memory to be patched later with correct size
tar_object::save_directory(path, ar);
const usz old_pos = ar.seek_end();
tar_object::save_directory(path, ar, {}, std::move(dir_entries), true);
const usz new_pos = ar.seek_end();
const usz tar_size = new_pos - old_pos - sizeof(usz);
// Check if breathe() actually did something, in this case memory needs to be discarded
const bool was_emptied = old_data_start != ar.data_offset;
const usz tar_size = new_pos - old_pos;
if (was_emptied)
if (tar_size != ar_null.pos)
{
ensure(ar.data_offset > old_data_start);
// Write to file directly (slower)
ar.m_file_handler->handle_file_op(ar, old_pos, sizeof(tar_size), &tar_size);
}
else
{
// If noty written to file, simply write to memory
std::memcpy(ar.data.data() + old_pos - old_data_start, &tar_size, sizeof(usz));
fmt::throw_exception("Unexpected TAR entry size (size=0x%x, expected=0x%x, entries=0x%x)", tar_size, ar_null.pos, dir_entries.size());
}
sys_log.success("Saved the contents of directory '%s' (size=0x%x)", path, tar_size);
@ -2969,7 +3016,18 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
ar("RPCS3SAV"_u64);
ar(std::endian::native == std::endian::little);
ar(g_cfg.savestate.state_inspection_mode.get());
ar(usz{0}); // Offset of versioning data, to be overwritten at the end of saving
ar(usz{10 + sizeof(usz) + sizeof(u8)}); // Offset of versioning data (fixed to the following data)
{
// Gather versions because with compressed format going back and patching offset is not efficient
utils::serial ar_temp;
ar_temp.m_file_handler = make_null_serialization_file_handler();
g_fxo->save(ar_temp);
ar(u8{1});
ar(read_used_savestate_versions());
}
ar(std::array<u8, 32>{}); // Reserved for future use
if (auto dir = vfs::get("/dev_bdvd/PS3_GAME"); fs::is_dir(dir) && !fs::is_file(fs::get_parent_dir(dir) + "/PS3_DISC.SFB"))
@ -3022,27 +3080,11 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
if (savestate)
{
// Identifer -> version
std::vector<std::pair<u16, u16>> used_serial = read_used_savestate_versions();
auto& ar = *to_ar;
const usz pos = ar.seek_end();
// Patch offset with a direct write
ar.m_file_handler->handle_file_op(ar, 10, sizeof(usz), &pos);
// Write the version data at the end
ar(used_serial);
// Final file write, the file is ready to be committed
ar.breathe(true);
#ifndef _WIN32
// The temporary file's contents must be on disk before rename
// Flush to file
ar.m_file_handler->handle_file_op(ar, umax, umax, nullptr);
#endif
ar.seek_end();
ar.m_file_handler->finalize(ar);
if (!file.commit())
{

496
rpcs3/Emu/savestate_utils.cpp
View File

@ -2,6 +2,8 @@
#include "util/types.hpp"
#include "util/logs.hpp"
#include "util/asm.hpp"
#include "util/v128.hpp"
#include "util/simd.hpp"
#include "Utilities/File.h"
#include "Utilities/StrFmt.h"
#include "system_config.h"
@ -10,6 +12,9 @@
#include "System.h"
#include <set>
#include <span>
#include <zlib.h>
LOG_CHANNEL(sys_log, "SYS");
@ -18,14 +23,14 @@ void fmt_class_string<utils::serial>::format(std::string& out, u64 arg)
{
const utils::serial& ar = get_object(arg);
fmt::append(out, "{ %s, 0x%x/0%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size());
fmt::append(out, "{ %s, 0x%x/0x%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size());
}
struct serial_ver_t
{
bool used = false;
s32 current_version = 0;
std::set<s32> compatible_versions;
u16 current_version = 0;
std::set<u16> compatible_versions;
};
static std::array<serial_ver_t, 26> s_serial_versions;
@ -89,7 +94,7 @@ SERIALIZATION_VER(sys_io, 23, 1)
SERIALIZATION_VER(LLE, 24, 1)
SERIALIZATION_VER(HLE, 25, 1)
std::vector<std::pair<u16, u16>> get_savestate_versioning_data(fs::file&& file)
std::vector<version_entry> get_savestate_versioning_data(fs::file&& file, std::string_view filepath)
{
if (!file)
{
@ -98,31 +103,36 @@ std::vector<std::pair<u16, u16>> get_savestate_versioning_data(fs::file&& file)
file.seek(0);
if (u64 r = 0; !file.read(r) || r != "RPCS3SAV"_u64)
utils::serial ar;
ar.set_reading_state();
ar.m_file_handler = filepath.ends_with(".gz") ? static_cast<std::unique_ptr<utils::serialization_file_handler>>(make_compressed_serialization_file_handler(std::move(file)))
: make_uncompressed_serialization_file_handler(std::move(file));
if (u64 r = 0; ar.try_read(r) != 0 || r != "RPCS3SAV"_u64)
{
return {};
}
file.seek(10);
ar.pos = 10;
u64 offs = 0;
file.read(offs);
u64 offs = ar.try_read<u64>().second;
const usz fsize = file.size();
const usz fsize = ar.get_size(offs);
if (!offs || fsize <= offs)
{
return {};
}
utils::serial ar;
ar.set_reading_state();
ar.m_file_handler = make_uncompressed_serialization_file_handler(std::move(file));
ar.seek_pos(offs);
return ar;
ar.breathe(true);
std::vector<version_entry> ver_data = ar.pop<std::vector<version_entry>>();
return std::move(ver_data);
}
bool is_savestate_version_compatible(const std::vector<std::pair<u16, u16>>& data, bool is_boot_check)
bool is_savestate_version_compatible(const std::vector<version_entry>& data, bool is_boot_check)
{
if (data.empty())
{
@ -178,24 +188,31 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_
// While not needing to keep a 59 chars long suffix at all times for this purpose
const char prefix = ::at32("0123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"sv, save_id.size());
return fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT";
std::string path = fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT";
if (std::string path_compressed = path + ".gz"; fs::is_file(path_compressed))
{
return std::move(path_compressed);
}
return std::move(path);
}
bool is_savestate_compatible(fs::file&& file)
bool is_savestate_compatible(fs::file&& file, std::string_view filepath)
{
return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file)), false);
return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file), filepath), false);
}
std::vector<std::pair<u16, u16>> read_used_savestate_versions()
std::vector<version_entry> read_used_savestate_versions()
{
std::vector<std::pair<u16, u16>> used_serial;
std::vector<version_entry> used_serial;
used_serial.reserve(s_serial_versions.size());
for (serial_ver_t& ver : s_serial_versions)
{
if (std::exchange(ver.used, false))
{
used_serial.emplace_back(&ver - s_serial_versions.data(), *ver.compatible_versions.rbegin());
used_serial.push_back(version_entry{static_cast<u16>(&ver - s_serial_versions.data()), *ver.compatible_versions.rbegin()});
}
ver.current_version = 0;
@ -208,8 +225,6 @@ bool boot_last_savestate(bool testing)
{
if (!g_cfg.savestate.suspend_emu && !Emu.GetTitleID().empty() && (Emu.IsRunning() || Emu.GetStatus() == system_state::paused))
{
extern bool is_savestate_compatible(fs::file&& file);
const std::string save_dir = fs::get_cache_dir() + "/savestates/";
std::string savestate_path;
@ -225,7 +240,12 @@ bool boot_last_savestate(bool testing)
// Find the latest savestate file compatible with the game (TODO: Check app version and anything more)
if (entry.name.find(Emu.GetTitleID()) != umax && mtime <= entry.mtime)
{
if (std::string path = save_dir + entry.name; is_savestate_compatible(fs::file(path)))
if (std::string path = save_dir + entry.name + ".gz"; is_savestate_compatible(fs::file(path), path))
{
savestate_path = std::move(path);
mtime = entry.mtime;
}
else if (std::string path = save_dir + entry.name; is_savestate_compatible(fs::file(path), path))
{
savestate_path = std::move(path);
mtime = entry.mtime;
@ -282,9 +302,9 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
m_file->sync();
}
ar.data_offset += ar.data.size();
ar.data.clear();
ar.seek_end();
ar.data_offset = ar.pos;
ar.data.clear();
return true;
}
@ -320,7 +340,7 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
}
// Discard all loaded data
ar.data_offset += ar.data.size();
ar.data_offset = ar.pos;
ar.data.clear();
if (ar.data.capacity() >= 0x200'0000)
@ -332,7 +352,7 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
return true;
}
if (~size < pos)
if (~pos < size - 1)
{
// Overflow
return false;
@ -340,11 +360,11 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
if (ar.data.empty() && pos != ar.pos)
{
// Relocate instead oof over-fetch
// Relocate instead of over-fetch
ar.seek_pos(pos);
}
const usz read_pre_buffer = utils::sub_saturate<usz>(ar.data_offset, pos);
const usz read_pre_buffer = ar.data.empty() ? 0 : utils::sub_saturate<usz>(ar.data_offset, pos);
if (read_pre_buffer)
{
@ -358,7 +378,10 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
ar.data_offset -= read_pre_buffer;
}
const usz read_past_buffer = utils::sub_saturate<usz>(pos + size, ar.data_offset + ar.data.size());
// Adjustment to prevent overflow
const usz subtrahend = ar.data.empty() ? 0 : 1;
const usz read_past_buffer = utils::sub_saturate<usz>(pos + (size - subtrahend), ar.data_offset + (ar.data.size() - subtrahend));
const usz read_limit = utils::sub_saturate<usz>(ar.m_max_data, ar.data_offset);
if (read_past_buffer)
{
@ -368,14 +391,13 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
const usz old_size = ar.data.size();
// Try to prefetch data by reading more than requested
ar.data.resize(std::max<usz>({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} }));
ar.data.resize(std::min<usz>(read_limit, std::max<usz>({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} })));
ar.data.resize(m_file->read_at(old_size + ar.data_offset, data ? const_cast<void*>(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size);
}
return true;
}
usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const
{
if (ar.is_writing())
@ -394,6 +416,416 @@ usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, u
return std::max<usz>(m_file->size(), memory_available);
}
void uncompressed_serialization_file_handler::finalize(utils::serial& ar)
{
ar.seek_end();
handle_file_op(ar, 0, umax, nullptr);
ar.data = {}; // Deallocate and clear
}
void compressed_serialization_file_handler::initialize(utils::serial& ar)
{
if (!m_stream.has_value())
{
m_stream.emplace<z_stream>();
}
z_stream& m_zs = std::any_cast<z_stream&>(m_stream);
if (ar.is_writing())
{
if (m_write_inited)
{
return;
}
#ifndef _MSC_VER
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
#endif
if (m_read_inited)
{
finalize(ar);
}
m_zs = {};
ensure(deflateInit2(&m_zs, 9, Z_DEFLATED, 16 + 15, 9, Z_DEFAULT_STRATEGY) == Z_OK);
#ifndef _MSC_VER
#pragma GCC diagnostic pop
#endif
m_write_inited = true;
}
else
{
if (m_read_inited)
{
return;
}
if (m_write_inited)
{
finalize(ar);
}
m_zs.avail_in = 0;
m_zs.avail_out = 0;
m_zs.next_in = nullptr;
m_zs.next_out = nullptr;
#ifndef _MSC_VER
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
#endif
ensure(inflateInit2(&m_zs, 16 + 15) == Z_OK);
m_read_inited = true;
}
}
bool compressed_serialization_file_handler::handle_file_op(utils::serial& ar, usz pos, usz size, const void* data)
{
if (ar.is_writing())
{
initialize(ar);
z_stream& m_zs = std::any_cast<z_stream&>(m_stream);
if (data)
{
ensure(false);
}
// Writing not at the end is forbidden
ensure(ar.pos == ar.data_offset + ar.data.size());
m_zs.avail_in = static_cast<uInt>(ar.data.size());
m_zs.next_in = ar.data.data();
do
{
m_stream_data.resize(std::max<usz>(m_stream_data.size(), ::compressBound(m_zs.avail_in)));
m_zs.avail_out = static_cast<uInt>(m_stream_data.size());
m_zs.next_out = m_stream_data.data();
if (deflate(&m_zs, Z_NO_FLUSH) == Z_STREAM_ERROR || m_file->write(m_stream_data.data(), m_stream_data.size() - m_zs.avail_out) != m_stream_data.size() - m_zs.avail_out)
{
deflateEnd(&m_zs);
//m_file->close();
break;
}
}
while (m_zs.avail_out == 0);
ar.seek_end();
ar.data_offset = ar.pos;
ar.data.clear();
if (pos == umax && size == umax && *m_file)
{
// Request to flush the file to disk
m_file->sync();
}
return true;
}
initialize(ar);
if (!size)
{
return true;
}
if (pos == 0 && size == umax)
{
// Discard loaded data until pos if profitable
const usz limit = ar.data_offset + ar.data.size();
if (ar.pos > ar.data_offset && ar.pos < limit)
{
const usz may_discard_bytes = ar.pos - ar.data_offset;
const usz moved_byte_count_on_discard = limit - ar.pos;
// Cheeck profitability (check recycled memory and std::memmove costs)
if (may_discard_bytes >= 0x50'0000 || (may_discard_bytes >= 0x20'0000 && moved_byte_count_on_discard / may_discard_bytes < 3))
{
ar.data_offset += may_discard_bytes;
ar.data.erase(ar.data.begin(), ar.data.begin() + may_discard_bytes);
if (ar.data.capacity() >= 0x200'0000)
{
// Discard memory
ar.data.shrink_to_fit();
}
}
return true;
}
// Discard all loaded data
ar.data_offset += ar.data.size();
ensure(ar.pos >= ar.data_offset);
ar.data.clear();
if (ar.data.capacity() >= 0x200'0000)
{
// Discard memory
ar.data.shrink_to_fit();
}
return true;
}
if (~pos < size - 1)
{
// Overflow
return false;
}
// TODO: Investigate if this optimization is worth an implementation for compressed stream
// if (ar.data.empty() && pos != ar.pos)
// {
// // Relocate instead of over-fetch
// ar.seek_pos(pos);
// }
const usz read_pre_buffer = utils::sub_saturate<usz>(ar.data_offset, pos);
if (read_pre_buffer)
{
// Not allowed with compressed data for now
// Unless someone implements mechanism for it
ensure(false);
}
// Adjustment to prevent overflow
const usz subtrahend = ar.data.empty() ? 0 : 1;
const usz read_past_buffer = utils::sub_saturate<usz>(pos + (size - subtrahend), ar.data_offset + (ar.data.size() - subtrahend));
const usz read_limit = utils::sub_saturate<usz>(ar.m_max_data, ar.data_offset);
if (read_past_buffer)
{
// Read proceeding data
// More lightweight operation, this is the common operation
// Allowed to fail, if memory is truly needed an assert would take place later
const usz old_size = ar.data.size();
// Try to prefetch data by reading more than requested
ar.data.resize(std::min<usz>(read_limit, std::max<usz>({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} })));
ar.data.resize(this->read_at(ar, old_size + ar.data_offset, data ? const_cast<void*>(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size);
}
return true;
}
usz compressed_serialization_file_handler::read_at(utils::serial& ar, usz read_pos, void* data, usz size)
{
ensure(read_pos == ar.data.size() + ar.data_offset - size);
if (!size)
{
return 0;
}
initialize(ar);
z_stream& m_zs = std::any_cast<z_stream&>(m_stream);
const usz total_to_read = size;
usz read_size = 0;
u8* out_data = static_cast<u8*>(data);
auto adjust_for_uint = [](usz size)
{
return static_cast<uInt>(std::min<usz>(uInt{umax}, size));
};
for (; read_size < total_to_read;)
{
// Drain extracted memory stash (also before first file read)
out_data = static_cast<u8*>(data) + read_size;
m_zs.avail_in = adjust_for_uint(m_stream_data.size() - m_stream_data_index);
m_zs.next_in = reinterpret_cast<const u8*>(m_stream_data.data() + m_stream_data_index);
m_zs.next_out = out_data;
m_zs.avail_out = adjust_for_uint(size - read_size);
while (read_size < total_to_read && m_zs.avail_in)
{
const int res = inflate(&m_zs, Z_BLOCK);
bool need_more_file_memory = false;
switch (res)
{
case Z_OK:
case Z_STREAM_END:
break;
case Z_BUF_ERROR:
{
if (m_zs.avail_in)
{
need_more_file_memory = true;
break;
}
[[fallthrough]];
}
default:
inflateEnd(&m_zs);
m_read_inited = false;
return read_size;
}
read_size = m_zs.next_out - static_cast<u8*>(data);
m_stream_data_index = m_zs.avail_in ? m_zs.next_in - m_stream_data.data() : m_stream_data.size();
// Adjust again in case the values simply did not fit into uInt
m_zs.avail_out = adjust_for_uint(utils::sub_saturate<usz>(total_to_read, read_size));
m_zs.avail_in = adjust_for_uint(m_stream_data.size() - m_stream_data_index);
if (need_more_file_memory)
{
break;
}
}
if (read_size >= total_to_read)
{
break;
}
const usz add_size = ar.m_avoid_large_prefetch ? 0x1'0000 : 0x10'0000;
const usz old_file_buf_size = m_stream_data.size();
m_stream_data.resize(old_file_buf_size + add_size);
m_stream_data.resize(old_file_buf_size + m_file->read_at(m_file_read_index, m_stream_data.data() + old_file_buf_size, add_size));
if (m_stream_data.size() == old_file_buf_size)
{
// EOF
break;
}
m_file_read_index += m_stream_data.size() - old_file_buf_size;
}
if (m_stream_data.size() - m_stream_data_index <= m_stream_data_index / 5)
{
// Shrink to required memory size
m_stream_data.erase(m_stream_data.begin(), m_stream_data.begin() + m_stream_data_index);
if (m_stream_data.capacity() >= 0x200'0000)
{
// Discard memory
m_stream_data.shrink_to_fit();
}
m_stream_data_index = 0;
}
return read_size;
}
void compressed_serialization_file_handler::skip_until(utils::serial& ar)
{
ensure(!ar.is_writing() && ar.pos >= ar.data_offset);
if (ar.pos > ar.data_offset)
{
handle_file_op(ar, ar.data_offset, ar.pos - ar.data_offset, nullptr);
}
}
void compressed_serialization_file_handler::finalize(utils::serial& ar)
{
handle_file_op(ar, 0, umax, nullptr);
if (!m_stream.has_value())
{
return;
}
z_stream& m_zs = std::any_cast<z_stream&>(m_stream);
if (m_read_inited)
{
m_read_inited = false;
ensure(inflateEnd(&m_zs) == Z_OK);
return;
}
m_write_inited = false;
m_zs.avail_in = 0;
m_zs.next_in = nullptr;
m_stream_data.resize(m_zs.avail_out);
do
{
m_zs.avail_out = static_cast<uInt>(m_stream_data.size());
m_zs.next_out = m_stream_data.data();
if (deflate(&m_zs, Z_FINISH) == Z_STREAM_ERROR)
{
break;
}
m_file->write(m_stream_data.data(), m_stream_data.size() - m_zs.avail_out);
}
while (m_zs.avail_out == 0);
m_stream_data = {};
ensure(deflateEnd(&m_zs) == Z_OK);
ar.data = {}; // Deallocate and clear
}
usz compressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const
{
if (ar.is_writing())
{
return m_file->size();
}
const usz memory_available = ar.data_offset + ar.data.size();
if (memory_available >= recommended)
{
// Avoid calling size() if possible
return memory_available;
}
return std::max<usz>(utils::mul_saturate<usz>(m_file->size(), 6), memory_available);
}
bool null_serialization_file_handler::handle_file_op(utils::serial&, usz, usz, const void*)
{
return true;
}
void null_serialization_file_handler::finalize(utils::serial&)
{
}
bool load_and_check_reserved(utils::serial& ar, usz size)
{
u8 bytes[4096];
std::memset(&bytes[size & (0 - sizeof(v128))], 0, sizeof(v128));
ensure(size <= std::size(bytes));
const usz old_pos = ar.pos;
ar(std::span<u8>(bytes, size));
// Check if all are 0
for (usz i = 0; i < size; i += sizeof(v128))
{
if (v128::loadu(&bytes[i]) != v128{})
{
return false;
}
}
return old_pos + size == ar.pos;
}
namespace stx
{
extern void serial_breathe(utils::serial& ar)

101
rpcs3/Emu/savestate_utils.hpp
View File

@ -1,10 +1,22 @@
#pragma once
#include "util/serialization.hpp"
#include <any>
namespace fs
{
class file;
}
struct version_entry
{
u16 type;
u16 version;
ENABLE_BITWISE_SERIALIZATION;
};
// Uncompressed file serialization handler
struct uncompressed_serialization_file_handler : utils::serialization_file_handler
{
@ -25,21 +37,102 @@ struct uncompressed_serialization_file_handler : utils::serialization_file_handl
{
}
uncompressed_serialization_file_handler(const uncompressed_serialization_file_handler&) = delete;
// Handle file read and write requests
bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override;
// Get available memory or file size
// Preferably memory size if is already greater/equal to recommended to avoid additional file ops
usz get_size(const utils::serial& ar, usz recommended) const override;
void finalize(utils::serial& ar) override;
};
inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(fs::file&& file)
template <typename File> requires (std::is_same_v<std::remove_cvref_t<File>, fs::file>)
inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(File&& file)
{
return std::make_unique<uncompressed_serialization_file_handler>(std::move(file));
ensure(file);
return std::make_unique<uncompressed_serialization_file_handler>(std::forward<File>(file));
}
inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(const fs::file& file)
// Compressed file serialization handler
struct compressed_serialization_file_handler : utils::serialization_file_handler
{
return std::make_unique<uncompressed_serialization_file_handler>(file);
const std::unique_ptr<fs::file> m_file_storage;
const std::add_pointer_t<const fs::file> m_file;
std::vector<u8> m_stream_data;
usz m_stream_data_index = 0;
usz m_file_read_index = 0;
bool m_write_inited = false;
bool m_read_inited = false;
std::any m_stream;
explicit compressed_serialization_file_handler(fs::file&& file) noexcept
: utils::serialization_file_handler()
, m_file_storage(std::make_unique<fs::file>(std::move(file)))
, m_file(m_file_storage.get())
{
}
explicit compressed_serialization_file_handler(const fs::file& file) noexcept
: utils::serialization_file_handler()
, m_file_storage(nullptr)
, m_file(std::addressof(file))
{
}
compressed_serialization_file_handler(const compressed_serialization_file_handler&) = delete;
// Handle file read and write requests
bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override;
// Get available memory or file size
// Preferably memory size if is already greater/equal to recommended to avoid additional file ops
usz get_size(const utils::serial& ar, usz recommended) const override;
void skip_until(utils::serial& ar) override;
void finalize(utils::serial& ar) override;
private:
usz read_at(utils::serial& ar, usz read_pos, void* data, usz size);
void initialize(utils::serial& ar);
};
template <typename File> requires (std::is_same_v<std::remove_cvref_t<File>, fs::file>)
inline std::unique_ptr<compressed_serialization_file_handler> make_compressed_serialization_file_handler(File&& file)
{
ensure(file);
return std::make_unique<compressed_serialization_file_handler>(std::forward<File>(file));
}
// Null file serialization handler
struct null_serialization_file_handler : utils::serialization_file_handler
{
explicit null_serialization_file_handler() noexcept
{
}
// Handle file read and write requests
bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override;
void finalize(utils::serial& ar) override;
bool is_null() const override
{
return true;
}
};
inline std::unique_ptr<null_serialization_file_handler> make_null_serialization_file_handler()
{
return std::make_unique<null_serialization_file_handler>();
}
bool load_and_check_reserved(utils::serial& ar, usz size);
bool is_savestate_version_compatible(const std::vector<version_entry>& data, bool is_boot_check);
std::vector<version_entry> get_savestate_versioning_data(fs::file&& file, std::string_view filepath);
bool is_savestate_compatible(fs::file&& file, std::string_view filepath);
std::vector<version_entry> read_used_savestate_versions();
std::string get_savestate_file(std::string_view title_id, std::string_view boot_path, s64 abs_id, s64 rel_id);

547
rpcs3/Loader/TAR.cpp
View File

@ -8,14 +8,30 @@
#include "TAR.h"
#include "util/asm.hpp"
#include "util/serialization.hpp"
#include "Emu/savestate_utils.hpp"
#include <charconv>
#include <span>
LOG_CHANNEL(tar_log, "TAR");
fs::file make_file_view(const fs::file& file, u64 offset, u64 size);
// File constructor
tar_object::tar_object(const fs::file& file)
: m_file(file)
: m_file(std::addressof(file))
, m_ar(nullptr)
, m_ar_tar_start(umax)
{
ensure(*m_file);
}
// Stream (pipe-like) constructor
tar_object::tar_object(utils::serial& ar)
: m_file(nullptr)
, m_ar(std::addressof(ar))
, m_ar_tar_start(ar.pos)
{
}
@ -23,12 +39,14 @@ TARHeader tar_object::read_header(u64 offset) const
{
TARHeader header{};
if (m_file.seek(offset) != offset)
if (m_ar)
{
ensure(m_ar->pos == m_ar_tar_start + offset);
m_ar->serialize(header);
return header;
}
if (!m_file.read(header))
if (m_file->read_at(offset, &header, sizeof(header)) != sizeof(header))
{
std::memset(&header, 0, sizeof(header));
}
@ -38,13 +56,13 @@ TARHeader tar_object::read_header(u64 offset) const
u64 octal_text_to_u64(std::string_view sv)
{
u64 i = -1;
u64 i = umax;
const auto ptr = std::from_chars(sv.data(), sv.data() + sv.size(), i, 8).ptr;
// Range must be terminated with either NUL or space
if (ptr == sv.data() + sv.size() || (*ptr && *ptr != ' '))
{
i = -1;
i = umax;
}
return i;
@ -53,114 +71,166 @@ u64 octal_text_to_u64(std::string_view sv)
std::vector<std::string> tar_object::get_filenames()
{
std::vector<std::string> vec;
get_file("");
for (auto it = m_map.cbegin(); it != m_map.cend(); ++it)
{
vec.push_back(it->first);
}
return vec;
}
fs::file tar_object::get_file(const std::string& path)
std::unique_ptr<utils::serial> tar_object::get_file(const std::string& path, std::string* new_file_path)
{
if (!m_file) return fs::file();
std::unique_ptr<utils::serial> m_out;
auto emplace_single_entry = [&](usz& offset, const usz max_size) -> std::pair<usz, std::string>
{
if (offset >= max_size)
{
return {};
}
TARHeader header = read_header(offset);
offset += 512;
u64 size = umax;
std::string filename;
if (std::memcmp(header.magic, "ustar", 5) == 0)
{
const std::string_view size_sv{header.size, std::size(header.size)};
size = octal_text_to_u64(size_sv);
// Check for overflows and if surpasses file size
if ((header.name[0] || header.prefix[0]) && ~size >= 512 && max_size >= size && max_size - size >= offset)
{
// Cache size in native u64 format
static_assert(sizeof(size) < sizeof(header.size));
std::memcpy(header.size, &size, 8);
std::string_view prefix_name{header.prefix, std::size(header.prefix)};
std::string_view name{header.name, std::size(header.name)};
prefix_name = prefix_name.substr(0, prefix_name.find_first_of('\0'));
name = name.substr(0, name.find_first_of('\0'));
filename += prefix_name;
filename += name;
// Save header and offset
m_map.insert_or_assign(filename, std::make_pair(offset, header));
if (new_file_path)
{
*new_file_path = filename;
}
return { size, std::move(filename) };
}
else
{
tar_log.error("tar_object::get_file() failed to convert header.size=%s, filesize=0x%x", size_sv, max_size);
}
}
else
{
tar_log.trace("tar_object::get_file() failed to parse header: offset=0x%x, filesize=0x%x", largest_offset, max_size);
}
return { size, {} };
};
if (auto it = m_map.find(path); it != m_map.end())
{
u64 size = 0;
std::memcpy(&size, it->second.second.size, sizeof(size));
std::vector<u8> buf(size);
m_file.seek(it->second.first);
m_file.read(buf, size);
return fs::make_stream(std::move(buf));
if (m_file)
{
m_out = std::make_unique<utils::serial>();
m_out->set_reading_state();
m_out->m_file_handler = make_uncompressed_serialization_file_handler(make_file_view(*m_file, it->second.first, size));
}
else
{
m_out = std::make_unique<utils::serial>();
*m_out = std::move(*m_ar);
m_out->m_max_data = m_ar_tar_start + it->second.first + size;
}
return m_out;
}
else //continue scanning from last file entered
else if (m_ar && path.empty())
{
const u64 max_size = m_file.size();
const u64 size = emplace_single_entry(largest_offset, m_ar->get_size(umax) - m_ar_tar_start).first;
// Advance offset to next block
largest_offset += utils::align(size, 512);
}
// Continue scanning from last file entered
else if (m_file)
{
const u64 max_size = m_file->size();
while (largest_offset < max_size)
{
TARHeader header = read_header(largest_offset);
u64 size = -1;
std::string filename;
if (std::memcmp(header.magic, "ustar", 5) == 0)
{
const std::string_view size_sv{header.size, std::size(header.size)};
size = octal_text_to_u64(size_sv);
// Check for overflows and if surpasses file size
if ((header.name[0] || header.prefix[0]) && size + 512 > size && max_size >= size + 512 && max_size - size - 512 >= largest_offset)
{
// Cache size in native u64 format
static_assert(sizeof(size) < sizeof(header.size));
std::memcpy(header.size, &size, 8);
std::string_view prefix_name{header.prefix, std::size(header.prefix)};
std::string_view name{header.name, std::size(header.name)};
prefix_name = prefix_name.substr(0, prefix_name.find_first_of('\0'));
name = name.substr(0, name.find_first_of('\0'));
filename += prefix_name;
filename += name;
// Save header and offset
m_map.insert_or_assign(filename, std::make_pair(largest_offset + 512, header));
}
else
{
// Invalid
size = -1;
tar_log.error("tar_object::get_file() failed to convert header.size=%s, filesize=0x%x", size_sv, max_size);
}
}
else
{
tar_log.trace("tar_object::get_file() failed to parse header: offset=0x%x, filesize=0x%x", largest_offset, max_size);
}
const auto [size, filename] = emplace_single_entry(largest_offset, max_size);
if (size == umax)
{
largest_offset += 512;
continue;
}
// Advance offset to next block
largest_offset += utils::align(size, 512) + 512;
largest_offset += utils::align(size, 512);
if (!path.empty() && path == filename)
{
// Path is equal, read file and advance offset to start of next block
std::vector<u8> buf(size);
if (m_file.read(buf, size))
{
return fs::make_stream(std::move(buf));
}
tar_log.error("tar_object::get_file() failed to read file entry %s (size=0x%x)", filename, size);
largest_offset -= utils::align(size, 512);
// Path is equal, return handle to the file data
return get_file(path);
}
}
return fs::file();
}
return m_out;
}
bool tar_object::extract(std::string prefix_path, bool is_vfs)
{
if (!m_file) return false;
std::vector<u8> filedata_buffer(0x80'0000);
std::span<u8> filedata_span{filedata_buffer.data(), filedata_buffer.size()};
get_file(""); // Make sure we have scanned all files
auto iter = m_map.begin();
for (auto& iter : m_map)
auto get_next = [&](bool is_first)
{
const TARHeader& header = iter.second.second;
const std::string& name = iter.first;
if (m_ar)
{
ensure(!is_first || m_map.empty()); // Must be empty on first call
std::string name_iter;
get_file("", &name_iter); // Get next entry
return m_map.find(name_iter);
}
else if (is_first)
{
get_file(""); // Scan entries
return m_map.begin();
}
else
{
return std::next(iter);
}
};
for (iter = get_next(true); iter != m_map.end(); iter = get_next(false))
{
const TARHeader& header = iter->second.second;
const std::string& name = iter->first;
std::string result = name;
@ -210,15 +280,53 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs)
return false;
}
auto data = get_file(name).release();
// For restoring m_ar->m_max_data
usz restore_limit = umax;
if (!m_file)
{
// Restore m_ar (remove limit)
restore_limit = m_ar->m_max_data;
}
std::unique_ptr<utils::serial> file_data = get_file(name);
fs::file file(result, fs::rewrite);
if (file)
if (file && file_data)
{
file.write(static_cast<fs::container_stream<std::vector<u8>>*>(data.get())->obj);
while (true)
{
const usz unread_size = file_data->try_read(filedata_span);
if (unread_size == 0)
{
file.write(filedata_span.data(), filedata_span.size());
continue;
}
// Tail data
if (usz read_size = filedata_span.size() - unread_size)
{
ensure(file_data->try_read(filedata_span.first(read_size)) == 0);
file.write(filedata_span.data(), read_size);
}
break;
}
file.close();
file_data->seek_pos(m_ar_tar_start + largest_offset, true);
if (!m_file)
{
// Restore m_ar
*m_ar = std::move(*file_data);
m_ar->m_max_data = restore_limit;
}
if (mtime != umax && !fs::utime(result, atime, mtime))
{
tar_log.error("TAR Loader: fs::utime failed on %s (%s)", result, fs::g_tls_error);
@ -229,6 +337,13 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs)
break;
}
if (!m_file)
{
// Restore m_ar
*m_ar = std::move(*file_data);
m_ar->m_max_data = restore_limit;
}
const auto old_error = fs::g_tls_error;
tar_log.error("TAR Loader: failed to write file %s (%s) (fs::exists=%s)", name, old_error, fs::exists(result));
return false;
@ -259,32 +374,14 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs)
return true;
}
void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func, std::string full_path)
void tar_object::save_directory(const std::string& target_path, utils::serial& ar, const process_func& func, std::vector<fs::dir_entry>&& entries, bool has_evaluated_results, usz src_dir_pos)
{
const std::string& target_path = full_path.empty() ? src_dir : full_path;
const bool is_null = ar.m_file_handler && ar.m_file_handler->is_null();
const bool reuse_entries = !is_null || has_evaluated_results;
fs::stat_t stat{};
if (!fs::get_stat(target_path, stat))
if (reuse_entries)
{
return;
}
if (stat.is_directory)
{
bool has_items = false;
for (auto& entry : fs::dir(target_path))
{
if (entry.name.find_first_not_of('.') == umax) continue;
save_directory(src_dir, ar, func, target_path + '/' + entry.name);
has_items = true;
}
if (has_items)
{
return;
}
ensure(!entries.empty());
}
auto write_octal = [](char* ptr, u64 i)
@ -303,60 +400,224 @@ void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, c
}
};
std::string saved_path{target_path.data() + src_dir.size(), target_path.size() - src_dir.size()};
const u64 old_size = ar.data.size();
ar.data.resize(old_size + sizeof(TARHeader));
if (!stat.is_directory)
auto save_file = [&](const fs::stat_t& file_stat, const std::string& file_name)
{
fs::file fd(target_path);
const u64 old_size2 = ar.data.size();
if (func)
if (!file_stat.size)
{
// Use custom function for file saving if provided
// Allows for example to compress PNG files as JPEG in the TAR itself
if (!func(fd, saved_path, ar))
return;
}
if (is_null && !func)
{
ar.pos += utils::align(file_stat.size, 512);
return;
}
if (fs::file fd{file_name})
{
const u64 old_pos = ar.pos;
const usz old_size = ar.data.size();
if (func)
{
std::string saved_path{&::at32(file_name, src_dir_pos), file_name.size() - src_dir_pos};
// Use custom function for file saving if provided
// Allows for example to compress PNG files as JPEG in the TAR itself
if (!func(fd, saved_path, ar))
{
// Revert (this entry should not be included if func returns false)
if (is_null)
{
ar.pos = old_pos;
return;
}
ar.data.resize(old_size);
ar.seek_end();
return;
}
if (is_null)
{
// Align
ar.pos += utils::align(ar.pos - old_pos, 512);
return;
}
}
else
{
constexpr usz transfer_block_size = 0x100'0000;
for (usz read_index = 0; read_index < file_stat.size; read_index += transfer_block_size)
{
const usz read_size = std::min<usz>(transfer_block_size, file_stat.size - read_index);
// Read file data
ar.data.resize(ar.data.size() + read_size);
ensure(fd.read_at(read_index, ar.data.data() + old_size, read_size) == read_size);
// Set position to the end of data, so breathe() would work correctly
ar.seek_end();
// Allow flushing to file if needed
ar.breathe();
}
}
// Align
const usz diff = ar.pos - old_pos;
ar.data.resize(ar.data.size() + utils::align(diff, 512) - diff);
ar.seek_end();
fd.close();
ensure(fs::utime(file_name, file_stat.atime, file_stat.mtime));
}
else
{
ensure(false);
}
};
auto save_header = [&](const fs::stat_t& stat, const std::string& name)
{
static_assert(sizeof(TARHeader) == 512);
std::string_view saved_path{name.size() == src_dir_pos ? name.c_str() : &::at32(name, src_dir_pos), name.size() - src_dir_pos};
if (is_null)
{
ar.pos += sizeof(TARHeader);
return;
}
if (usz pos = saved_path.find_first_not_of(fs::delim); pos != umax)
{
saved_path = saved_path.substr(pos, saved_path.size());
}
else
{
// Target the destination directory, I do not know if this is compliant with TAR format
saved_path = "/"sv;
}
TARHeader header{};
std::memcpy(header.magic, "ustar ", 6);
// Prefer saving to name field as much as we can
// If it doesn't fit, save 100 characters at name and 155 characters preceding to it at max
const u64 prefix_size = std::clamp<usz>(saved_path.size(), 100, 255) - 100;
std::memcpy(header.prefix, saved_path.data(), prefix_size);
const u64 name_size = std::min<usz>(saved_path.size(), 255) - prefix_size;
std::memcpy(header.name, saved_path.data() + prefix_size, name_size);
write_octal(header.size, stat.is_directory ? 0 : stat.size);
write_octal(header.mtime, stat.mtime);
write_octal(header.padding, stat.atime);
header.filetype = stat.is_directory ? '5' : '0';
ar(header);
ar.breathe();
};
fs::stat_t stat{};
if (src_dir_pos == umax)
{
// First call, get source directory string size so it can be cut from entry paths
src_dir_pos = target_path.size();
}
if (has_evaluated_results)
{
// Save from cached data by previous call
for (auto&& entry : entries)
{
ensure(entry.name.starts_with(target_path));
save_header(entry, entry.name);
if (!entry.is_directory)
{
save_file(entry, entry.name);
}
}
}
else
{
if (entries.empty())
{
if (!fs::get_stat(target_path, stat))
{
// Revert (this entry should not be included if func returns false)
ar.data.resize(old_size);
return;
}
save_header(stat, target_path);
// Optimization: avoid saving to list if this is not an evaluation call
if (is_null)
{
static_cast<fs::stat_t&>(entries.emplace_back()) = stat;
entries.back().name = target_path;
}
}
else
{
ar.data.resize(ar.data.size() + stat.size);
ensure(fd.read(ar.data.data() + old_size2, stat.size) == stat.size);
stat = entries.back();
save_header(stat, entries.back().name);
}
// Align
ar.data.resize(old_size2 + utils::align(ar.data.size() - old_size2, 512));
if (stat.is_directory)
{
bool exists = false;
fd.close();
fs::utime(target_path, stat.atime, stat.mtime);
for (auto&& entry : fs::dir(target_path))
{
exists = true;
if (entry.name.find_first_not_of('.') == umax)
{
continue;
}
entry.name = target_path.ends_with('/') ? target_path + entry.name : target_path + '/' + entry.name;
if (!entry.is_directory)
{
save_header(entry, entry.name);
save_file(entry, entry.name);
// TAR is an old format which does not depend on previous data so memory ventilation is trivial here
ar.breathe();
entries.emplace_back(std::move(entry));
}
else
{
if (!is_null)
{
// Optimization: avoid saving to list if this is not an evaluation call
entries.clear();
}
entries.emplace_back(std::move(entry));
save_directory(::as_rvalue(entries.back().name), ar, func, std::move(entries), false, src_dir_pos);
}
}
ensure(exists);
}
else
{
fs::dir_entry entry{};
entry.name = target_path;
static_cast<fs::stat_t&>(entry) = stat;
save_file(entry, entry.name);
}
ar.breathe();
}
TARHeader header{};
std::memcpy(header.magic, "ustar ", 6);
// Prefer saving to name field as much as we can
// If it doesn't fit, save 100 characters at name and 155 characters preceding to it at max
const u64 prefix_size = std::clamp<usz>(saved_path.size(), 100, 255) - 100;
std::memcpy(header.prefix, saved_path.data(), prefix_size);
const u64 name_size = std::min<usz>(saved_path.size(), 255) - prefix_size;
std::memcpy(header.name, saved_path.data() + prefix_size, name_size);
write_octal(header.size, stat.is_directory ? 0 : stat.size);
write_octal(header.mtime, stat.mtime);
write_octal(header.padding, stat.atime);
header.filetype = stat.is_directory ? '5' : '0';
std::memcpy(ar.data.data() + old_size, &header, sizeof(header));
// TAR is an old format which does not depend on previous data so memory ventilation is trivial here
ar.breathe();
}
bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file)

14
rpcs3/Loader/TAR.h
View File

@ -1,5 +1,7 @@
#pragma once
#include "util/types.hpp"
#include <map>
struct TARHeader
@ -15,11 +17,14 @@ struct TARHeader
char dontcare2[82];
char prefix[155];
char padding[12]; // atime for RPCS3
ENABLE_BITWISE_SERIALIZATION;
};
namespace fs
{
class file;
struct dir_entry;
}
namespace utils
@ -29,7 +34,9 @@ namespace utils
class tar_object
{
const fs::file& m_file;
const fs::file* m_file;
utils::serial* m_ar;
const usz m_ar_tar_start;
usz largest_offset = 0; // We store the largest offset so we can continue to scan from there.
std::map<std::string, std::pair<u64, TARHeader>> m_map{}; // Maps path to offset of file data and its header
@ -38,10 +45,11 @@ class tar_object
public:
tar_object(const fs::file& file);
tar_object(utils::serial& ar);
std::vector<std::string> get_filenames();
fs::file get_file(const std::string& path);
std::unique_ptr<utils::serial> get_file(const std::string& path, std::string* new_file_path = nullptr);
using process_func = std::function<bool(const fs::file&, std::string&, utils::serial&)>;
@ -49,7 +57,7 @@ public:
// Allow to optionally specify explicit mount point (which may be directory meant for extraction)
bool extract(std::string prefix_path = {}, bool is_vfs = false);
static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::string append_path = {});
static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::vector<fs::dir_entry>&& = std::vector<fs::dir_entry>{}, bool has_evaluated_results = false, usz src_dir_pos = umax);
};
bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file = {});

4
rpcs3/rpcs3qt/game_list_frame.cpp
View File

@ -1078,9 +1078,9 @@ void game_list_frame::ShowContextMenu(const QPoint &pos)
});
}
extern bool is_savestate_compatible(fs::file&& file);
extern bool is_savestate_compatible(fs::file&& file, std::string_view filepath);
if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate)))
if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate), sstate))
{
QAction* boot_state = menu.addAction(is_current_running_game
? tr("&Reboot with savestate")

18
rpcs3/rpcs3qt/main_window.cpp
View File

@ -56,6 +56,7 @@
#include "Emu/vfs_config.h"
#include "Emu/System.h"
#include "Emu/system_utils.hpp"
#include "Emu/savestate_utils.hpp"
#include "Crypto/unpkg.h"
#include "Crypto/unself.h"
@ -589,7 +590,7 @@ void main_window::BootSavestate()
}
const QString file_path = QFileDialog::getOpenFileName(this, tr("Select Savestate To Boot"), qstr(fs::get_cache_dir() + "/savestates/"), tr(
"Savestate files (*.SAVESTAT);;"
"Savestate files (*.SAVESTAT *.SAVESTAT.gz);;"
"All files (*.*)"),
Q_NULLPTR, QFileDialog::DontResolveSymlinks);
@ -1529,7 +1530,18 @@ void main_window::HandlePupInstallation(const QString& file_path, const QString&
{
for (const auto& update_filename : update_filenames)
{
fs::file update_file = update_files.get_file(update_filename);
auto update_file_stream = update_files.get_file(update_filename);
if (update_file_stream->m_file_handler)
{
// Forcefully read all the data
update_file_stream->pop<char>();
update_file_stream->pos = umax;
update_file_stream->pos /= 2; // Avoid internal overflows
update_file_stream->m_file_handler->handle_file_op(*update_file_stream, update_file_stream->pos, 1, nullptr);
}
fs::file update_file = fs::make_stream(std::move(update_file_stream->data));
SCEDecrypter self_dec(update_file);
self_dec.LoadHeaders();
@ -3558,7 +3570,7 @@ main_window::drop_type main_window::IsValidFile(const QMimeData& md, QStringList
type = drop_type::drop_rrc;
}
// The emulator allows to execute ANY filetype, just not from drag-and-drop because it is confusing to users
else if (suffix_lo == "savestat" || suffix_lo == "sprx" || suffix_lo == "self" || suffix_lo == "bin" || suffix_lo == "prx" || suffix_lo == "elf" || suffix_lo == "o")
else if (path.toLower().endsWith(".savestat.gz") || suffix_lo == "savestat" || suffix_lo == "sprx" || suffix_lo == "self" || suffix_lo == "bin" || suffix_lo == "prx" || suffix_lo == "elf" || suffix_lo == "o")
{
type = drop_type::drop_game;
}

152
rpcs3/util/serialization.hpp
View File

@ -39,8 +39,31 @@ namespace utils
serialization_file_handler() = default;
virtual ~serialization_file_handler() = default;
// Handle read/write operations
virtual bool handle_file_op(serial& ar, usz pos, usz size, const void* data = nullptr) = 0;
virtual usz get_size(const utils::serial& ar, usz recommended = umax) const = 0;
// Obtain data size (targets to be only higher than 'recommended' and thus may not be accurate)
virtual usz get_size(const utils::serial& /*ar*/, usz /*recommended*/) const
{
return 0;
}
// Skip reading some (compressed) data
virtual void skip_until(utils::serial& /*ar*/)
{
}
// Detect empty stream (TODO: Clean this, instead perhaps use a magic static representing empty stream)
virtual bool is_null() const
{
return false;
}
virtual void finalize_block(utils::serial& /*ar*/)
{
}
virtual void finalize(utils::serial&) = 0;
};
struct serial
@ -48,12 +71,16 @@ namespace utils
std::vector<u8> data;
usz data_offset = 0;
usz pos = 0;
usz m_max_data = umax;
bool m_is_writing = true;
bool m_avoid_large_prefetch = false;
std::unique_ptr<serialization_file_handler> m_file_handler;
serial() noexcept = default;
serial(const serial&) = delete;
serial& operator=(const serial&) = delete;
explicit serial(serial&&) noexcept = default;
serial& operator=(serial&&) noexcept = default;
~serial() noexcept = default;
// Checks if this instance is currently used for serialization
@ -80,27 +107,34 @@ namespace utils
return true;
}
if (m_file_handler && m_file_handler->is_null())
{
// Instead of doing nothing at all, increase pos so it would be possible to estimate memory requirements
pos += size;
return true;
}
// Overflow check
ensure(~pos >= size);
ensure(~pos >= size - 1);
if (is_writing())
{
ensure(pos >= data_offset);
const auto ptr = reinterpret_cast<const u8*>(memory_provider());
data.insert(data.begin() + pos - data_offset, ptr, ptr + size);
data.insert(data.begin() + (pos - data_offset), ptr, ptr + size);
pos += size;
return true;
}
if (data.empty() || pos < data_offset || pos + size > data.size() + data_offset)
if (data.empty() || pos < data_offset || pos + (size - 1) > (data.size() - 1) + data_offset)
{
// Load from file
ensure(m_file_handler);
ensure(m_file_handler->handle_file_op(*this, pos, size, nullptr));
ensure(!data.empty() && pos >= data_offset && pos + size <= data.size() + data_offset);
ensure(!data.empty() && pos >= data_offset && pos + (size - 1) <= (data.size() - 1) + data_offset);
}
std::memcpy(const_cast<void*>(static_cast<const void*>(memory_provider())), data.data() + pos - data_offset, size);
std::memcpy(const_cast<void*>(static_cast<const void*>(memory_provider())), data.data() + (pos - data_offset), size);
pos += size;
return true;
}
@ -169,7 +203,8 @@ namespace utils
}
// std::vector, std::basic_string
template <typename T> requires FastRandomAccess<T> && ListAlike<T>
// Discourage using std::pair/tuple with vectors because it eliminates the possibility of bitwise optimization
template <typename T> requires FastRandomAccess<T> && ListAlike<T> && (!TupleAlike<typename T::value_type>)
bool serialize(T& obj)
{
if (is_writing())
@ -194,6 +229,13 @@ namespace utils
return true;
}
obj.clear();
if (m_file_handler && m_file_handler->is_null())
{
return true;
}
usz size = 0;
if (!deserialize_vle(size))
{
@ -211,7 +253,6 @@ namespace utils
else
{
// TODO: Postpone resizing to after file bounds checks
obj.clear();
obj.resize(size);
for (auto&& value : obj)
@ -270,6 +311,11 @@ namespace utils
obj.clear();
if (m_file_handler && m_file_handler->is_null())
{
return true;
}
usz size = 0;
if (!deserialize_vle(size))
{
@ -326,16 +372,23 @@ namespace utils
}
// Wrapper for serialize(T&), allows to pass multiple objects at once
template <typename... Args>
bool operator()(Args&&... args)
template <typename... Args> requires (sizeof...(Args) != 0)
bool operator()(Args&&... args) noexcept
{
return ((AUDIT(!std::is_const_v<std::remove_reference_t<Args>> || is_writing())
, serialize(const_cast<std::remove_cvref_t<Args>&>(static_cast<const Args&>(args)))), ...);
}
// Code style utility, for when utils::serial is a pointer for example
template <typename... Args> requires (sizeof...(Args) > 1 || !(std::is_convertible_v<Args&&, Args&> && ...))
bool serialize(Args&&... args)
{
return this->operator()(std::forward<Args>(args)...);
}
// Convert serialization manager to deserializion manager
// If no arg is provided reuse saved buffer
void set_reading_state(std::vector<u8>&& _data = std::vector<u8>{})
void set_reading_state(std::vector<u8>&& _data = std::vector<u8>{}, bool avoid_large_prefetch = false)
{
if (!_data.empty())
{
@ -343,7 +396,7 @@ namespace utils
}
m_is_writing = false;
m_avoid_large_prefetch = false;
m_avoid_large_prefetch = avoid_large_prefetch;
pos = 0;
data_offset = 0;
}
@ -365,15 +418,19 @@ namespace utils
return pos;
}
usz seek_pos(usz val, bool empty_data = false)
usz seek_pos(usz val, bool cleanup = false)
{
const usz old_pos = std::exchange(pos, val);
if (empty_data || data.empty())
if (cleanup || data.empty())
{
// Relocate future data
data.clear();
data_offset = pos;
if (m_file_handler)
{
m_file_handler->skip_until(*this);
}
breathe();
}
return old_pos;
@ -403,7 +460,7 @@ namespace utils
template <typename T> requires (std::is_copy_constructible_v<std::remove_const_t<T>>) && (std::is_constructible_v<std::remove_const_t<T>> || Bitcopy<std::remove_const_t<T>> ||
std::is_constructible_v<std::remove_const_t<T>, stx::exact_t<serial&>> || TupleAlike<std::remove_const_t<T>>)
operator T()
operator T() noexcept
{
AUDIT(!is_writing());
@ -433,6 +490,13 @@ namespace utils
}
}
// Code style utility wrapper for operator T()
template <typename T>
T pop()
{
return this->operator T();
}
void swap_handler(serial& ar)
{
std::swap(ar.m_file_handler, this->m_file_handler);
@ -440,7 +504,40 @@ namespace utils
usz get_size(usz recommended = umax) const
{
return m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size();
recommended = std::min<usz>(recommended, m_max_data);
return std::min<usz>(m_max_data, m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size());
}
template <typename T> requires (Bitcopy<T>)
usz predict_object_size(const T&)
{
return sizeof(T);
}
template <typename T> requires FastRandomAccess<T> && (!ListAlike<T>) && (!Bitcopy<T>)
usz predict_object_size(const T& obj)
{
return std::size(obj) * sizeof(obj[0]);
}
template <typename T> requires (std::is_copy_constructible_v<std::remove_reference_t<T>> && std::is_constructible_v<std::remove_reference_t<T>>)
usz try_read(T&& obj)
{
if (is_writing())
{
return 0;
}
const usz end_pos = pos + predict_object_size(std::forward<T>(obj));
const usz size = get_size(end_pos);
if (size >= end_pos)
{
serialize(std::forward<T>(obj));
return 0;
}
return end_pos - size;
}
template <typename T> requires (std::is_copy_constructible_v<T> && std::is_constructible_v<T> && Bitcopy<T>)
@ -457,14 +554,27 @@ namespace utils
if (size >= end_pos)
{
u8 buf[sizeof(type)]{};
ensure(raw_serialize(buf, sizeof(buf)));
return {true, std::bit_cast<type>(buf)};
return {true, this->operator T()};
}
return {};
}
void patch_raw_data(usz pos, const void* data, usz size)
{
if (m_file_handler && m_file_handler->is_null())
{
return;
}
if (!size)
{
return;
}
std::memcpy(&::at32(this->data, pos - data_offset + size - 1) - (size - 1), data, size);
}
// Returns true if valid, can be invalidated by setting pos to umax
// Used when an invalid state is encountered somewhere in a place we can't check success code such as constructor)
bool is_valid() const