From 19944eeed0a1b1686024616836c7b892eb587c85 Mon Sep 17 00:00:00 2001 From: Nekotekina Date: Sat, 17 Mar 2018 20:41:35 +0300 Subject: [PATCH] Implement SPRX precompilation Automatically precompile firmware modules Add "Create LLVM Cache" menu (for games) Reimplement jit_compiler::cpu as static method --- Utilities/JIT.cpp | 13 ++- Utilities/JIT.h | 7 +- rpcs3/Emu/Cell/PPUAnalyser.h | 11 +++ rpcs3/Emu/Cell/PPUModule.cpp | 3 +- rpcs3/Emu/Cell/PPUThread.cpp | 72 ++++++++++------ rpcs3/Emu/System.cpp | 134 +++++++++++++++++++++++++++++- rpcs3/rpcs3qt/game_list_frame.cpp | 8 ++ rpcs3/rpcs3qt/main_window.cpp | 3 + 8 files changed, 214 insertions(+), 37 deletions(-) diff --git a/Utilities/JIT.cpp b/Utilities/JIT.cpp index 2b262d0531..ad87a2ebce 100644 --- a/Utilities/JIT.cpp +++ b/Utilities/JIT.cpp @@ -403,10 +403,10 @@ public: } }; -jit_compiler::jit_compiler(const std::unordered_map& _link, std::string _cpu) - : m_link(_link) - , m_cpu(std::move(_cpu)) +std::string jit_compiler::cpu(const std::string& _cpu) { + std::string m_cpu = _cpu; + if (m_cpu.empty()) { m_cpu = llvm::sys::getHostCPUName(); @@ -426,6 +426,13 @@ jit_compiler::jit_compiler(const std::unordered_map& _link, st } } + return m_cpu; +} + +jit_compiler::jit_compiler(const std::unordered_map& _link, const std::string& _cpu) + : m_link(_link) + , m_cpu(cpu(_cpu)) +{ std::string result; if (m_link.empty()) diff --git a/Utilities/JIT.h b/Utilities/JIT.h index 8b7da39bc8..6ba6b1aff4 100644 --- a/Utilities/JIT.h +++ b/Utilities/JIT.h @@ -40,7 +40,7 @@ class jit_compiler final std::string m_cpu; public: - jit_compiler(const std::unordered_map& _link, std::string _cpu); + jit_compiler(const std::unordered_map& _link, const std::string& _cpu); ~jit_compiler(); // Get LLVM context @@ -65,10 +65,7 @@ public: static std::unordered_map add(std::unordered_map); // Get CPU info - const std::string& cpu() const - { - return m_cpu; - } + static std::string cpu(const std::string& _cpu); // Check JIT purpose bool is_primary() const diff --git a/rpcs3/Emu/Cell/PPUAnalyser.h b/rpcs3/Emu/Cell/PPUAnalyser.h index e2bc5eedd2..6270593245 100644 --- a/rpcs3/Emu/Cell/PPUAnalyser.h +++ b/rpcs3/Emu/Cell/PPUAnalyser.h @@ -42,6 +42,17 @@ struct ppu_reloc u32 addr; u32 type; u64 data; + + // Operator for sorting + bool operator <(const ppu_reloc& rhs) const + { + return addr < rhs.addr; + } + + bool operator <(u32 rhs) const + { + return addr < rhs; + } }; // PPU Segment Information diff --git a/rpcs3/Emu/Cell/PPUModule.cpp b/rpcs3/Emu/Cell/PPUModule.cpp index 2421bfadfe..afd57d673e 100644 --- a/rpcs3/Emu/Cell/PPUModule.cpp +++ b/rpcs3/Emu/Cell/PPUModule.cpp @@ -929,7 +929,7 @@ std::shared_ptr ppu_load_prx(const ppu_prx_object& elf, const std::stri prx->specials = ppu_load_exports(link, lib_info->exports_start, lib_info->exports_end); prx->imports = ppu_load_imports(prx->relocs, link, lib_info->imports_start, lib_info->imports_end); - + std::stable_sort(prx->relocs.begin(), prx->relocs.end()); prx->analyse(lib_info->toc, 0); } else @@ -1191,6 +1191,7 @@ void ppu_load_exec(const ppu_exec_object& elf) ppu_load_exports(link, proc_prx_param.libent_start, proc_prx_param.libent_end); ppu_load_imports(_main->relocs, link, proc_prx_param.libstub_start, proc_prx_param.libstub_end); + std::stable_sort(_main->relocs.begin(), _main->relocs.end()); } break; } diff --git a/rpcs3/Emu/Cell/PPUThread.cpp b/rpcs3/Emu/Cell/PPUThread.cpp index e9cc8cf209..995799ccac 100644 --- a/rpcs3/Emu/Cell/PPUThread.cpp +++ b/rpcs3/Emu/Cell/PPUThread.cpp @@ -165,7 +165,7 @@ extern const ppu_decoder g_ppu_interpreter_fast([](auto& t extern void ppu_initialize(); extern void ppu_initialize(const ppu_module& info); -static void ppu_initialize2(class jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name, u32 fragment_index, atomic_t&); +static void ppu_initialize2(class jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name, u32 fragment_index, const std::shared_ptr>&); extern void ppu_execute_syscall(ppu_thread& ppu, u64 code); // Get pointer to executable cache @@ -1037,7 +1037,7 @@ static bool adde_carry(u64 a, u64 b, bool c) extern void ppu_initialize() { - const auto _main = fxm::withdraw(); + const auto _main = fxm::get(); if (!_main) { @@ -1165,16 +1165,10 @@ extern void ppu_initialize(const ppu_module& info) // Compiler mutex (global) static semaphore<> jmutex; - // Initialize semaphore with the max number of threads + // Initialize global semaphore with the max number of threads u32 max_threads = static_cast(g_cfg.core.llvm_threads); s32 thread_count = max_threads > 0 ? std::min(max_threads, std::thread::hardware_concurrency()) : std::thread::hardware_concurrency(); - semaphore jcores(thread_count); - - if (!jcores.get()) - { - // Min value 1 - jcores.post(); - } + static semaphore jcores(std::max(thread_count, 1)); // Worker threads std::vector jthreads; @@ -1188,14 +1182,14 @@ extern void ppu_initialize(const ppu_module& info) // Difference between function name and current location const u32 reloc = info.name.empty() ? 0 : info.segs.at(0).addr; - atomic_t fragment_sync{0}; + std::shared_ptr> fragment_sync = std::make_shared>(0); u32 fragment_count{0}; while (jit_mod.vars.empty() && fpos < info.funcs.size()) { // Initialize compiler instance - if (!jit) + if (!jit && get_current_cpu_thread()) { jit = std::make_shared(s_link_table, g_cfg.core.llvm_cpu); } @@ -1278,8 +1272,32 @@ extern void ppu_initialize(const ppu_module& info) continue; } - // TODO: relocations must be taken into account (TODO) - sha1_update(&ctx, vm::_ptr(block.first), block.second); + // Find relevant relocations + auto low = std::lower_bound(part.relocs.cbegin(), part.relocs.cend(), block.first); + auto high = std::lower_bound(low, part.relocs.cend(), block.first + block.second); + auto addr = block.first; + + for (; low != high; ++low) + { + // Aligned relocation address + const u32 roff = low->addr & ~3; + + if (roff > addr) + { + // Hash from addr to the beginning of the relocation + sha1_update(&ctx, vm::_ptr(addr), roff - addr); + } + + // Hash relocation type instead + const be_t type = low->type; + sha1_update(&ctx, reinterpret_cast(&type), sizeof(type)); + + // Set the next addr + addr = roff + 4; + } + + // Hash from addr to the end of the block + sha1_update(&ctx, vm::_ptr(addr), block.second - (addr - block.first)); } if (reloc) @@ -1297,7 +1315,7 @@ extern void ppu_initialize(const ppu_module& info) } sha1_finish(&ctx, output); - fmt::append(obj_name, "-%016X-%s.obj", reinterpret_cast&>(output), jit->cpu()); + fmt::append(obj_name, "-%016X-%s.obj", reinterpret_cast&>(output), jit_compiler::cpu(g_cfg.core.llvm_cpu)); } if (Emu.IsStopped()) @@ -1317,6 +1335,12 @@ extern void ppu_initialize(const ppu_module& info) // Check object file if (fs::is_file(cache_path + obj_name)) { + if (!jit) + { + LOG_SUCCESS(PPU, "LLVM: Already exists: %s", obj_name); + continue; + } + semaphore_lock lock(jmutex); jit->add(cache_path + obj_name); @@ -1325,7 +1349,7 @@ extern void ppu_initialize(const ppu_module& info) } // Create worker thread for compilation - jthreads.emplace_back([&jit, &jcores, obj_name = obj_name, part = std::move(part), &cache_path, &fragment_sync, findex = ::size32(jthreads)]() + jthreads.emplace_back([&jit, obj_name = obj_name, part = std::move(part), &cache_path, fragment_sync, findex = ::size32(jthreads)]() { // Set low priority thread_ctrl::set_native_priority(-1); @@ -1344,7 +1368,7 @@ extern void ppu_initialize(const ppu_module& info) ppu_initialize2(jit2, part, cache_path, obj_name, findex, fragment_sync); } - if (Emu.IsStopped() || !fs::is_file(cache_path + obj_name)) + if (Emu.IsStopped() || !jit || !fs::is_file(cache_path + obj_name)) { return; } @@ -1356,7 +1380,7 @@ extern void ppu_initialize(const ppu_module& info) } // Initialize fragment count sync var - fragment_sync.exchange(::size32(jthreads)); + fragment_sync->exchange(::size32(jthreads)); // Join worker threads for (auto& thread : jthreads) @@ -1364,7 +1388,7 @@ extern void ppu_initialize(const ppu_module& info) thread.join(); } - if (Emu.IsStopped()) + if (Emu.IsStopped() || !get_current_cpu_thread()) { return; } @@ -1441,7 +1465,7 @@ extern void ppu_initialize(const ppu_module& info) #endif } -static void ppu_initialize2(jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name, u32 fragment_index, atomic_t& fragment_sync) +static void ppu_initialize2(jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name, u32 fragment_index, const std::shared_ptr>& fragment_sync) { #ifdef LLVM_AVAILABLE using namespace llvm; @@ -1525,14 +1549,14 @@ static void ppu_initialize2(jit_compiler& jit, const ppu_module& module_part, co if (module_part.funcs[fi].size) { // Update dialog - Emu.CallAfter([=, max = module_part.funcs.size(), &fragment_sync]() + Emu.CallAfter([=, max = module_part.funcs.size()]() { dlg->ProgressBarSetMsg(0, fmt::format("Compiling %u of %u", fi + 1, fmax)); if (fi * 100 / fmax != (fi + 1) * 100 / fmax) dlg->ProgressBarInc(0, 1); - if (u32 fragment_count = fragment_sync.load()) + if (u32 fragment_count = fragment_sync->load()) dlg->SetMsg(fmt::format("Compiling PPU module (%u of %u):\n%s\nPlease wait...", fragment_index + 1, fragment_count, obj_name)); }); @@ -1559,12 +1583,12 @@ static void ppu_initialize2(jit_compiler& jit, const ppu_module& module_part, co //mpm.run(*module); // Update dialog - Emu.CallAfter([=, &fragment_sync]() + Emu.CallAfter([=]() { dlg->ProgressBarSetMsg(0, "Generating code, this may take a long time..."); dlg->ProgressBarInc(0, 100); - if (u32 fragment_count = fragment_sync.load()) + if (u32 fragment_count = fragment_sync->load()) dlg->SetMsg(fmt::format("Compiling PPU module (%u of %u):\n%s\nPlease wait...", fragment_index + 1, fragment_count, obj_name)); }); diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index f5e7b80eb6..54d1acd6ba 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -8,9 +8,11 @@ #include "Emu/Cell/PPUCallback.h" #include "Emu/Cell/PPUOpcodes.h" #include "Emu/Cell/PPUDisAsm.h" +#include "Emu/Cell/PPUAnalyser.h" #include "Emu/Cell/SPUThread.h" #include "Emu/Cell/RawSPUThread.h" #include "Emu/Cell/lv2/sys_sync.h" +#include "Emu/Cell/lv2/sys_prx.h" #include "Emu/IdManager.h" #include "Emu/RSX/GSRender.h" @@ -26,6 +28,7 @@ #include #include +#include #include "Utilities/GDBDebugServer.h" @@ -39,7 +42,9 @@ extern u64 get_system_time(); extern void ppu_load_exec(const ppu_exec_object&); extern void spu_load_exec(const spu_exec_object&); -extern std::shared_ptr ppu_load_prx(const ppu_prx_object&, const std::string&); +extern void ppu_initialize(const ppu_module&); +extern void ppu_unload_prx(const lv2_prx&); +extern std::shared_ptr ppu_load_prx(const ppu_prx_object&, const std::string&); extern void network_thread_init(); @@ -238,11 +243,10 @@ bool Emulator::BootGame(const std::string& path, bool direct, bool add_only) "/eboot.bin", }; - if (direct && fs::is_file(path)) + if (direct && fs::exists(path)) { m_path = path; Load(add_only); - return true; } @@ -254,7 +258,6 @@ bool Emulator::BootGame(const std::string& path, bool direct, bool add_only) { m_path = elf; Load(add_only); - return true; } } @@ -359,6 +362,17 @@ void Emulator::Load(bool add_only) return sfov; } + if (fs::is_dir(m_path)) + { + // Special case (directory scan) + if (fs::file sfo{m_path + "/PS3_GAME/PARAM.SFO"}) + { + return sfo; + } + + return fs::file{m_path + "/PARAM.SFO"}; + } + if (disc.size()) { // Check previously used category before it's overwritten @@ -436,6 +450,118 @@ void Emulator::Load(bool add_only) vfs::mount("dev_usb000", fmt::replace_all(g_cfg.vfs.dev_usb000, "$(EmulatorDir)", emu_dir)); vfs::mount("app_home", home_dir.empty() ? elf_dir + '/' : fmt::replace_all(home_dir, "$(EmulatorDir)", emu_dir)); + // Special boot mode (directory scan) + if (fs::is_dir(m_path)) + { + m_state = system_state::ready; + GetCallbacks().on_ready(); + vm::init(); + Run(); + m_force_boot = false; + + // Force LLVM recompiler + g_cfg.core.ppu_decoder.from_default(); + + return thread_ctrl::spawn("SPRX Loader", [this] + { + std::vector dir_queue; + dir_queue.emplace_back(m_path + '/'); + + std::queue> thread_queue; + const uint max_threads = std::thread::hardware_concurrency(); + + // Find all .sprx files recursively (TODO: process .mself files) + for (std::size_t i = 0; i < dir_queue.size(); i++) + { + if (Emu.IsStopped()) + { + break; + } + + LOG_NOTICE(LOADER, "Scanning directory: %s", dir_queue[i]); + + for (auto&& entry : fs::dir(dir_queue[i])) + { + if (Emu.IsStopped()) + { + break; + } + + if (entry.is_directory) + { + if (entry.name != "." && entry.name != "..") + { + dir_queue.emplace_back(dir_queue[i] + entry.name + '/'); + } + + continue; + } + + // Check .sprx filename + if (entry.name.size() >= 5 && fmt::to_upper(entry.name).compare(entry.name.size() - 5, 5, ".SPRX", 5) == 0) + { + if (entry.name == "libfs_155.sprx") + { + continue; + } + + // Get full path + const std::string path = dir_queue[i] + entry.name; + + LOG_NOTICE(LOADER, "Trying to load SPRX: %s", path); + + // Some files may fail to decrypt due to the lack of klic + const ppu_prx_object obj = decrypt_self(fs::file(path)); + + if (obj == elf_error::ok) + { + if (auto prx = ppu_load_prx(obj, path)) + { + while (g_thread_count >= max_threads + 2) + { + std::this_thread::sleep_for(10ms); + } + + thread_queue.emplace(); + + thread_ctrl::spawn(thread_queue.back(), "Worker " + std::to_string(thread_queue.size()), [_prx = std::move(prx)] + { + ppu_initialize(*_prx); + ppu_unload_prx(*_prx); + }); + } + } + else + { + LOG_ERROR(LOADER, "Failed to load SPRX '%s' (%s)", path, obj.get_error()); + } + } + } + } + + // Join every thread and print exceptions + while (!thread_queue.empty()) + { + try + { + thread_queue.front()->join(); + } + catch (const std::exception& e) + { + LOG_FATAL(LOADER, "[%s] %s thrown: %s", thread_queue.front()->get_name(), typeid(e).name(), e.what()); + } + + thread_queue.pop(); + } + + // Exit "process" + Emu.CallAfter([] + { + Emu.Stop(); + }); + }); + } + // Detect boot location const std::string hdd0_game = vfs::get("/dev_hdd0/game/"); const std::string hdd0_disc = vfs::get("/dev_hdd0/disc/"); diff --git a/rpcs3/rpcs3qt/game_list_frame.cpp b/rpcs3/rpcs3qt/game_list_frame.cpp index 28596ea20d..45764fc35b 100644 --- a/rpcs3/rpcs3qt/game_list_frame.cpp +++ b/rpcs3/rpcs3qt/game_list_frame.cpp @@ -549,6 +549,7 @@ void game_list_frame::ShowSpecifiedContextMenu(const QPoint &pos, int row) f.setBold(true); boot->setFont(f); QAction* configure = myMenu.addAction(tr("&Configure")); + QAction* createLLVMCache = myMenu.addAction(tr("&Create LLVM Cache")); myMenu.addSeparator(); QAction* hide_serial = myMenu.addAction(tr("&Hide From Game List")); hide_serial->setCheckable(true); @@ -591,6 +592,13 @@ void game_list_frame::ShowSpecifiedContextMenu(const QPoint &pos, int row) xgui_settings->SetValue(gui::gl_hidden_list, QStringList(m_hidden_list.toList())); Refresh(); }); + connect(createLLVMCache, &QAction::triggered, [=] + { + Emu.SetForceBoot(true); + Emu.Stop(); + Emu.SetForceBoot(true); + Emu.BootGame(currGame.path, true); + }); connect(removeGame, &QAction::triggered, [=] { QMessageBox* mb = new QMessageBox(QMessageBox::Question, tr("Confirm %1 Removal").arg(qstr(currGame.category)), tr("Permanently remove %1 from drive?").arg(qstr(currGame.name)), QMessageBox::Yes | QMessageBox::No, this); diff --git a/rpcs3/rpcs3qt/main_window.cpp b/rpcs3/rpcs3qt/main_window.cpp index 43ebcc4e68..42c8c57ba5 100644 --- a/rpcs3/rpcs3qt/main_window.cpp +++ b/rpcs3/rpcs3qt/main_window.cpp @@ -526,6 +526,9 @@ void main_window::InstallPup(const QString& dropPath) { LOG_SUCCESS(GENERAL, "Successfully installed PS3 firmware version %s.", version_string); guiSettings->ShowInfoBox(gui::ib_pup_success, tr("Success!"), tr("Successfully installed PS3 firmware and LLE Modules!"), this); + + Emu.SetForceBoot(true); + Emu.BootGame(Emu.GetLibDir(), true); } }