Implement SPRX precompilation

Automatically precompile firmware modules
Add "Create LLVM Cache" menu (for games)
Reimplement jit_compiler::cpu as static method

Utilities/JIT.cpp

@@ -403,10 +403,10 @@ public:
 	}
 };
 
-jit_compiler::jit_compiler(const std::unordered_map<std::string, u64>& _link, std::string _cpu)
-	: m_link(_link)
-	, m_cpu(std::move(_cpu))
+std::string jit_compiler::cpu(const std::string& _cpu)
 {
+	std::string m_cpu = _cpu;
+
 	if (m_cpu.empty())
 	{
 		m_cpu = llvm::sys::getHostCPUName();
@@ -426,6 +426,13 @@ jit_compiler::jit_compiler(const std::unordered_map<std::string, u64>& _link, st
 		}
 	}
 
+	return m_cpu;
+}
+
+jit_compiler::jit_compiler(const std::unordered_map<std::string, u64>& _link, const std::string& _cpu)
+	: m_link(_link)
+	, m_cpu(cpu(_cpu))
+{
 	std::string result;
 
 	if (m_link.empty())

Utilities/JIT.h

@@ -40,7 +40,7 @@ class jit_compiler final
 	std::string m_cpu;
 
 public:
-	jit_compiler(const std::unordered_map<std::string, u64>& _link, std::string _cpu);
+	jit_compiler(const std::unordered_map<std::string, u64>& _link, const std::string& _cpu);
 	~jit_compiler();
 
 	// Get LLVM context
@@ -65,10 +65,7 @@ public:
 	static std::unordered_map<std::string, u64> add(std::unordered_map<std::string, std::string>);
 
 	// Get CPU info
-	const std::string& cpu() const
-	{
-		return m_cpu;
-	}
+	static std::string cpu(const std::string& _cpu);
 
 	// Check JIT purpose
 	bool is_primary() const

rpcs3/Emu/Cell/PPUAnalyser.h

@@ -42,6 +42,17 @@ struct ppu_reloc
 	u32 addr;
 	u32 type;
 	u64 data;
+
+	// Operator for sorting
+	bool operator <(const ppu_reloc& rhs) const
+	{
+		return addr < rhs.addr;
+	}
+
+	bool operator <(u32 rhs) const
+	{
+		return addr < rhs;
+	}
 };
 
 // PPU Segment Information

rpcs3/Emu/Cell/PPUModule.cpp

@@ -929,7 +929,7 @@ std::shared_ptr<lv2_prx> ppu_load_prx(const ppu_prx_object& elf, const std::stri
 
 		prx->specials = ppu_load_exports(link, lib_info->exports_start, lib_info->exports_end);
 		prx->imports = ppu_load_imports(prx->relocs, link, lib_info->imports_start, lib_info->imports_end);
-
+		std::stable_sort(prx->relocs.begin(), prx->relocs.end());
 		prx->analyse(lib_info->toc, 0);
 	}
 	else
@@ -1191,6 +1191,7 @@ void ppu_load_exec(const ppu_exec_object& elf)
 
 				ppu_load_exports(link, proc_prx_param.libent_start, proc_prx_param.libent_end);
 				ppu_load_imports(_main->relocs, link, proc_prx_param.libstub_start, proc_prx_param.libstub_end);
+				std::stable_sort(_main->relocs.begin(), _main->relocs.end());
 			}
 			break;
 		}

rpcs3/Emu/Cell/PPUThread.cpp

@@ -165,7 +165,7 @@ extern const ppu_decoder<ppu_interpreter_fast> g_ppu_interpreter_fast([](auto& t
 
 extern void ppu_initialize();
 extern void ppu_initialize(const ppu_module& info);
-static void ppu_initialize2(class jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name, u32 fragment_index, atomic_t<u32>&);
+static void ppu_initialize2(class jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name, u32 fragment_index, const std::shared_ptr<atomic_t<u32>>&);
 extern void ppu_execute_syscall(ppu_thread& ppu, u64 code);
 
 // Get pointer to executable cache
@@ -1037,7 +1037,7 @@ static bool adde_carry(u64 a, u64 b, bool c)
 
 extern void ppu_initialize()
 {
-	const auto _main = fxm::withdraw<ppu_module>();
+	const auto _main = fxm::get<ppu_module>();
 
 	if (!_main)
 	{
@@ -1165,16 +1165,10 @@ extern void ppu_initialize(const ppu_module& info)
 	// Compiler mutex (global)
 	static semaphore<> jmutex;
 
-	// Initialize semaphore with the max number of threads
+	// Initialize global semaphore with the max number of threads
 	u32 max_threads = static_cast<u32>(g_cfg.core.llvm_threads);
 	s32 thread_count = max_threads > 0 ? std::min(max_threads, std::thread::hardware_concurrency()) : std::thread::hardware_concurrency();
-	semaphore<INT32_MAX> jcores(thread_count);
-
-	if (!jcores.get())
-	{
-		// Min value 1
-		jcores.post();
-	}
+	static semaphore<INT32_MAX> jcores(std::max<s32>(thread_count, 1));
 
 	// Worker threads
 	std::vector<std::thread> jthreads;
@@ -1188,14 +1182,14 @@ extern void ppu_initialize(const ppu_module& info)
 	// Difference between function name and current location
 	const u32 reloc = info.name.empty() ? 0 : info.segs.at(0).addr;
 
-	atomic_t<u32> fragment_sync{0};
+	std::shared_ptr<atomic_t<u32>> fragment_sync = std::make_shared<atomic_t<u32>>(0);
 
 	u32 fragment_count{0};
 
 	while (jit_mod.vars.empty() && fpos < info.funcs.size())
 	{
 		// Initialize compiler instance
-		if (!jit)
+		if (!jit && get_current_cpu_thread())
 		{
 			jit = std::make_shared<jit_compiler>(s_link_table, g_cfg.core.llvm_cpu);
 		}
@@ -1278,8 +1272,32 @@ extern void ppu_initialize(const ppu_module& info)
 						continue;
 					}
 
-					// TODO: relocations must be taken into account (TODO)
-					sha1_update(&ctx, vm::_ptr<const u8>(block.first), block.second);
+					// Find relevant relocations
+					auto low = std::lower_bound(part.relocs.cbegin(), part.relocs.cend(), block.first);
+					auto high = std::lower_bound(low, part.relocs.cend(), block.first + block.second);
+					auto addr = block.first;
+
+					for (; low != high; ++low)
+					{
+						// Aligned relocation address
+						const u32 roff = low->addr & ~3;
+
+						if (roff > addr)
+						{
+							// Hash from addr to the beginning of the relocation
+							sha1_update(&ctx, vm::_ptr<const u8>(addr), roff - addr);
+						}
+
+						// Hash relocation type instead
+						const be_t<u32> type = low->type;
+						sha1_update(&ctx, reinterpret_cast<const u8*>(&type), sizeof(type));
+
+						// Set the next addr
+						addr = roff + 4;
+					}
+
+					// Hash from addr to the end of the block
+					sha1_update(&ctx, vm::_ptr<const u8>(addr), block.second - (addr - block.first));
 				}
 
 				if (reloc)
@@ -1297,7 +1315,7 @@ extern void ppu_initialize(const ppu_module& info)
 			}
 
 			sha1_finish(&ctx, output);
-			fmt::append(obj_name, "-%016X-%s.obj", reinterpret_cast<be_t<u64>&>(output), jit->cpu());
+			fmt::append(obj_name, "-%016X-%s.obj", reinterpret_cast<be_t<u64>&>(output), jit_compiler::cpu(g_cfg.core.llvm_cpu));
 		}
 
 		if (Emu.IsStopped())
@@ -1317,6 +1335,12 @@ extern void ppu_initialize(const ppu_module& info)
 		// Check object file
 		if (fs::is_file(cache_path + obj_name))
 		{
+			if (!jit)
+			{
+				LOG_SUCCESS(PPU, "LLVM: Already exists: %s", obj_name);
+				continue;
+			}
+
 			semaphore_lock lock(jmutex);
 			jit->add(cache_path + obj_name);
 
@@ -1325,7 +1349,7 @@ extern void ppu_initialize(const ppu_module& info)
 		}
 
 		// Create worker thread for compilation
-		jthreads.emplace_back([&jit, &jcores, obj_name = obj_name, part = std::move(part), &cache_path, &fragment_sync, findex = ::size32(jthreads)]()
+		jthreads.emplace_back([&jit, obj_name = obj_name, part = std::move(part), &cache_path, fragment_sync, findex = ::size32(jthreads)]()
 		{
 			// Set low priority
 			thread_ctrl::set_native_priority(-1);
@@ -1344,7 +1368,7 @@ extern void ppu_initialize(const ppu_module& info)
 				ppu_initialize2(jit2, part, cache_path, obj_name, findex, fragment_sync);
 			}
 
-			if (Emu.IsStopped() || !fs::is_file(cache_path + obj_name))
+			if (Emu.IsStopped() || !jit || !fs::is_file(cache_path + obj_name))
 			{
 				return;
 			}
@@ -1356,7 +1380,7 @@ extern void ppu_initialize(const ppu_module& info)
 	}
 
 	// Initialize fragment count sync var
-	fragment_sync.exchange(::size32(jthreads));
+	fragment_sync->exchange(::size32(jthreads));
 
 	// Join worker threads
 	for (auto& thread : jthreads)
@@ -1364,7 +1388,7 @@ extern void ppu_initialize(const ppu_module& info)
 		thread.join();
 	}
 
-	if (Emu.IsStopped())
+	if (Emu.IsStopped() || !get_current_cpu_thread())
 	{
 		return;
 	}
@@ -1441,7 +1465,7 @@ extern void ppu_initialize(const ppu_module& info)
 #endif
 }
 
-static void ppu_initialize2(jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name, u32 fragment_index, atomic_t<u32>& fragment_sync)
+static void ppu_initialize2(jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name, u32 fragment_index, const std::shared_ptr<atomic_t<u32>>& fragment_sync)
 {
 #ifdef LLVM_AVAILABLE
 	using namespace llvm;
@@ -1525,14 +1549,14 @@ static void ppu_initialize2(jit_compiler& jit, const ppu_module& module_part, co
 			if (module_part.funcs[fi].size)
 			{
 				// Update dialog
-				Emu.CallAfter([=, max = module_part.funcs.size(), &fragment_sync]()
+				Emu.CallAfter([=, max = module_part.funcs.size()]()
 				{
 					dlg->ProgressBarSetMsg(0, fmt::format("Compiling %u of %u", fi + 1, fmax));
 
 					if (fi * 100 / fmax != (fi + 1) * 100 / fmax)
 						dlg->ProgressBarInc(0, 1);
 
-					if (u32 fragment_count = fragment_sync.load())
+					if (u32 fragment_count = fragment_sync->load())
 						dlg->SetMsg(fmt::format("Compiling PPU module (%u of %u):\n%s\nPlease wait...", fragment_index + 1, fragment_count, obj_name));
 				});
 
@@ -1559,12 +1583,12 @@ static void ppu_initialize2(jit_compiler& jit, const ppu_module& module_part, co
 		//mpm.run(*module);
 
 		// Update dialog
-		Emu.CallAfter([=, &fragment_sync]()
+		Emu.CallAfter([=]()
 		{
 			dlg->ProgressBarSetMsg(0, "Generating code, this may take a long time...");
 			dlg->ProgressBarInc(0, 100);
 
-			if (u32 fragment_count = fragment_sync.load())
+			if (u32 fragment_count = fragment_sync->load())
 				dlg->SetMsg(fmt::format("Compiling PPU module (%u of %u):\n%s\nPlease wait...", fragment_index + 1, fragment_count, obj_name));
 		});
 

rpcs3/Emu/System.cpp

@@ -8,9 +8,11 @@
 #include "Emu/Cell/PPUCallback.h"
 #include "Emu/Cell/PPUOpcodes.h"
 #include "Emu/Cell/PPUDisAsm.h"
+#include "Emu/Cell/PPUAnalyser.h"
 #include "Emu/Cell/SPUThread.h"
 #include "Emu/Cell/RawSPUThread.h"
 #include "Emu/Cell/lv2/sys_sync.h"
+#include "Emu/Cell/lv2/sys_prx.h"
 
 #include "Emu/IdManager.h"
 #include "Emu/RSX/GSRender.h"
@@ -26,6 +28,7 @@
 
 #include <thread>
 #include <typeinfo>
+#include <queue>
 
 #include "Utilities/GDBDebugServer.h"
 
@@ -39,7 +42,9 @@ extern u64 get_system_time();
 
 extern void ppu_load_exec(const ppu_exec_object&);
 extern void spu_load_exec(const spu_exec_object&);
-extern std::shared_ptr<struct lv2_prx> ppu_load_prx(const ppu_prx_object&, const std::string&);
+extern void ppu_initialize(const ppu_module&);
+extern void ppu_unload_prx(const lv2_prx&);
+extern std::shared_ptr<lv2_prx> ppu_load_prx(const ppu_prx_object&, const std::string&);
 
 extern void network_thread_init();
 
@@ -238,11 +243,10 @@ bool Emulator::BootGame(const std::string& path, bool direct, bool add_only)
 		"/eboot.bin",
 	};
 
-	if (direct && fs::is_file(path))
+	if (direct && fs::exists(path))
 	{
 		m_path = path;
 		Load(add_only);
-
 		return true;
 	}
 
@@ -254,7 +258,6 @@ bool Emulator::BootGame(const std::string& path, bool direct, bool add_only)
 		{
 			m_path = elf;
 			Load(add_only);
-
 			return true;
 		}
 	}
@@ -359,6 +362,17 @@ void Emulator::Load(bool add_only)
 				return sfov;
 			}
 
+			if (fs::is_dir(m_path))
+			{
+				// Special case (directory scan)
+				if (fs::file sfo{m_path + "/PS3_GAME/PARAM.SFO"})
+				{
+					return sfo;
+				}
+
+				return fs::file{m_path + "/PARAM.SFO"};
+			}
+
 			if (disc.size())
 			{
 				// Check previously used category before it's overwritten
@@ -436,6 +450,118 @@ void Emulator::Load(bool add_only)
 		vfs::mount("dev_usb000", fmt::replace_all(g_cfg.vfs.dev_usb000, "$(EmulatorDir)", emu_dir));
 		vfs::mount("app_home", home_dir.empty() ? elf_dir + '/' : fmt::replace_all(home_dir, "$(EmulatorDir)", emu_dir));
 
+		// Special boot mode (directory scan)
+		if (fs::is_dir(m_path))
+		{
+			m_state = system_state::ready;
+			GetCallbacks().on_ready();
+			vm::init();
+			Run();
+			m_force_boot = false;
+
+			// Force LLVM recompiler
+			g_cfg.core.ppu_decoder.from_default();
+
+			return thread_ctrl::spawn("SPRX Loader", [this]
+			{
+				std::vector<std::string> dir_queue;
+				dir_queue.emplace_back(m_path + '/');
+
+				std::queue<std::shared_ptr<thread_ctrl>> thread_queue;
+				const uint max_threads = std::thread::hardware_concurrency();
+
+				// Find all .sprx files recursively (TODO: process .mself files)
+				for (std::size_t i = 0; i < dir_queue.size(); i++)
+				{
+					if (Emu.IsStopped())
+					{
+						break;
+					}
+
+					LOG_NOTICE(LOADER, "Scanning directory: %s", dir_queue[i]);
+
+					for (auto&& entry : fs::dir(dir_queue[i]))
+					{
+						if (Emu.IsStopped())
+						{
+							break;
+						}
+
+						if (entry.is_directory)
+						{
+							if (entry.name != "." && entry.name != "..")
+							{
+								dir_queue.emplace_back(dir_queue[i] + entry.name + '/');
+							}
+
+							continue;
+						}
+
+						// Check .sprx filename
+						if (entry.name.size() >= 5 && fmt::to_upper(entry.name).compare(entry.name.size() - 5, 5, ".SPRX", 5) == 0)
+						{
+							if (entry.name == "libfs_155.sprx")
+							{
+								continue;
+							}
+
+							// Get full path
+							const std::string path = dir_queue[i] + entry.name;
+
+							LOG_NOTICE(LOADER, "Trying to load SPRX: %s", path);
+
+							// Some files may fail to decrypt due to the lack of klic
+							const ppu_prx_object obj = decrypt_self(fs::file(path));
+
+							if (obj == elf_error::ok)
+							{
+								if (auto prx = ppu_load_prx(obj, path))
+								{
+									while (g_thread_count >= max_threads + 2)
+									{
+										std::this_thread::sleep_for(10ms);
+									}
+
+									thread_queue.emplace();
+
+									thread_ctrl::spawn(thread_queue.back(), "Worker " + std::to_string(thread_queue.size()), [_prx = std::move(prx)]
+									{
+										ppu_initialize(*_prx);
+										ppu_unload_prx(*_prx);
+									});
+								}
+							}
+							else
+							{
+								LOG_ERROR(LOADER, "Failed to load SPRX '%s' (%s)", path, obj.get_error());
+							}
+						}
+					}
+				}
+
+				// Join every thread and print exceptions
+				while (!thread_queue.empty())
+				{
+					try
+					{
+						thread_queue.front()->join();
+					}
+					catch (const std::exception& e)
+					{
+						LOG_FATAL(LOADER, "[%s] %s thrown: %s", thread_queue.front()->get_name(), typeid(e).name(), e.what());
+					}
+
+					thread_queue.pop();
+				}
+
+				// Exit "process"
+				Emu.CallAfter([]
+				{
+					Emu.Stop();
+				});
+			});
+		}
+
 		// Detect boot location
 		const std::string hdd0_game = vfs::get("/dev_hdd0/game/");
 		const std::string hdd0_disc = vfs::get("/dev_hdd0/disc/");

rpcs3/rpcs3qt/game_list_frame.cpp

@@ -549,6 +549,7 @@ void game_list_frame::ShowSpecifiedContextMenu(const QPoint &pos, int row)
 	f.setBold(true);
 	boot->setFont(f);
 	QAction* configure = myMenu.addAction(tr("&Configure"));
+	QAction* createLLVMCache = myMenu.addAction(tr("&Create LLVM Cache"));
 	myMenu.addSeparator();
 	QAction* hide_serial = myMenu.addAction(tr("&Hide From Game List"));
 	hide_serial->setCheckable(true);
@@ -591,6 +592,13 @@ void game_list_frame::ShowSpecifiedContextMenu(const QPoint &pos, int row)
 		xgui_settings->SetValue(gui::gl_hidden_list, QStringList(m_hidden_list.toList()));
 		Refresh();
 	});
+	connect(createLLVMCache, &QAction::triggered, [=]
+	{
+		Emu.SetForceBoot(true);
+		Emu.Stop();
+		Emu.SetForceBoot(true);
+		Emu.BootGame(currGame.path, true);
+	});
 	connect(removeGame, &QAction::triggered, [=]
 	{
 		QMessageBox* mb = new QMessageBox(QMessageBox::Question, tr("Confirm %1 Removal").arg(qstr(currGame.category)), tr("Permanently remove %1 from drive?").arg(qstr(currGame.name)), QMessageBox::Yes | QMessageBox::No, this);

rpcs3/rpcs3qt/main_window.cpp

@@ -526,6 +526,9 @@ void main_window::InstallPup(const QString& dropPath)
 	{
 		LOG_SUCCESS(GENERAL, "Successfully installed PS3 firmware version %s.", version_string);
 		guiSettings->ShowInfoBox(gui::ib_pup_success, tr("Success!"), tr("Successfully installed PS3 firmware and LLE Modules!"), this);
+
+		Emu.SetForceBoot(true);
+		Emu.BootGame(Emu.GetLibDir(), true);
 	}
 }