From f9a62667cf79fb10c50c233b764ffec0a8327fe7 Mon Sep 17 00:00:00 2001 From: Eladash Date: Wed, 27 Apr 2022 19:46:09 +0300 Subject: [PATCH] SPU/PPU Loader: Implement linker/PS3 compiler executable files loading --- Utilities/bit_set.h | 4 +- rpcs3/Emu/Cell/PPUModule.cpp | 116 ++++++++++++++++++++++++++++---- rpcs3/Emu/Cell/RawSPUThread.cpp | 38 +++++++++++ rpcs3/Emu/Cell/lv2/sys_spu.cpp | 2 +- rpcs3/Emu/System.cpp | 26 ++++++- rpcs3/Loader/ELF.h | 115 ++++++++++++++++++++++++++++--- 6 files changed, 274 insertions(+), 27 deletions(-) diff --git a/Utilities/bit_set.h b/Utilities/bit_set.h index 494700d0b3..9f21fbecd7 100644 --- a/Utilities/bit_set.h +++ b/Utilities/bit_set.h @@ -164,12 +164,12 @@ public: return r; } - constexpr bool all_of(bs_t arg) + constexpr bool all_of(bs_t arg) const { return (m_data & arg.m_data) == arg.m_data; } - constexpr bool none_of(bs_t arg) + constexpr bool none_of(bs_t arg) const { return (m_data & arg.m_data) == 0; } diff --git a/rpcs3/Emu/Cell/PPUModule.cpp b/rpcs3/Emu/Cell/PPUModule.cpp index d22cb01b8e..dc12f3023e 100644 --- a/rpcs3/Emu/Cell/PPUModule.cpp +++ b/rpcs3/Emu/Cell/PPUModule.cpp @@ -904,7 +904,7 @@ void try_spawn_ppu_if_exclusive_program(const ppu_module& m) auto ppu = idm::make_ptr>(p, "test_thread", 0); - ppu->cia = m.funcs[0].addr; + ppu->cia = m.funcs.empty() ? m.secs[0].addr : m.funcs[0].addr; // For kernel explorer g_fxo->init(4096); @@ -995,9 +995,9 @@ std::shared_ptr ppu_load_prx(const ppu_prx_object& elf, const std::stri for (const auto& s : elf.shdrs) { - ppu_loader.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", s.sh_type, s.sh_addr, s.sh_size, s.sh_flags); + ppu_loader.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", std::bit_cast(s.sh_type), s.sh_addr, s.sh_size, s._sh_flags); - if (s.sh_type != 1u) continue; + if (s.sh_type != sec_type::sht_progbits) continue; const u32 addr = vm::cast(s.sh_addr); const u32 size = vm::cast(s.sh_size); @@ -1013,8 +1013,8 @@ std::shared_ptr ppu_load_prx(const ppu_prx_object& elf, const std::stri ppu_segment _sec; _sec.addr = addr - saddr + prx->segs[i].addr; _sec.size = size; - _sec.type = s.sh_type; - _sec.flags = static_cast(s.sh_flags & 7); + _sec.type = std::bit_cast(s.sh_type); + _sec.flags = static_cast(s._sh_flags & 7); _sec.filesz = 0; prx->secs.emplace_back(_sec); @@ -1421,16 +1421,16 @@ bool ppu_load_exec(const ppu_exec_object& elf) // Load section list, used by the analyser for (const auto& s : elf.shdrs) { - ppu_loader.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", s.sh_type, s.sh_addr, s.sh_size, s.sh_flags); + ppu_loader.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", std::bit_cast(s.sh_type), s.sh_addr, s.sh_size, s._sh_flags); - if (s.sh_type != 1u) continue; + if (s.sh_type != sec_type::sht_progbits) continue; ppu_segment _sec; const u32 addr = _sec.addr = vm::cast(s.sh_addr); const u32 size = _sec.size = vm::cast(s.sh_size); - _sec.type = s.sh_type; - _sec.flags = static_cast(s.sh_flags & 7); + _sec.type = std::bit_cast(s.sh_type); + _sec.flags = static_cast(s._sh_flags & 7); _sec.filesz = 0; if (addr && size) @@ -1997,16 +1997,16 @@ std::pair, CellError> ppu_load_overlay(const ppu_ex // Load section list, used by the analyser for (const auto& s : elf.shdrs) { - ppu_loader.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", s.sh_type, s.sh_addr, s.sh_size, s.sh_flags); + ppu_loader.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", std::bit_cast(s.sh_type), s.sh_addr, s.sh_size, s._sh_flags); - if (s.sh_type != 1u) continue; + if (s.sh_type != sec_type::sht_progbits) continue; ppu_segment _sec; const u32 addr = _sec.addr = vm::cast(s.sh_addr); const u32 size = _sec.size = vm::cast(s.sh_size); - _sec.type = s.sh_type; - _sec.flags = static_cast(s.sh_flags & 7); + _sec.type = std::bit_cast(s.sh_type); + _sec.flags = static_cast(s._sh_flags & 7); _sec.filesz = 0; if (addr && size) @@ -2152,3 +2152,93 @@ std::pair, CellError> ppu_load_overlay(const ppu_ex return {std::move(ovlm), {}}; } + +bool ppu_load_rel_exec(const ppu_rel_object& elf) +{ + ppu_module relm{}; + + struct on_fatal_error + { + ppu_module& relm; + bool errored = true; + + ~on_fatal_error() + { + if (!errored) + { + return; + } + + // Revert previous allocations on an error + for (const auto& seg : relm.secs) + { + vm::dealloc(seg.addr); + } + } + + } error_handler{relm}; + + u32 memsize = 0; + + for (const auto& s : elf.shdrs) + { + if (s.sh_type != sec_type::sht_progbits) + { + memsize = utils::align(memsize + vm::cast(s.sh_size), 128); + } + } + + u32 addr = vm::alloc(memsize, vm::main); + + if (!addr) + { + ppu_loader.fatal("ppu_load_rel_exec(): vm::alloc() failed (memsz=0x%x)", memsize); + return false; + } + + ppu_register_range(addr, memsize); + + // Copy references to sections for the purpose of sorting executable sections before non-executable ones + std::vector*> shdrs(elf.shdrs.size()); + + for (auto& ref : shdrs) + { + ref = &elf.shdrs[&ref - shdrs.data()]; + } + + std::stable_sort(shdrs.begin(), shdrs.end(), [](auto& a, auto& b) -> bool + { + const bs_t flags_a_has = a->sh_flags() - b->sh_flags(); + return flags_a_has.all_of(sh_flag::shf_execinstr); + }); + + // Load sections + for (auto ptr : shdrs) + { + const auto& s = *ptr; + + ppu_loader.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", std::bit_cast(s.sh_type), s.sh_addr, s.sh_size, s._sh_flags); + + if (s.sh_type == sec_type::sht_progbits && s.sh_size && s.sh_flags().all_of(sh_flag::shf_alloc)) + { + ppu_segment _sec; + const u32 size = _sec.size = vm::cast(s.sh_size); + + _sec.type = std::bit_cast(s.sh_type); + _sec.flags = static_cast(s._sh_flags & 7); + _sec.filesz = size; + + _sec.addr = addr; + relm.secs.emplace_back(_sec); + + std::memcpy(vm::base(addr), s.bin.data(), size); + addr = utils::align(addr + size, 128); + } + } + + try_spawn_ppu_if_exclusive_program(relm); + + error_handler.errored = false; + + return true; +} diff --git a/rpcs3/Emu/Cell/RawSPUThread.cpp b/rpcs3/Emu/Cell/RawSPUThread.cpp index 81d943179e..5dc749041b 100644 --- a/rpcs3/Emu/Cell/RawSPUThread.cpp +++ b/rpcs3/Emu/Cell/RawSPUThread.cpp @@ -1,6 +1,7 @@ #include "stdafx.h" #include "Emu/IdManager.h" #include "Loader/ELF.h" +#include "util/asm.hpp" #include "Emu/Cell/RawSPUThread.h" @@ -336,3 +337,40 @@ void spu_load_exec(const spu_exec_object& elf) spu->status_npc = {SPU_STATUS_RUNNING, elf.header.e_entry}; atomic_storage::release(spu->pc, elf.header.e_entry); } + +void spu_load_rel_exec(const spu_rel_object& elf) +{ + spu_thread::g_raw_spu_ctr++; + + auto spu = idm::make_ptr>(nullptr, 0, "test_spu", 0); + + u64 total_memsize = 0; + + // Compute executable data size + for (const auto& shdr : elf.shdrs) + { + if (shdr.sh_type == sec_type::sht_progbits && shdr.sh_flags().all_of(sh_flag::shf_alloc)) + { + total_memsize = utils::align(total_memsize + shdr.sh_size, 4); + } + } + + // Place executable data in SPU local memory + u32 offs = 0; + + for (const auto& shdr : elf.shdrs) + { + if (shdr.sh_type == sec_type::sht_progbits && shdr.sh_flags().all_of(sh_flag::shf_alloc)) + { + std::memcpy(spu->_ptr(offs), shdr.bin.data(), shdr.sh_size); + offs = utils::align(offs + shdr.sh_size, 4); + } + } + + spu_log.success("Loaded 0x%x of SPU relocatable executable data", total_memsize); + + spu_thread::g_raw_spu_id[0] = spu->id; + + spu->status_npc = {SPU_STATUS_RUNNING, elf.header.e_entry}; + atomic_storage::release(spu->pc, elf.header.e_entry); +} diff --git a/rpcs3/Emu/Cell/lv2/sys_spu.cpp b/rpcs3/Emu/Cell/lv2/sys_spu.cpp index 5aa65ef7e4..f4991bcab5 100644 --- a/rpcs3/Emu/Cell/lv2/sys_spu.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_spu.cpp @@ -78,7 +78,7 @@ void sys_spu_image::load(const fs::file& stream) for (const auto& shdr : obj.shdrs) { - spu_log.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", shdr.sh_type, shdr.sh_addr, shdr.sh_size, shdr.sh_flags); + spu_log.notice("** Section: sh_type=0x%x, addr=0x%llx, size=0x%llx, flags=0x%x", std::bit_cast(shdr.sh_type), shdr.sh_addr, shdr.sh_size, shdr._sh_flags); } for (const auto& prog : obj.progs) diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index f4185cc9cd..7abf74bef7 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -64,12 +64,14 @@ atomic_t g_watchdog_hold_ctr{0}; extern bool ppu_load_exec(const ppu_exec_object&); extern void spu_load_exec(const spu_exec_object&); +extern void spu_load_rel_exec(const spu_rel_object&); extern void ppu_precompile(std::vector& dir_queue, std::vector* loaded_prx); extern bool ppu_initialize(const ppu_module&, bool = false); extern void ppu_finalize(const ppu_module&); extern void ppu_unload_prx(const lv2_prx&); extern std::shared_ptr ppu_load_prx(const ppu_prx_object&, const std::string&, s64 = 0); extern std::pair, CellError> ppu_load_overlay(const ppu_exec_object&, const std::string& path, s64 = 0); +extern bool ppu_load_rel_exec(const ppu_rel_object&); fs::file g_tty; atomic_t g_tty_size{0}; @@ -1340,7 +1342,9 @@ game_boot_result Emulator::Load(const std::string& title_id, bool add_only, bool ppu_exec_object ppu_exec; ppu_prx_object ppu_prx; + ppu_rel_object ppu_rel; spu_exec_object spu_exec; + spu_rel_object spu_rel; if (ppu_exec.open(elf_file) == elf_error::ok) { @@ -1449,7 +1453,7 @@ game_boot_result Emulator::Load(const std::string& title_id, bool add_only, bool } else if (ppu_prx.open(elf_file) == elf_error::ok) { - // PPU PRX (experimental) + // PPU PRX GetCallbacks().on_ready(); g_fxo->init(false); ppu_load_prx(ppu_prx, m_path); @@ -1457,12 +1461,28 @@ game_boot_result Emulator::Load(const std::string& title_id, bool add_only, bool } else if (spu_exec.open(elf_file) == elf_error::ok) { - // SPU executable (experimental) + // SPU executable GetCallbacks().on_ready(); g_fxo->init(false); spu_load_exec(spu_exec); Pause(true); } + else if (spu_rel.open(elf_file) == elf_error::ok) + { + // SPU linker file + GetCallbacks().on_ready(); + g_fxo->init(false); + spu_load_rel_exec(spu_rel); + Pause(true); + } + else if (ppu_rel.open(elf_file) == elf_error::ok) + { + // PPU linker file + GetCallbacks().on_ready(); + g_fxo->init(false); + ppu_load_rel_exec(ppu_rel); + Pause(true); + } else { sys_log.error("Invalid or unsupported file format: %s", elf_path); @@ -1470,6 +1490,8 @@ game_boot_result Emulator::Load(const std::string& title_id, bool add_only, bool sys_log.warning("** ppu_exec -> %s", ppu_exec.get_error()); sys_log.warning("** ppu_prx -> %s", ppu_prx.get_error()); sys_log.warning("** spu_exec -> %s", spu_exec.get_error()); + sys_log.warning("** spu_rel -> %s", spu_rel.get_error()); + sys_log.warning("** ppu_rel -> %s", ppu_rel.get_error()); Kill(false); return game_boot_result::invalid_file_or_folder; diff --git a/rpcs3/Loader/ELF.h b/rpcs3/Loader/ELF.h index 064dd05c73..5239ff1215 100644 --- a/rpcs3/Loader/ELF.h +++ b/rpcs3/Loader/ELF.h @@ -33,6 +33,50 @@ enum class elf_machine : u16 mips = 0x08, }; +enum class sec_type : u32 +{ + sht_null = 0, + sht_progbits = 1, + sht_symtab = 2, + sht_strtab = 3, + sht_rela = 4, + sht_hash = 5, + sht_dynamic = 6, + sht_note = 7, + sht_nobits = 8, + sht_rel = 9, +}; + +enum class sh_flag : u32 +{ + shf_write = 1, + shf_alloc = 2, + shf_execinstr = 4, + + __bitset_enum_max +}; + +constexpr bool is_memorizable_section(sec_type type) +{ + switch (type) + { + case sec_type::sht_null: + case sec_type::sht_nobits: + { + return false; + } + default: + { + if (type > sec_type::sht_rel) + { + return false; + } + + return true; + } + } +} + template using elf_be = be_t; @@ -123,8 +167,8 @@ template class en_t, typename sz_t> struct elf_shdr { en_t sh_name; - en_t sh_type; - en_t sh_flags; + en_t sh_type; + en_t _sh_flags; en_t sh_addr; en_t sh_offset; en_t sh_size; @@ -132,6 +176,21 @@ struct elf_shdr en_t sh_info; en_t sh_addralign; en_t sh_entsize; + + bs_t sh_flags() const + { + return std::bit_cast>(static_cast(+_sh_flags)); + } +}; + +template class en_t, typename sz_t> +struct elf_shdata final : elf_shdr +{ + std::vector bin{}; + + using base = elf_shdr; + + elf_shdata() = default; }; // ELF loading options @@ -177,11 +236,12 @@ public: using phdr_t = elf_phdr; using shdr_t = elf_shdr; using prog_t = elf_prog; + using shdata_t = elf_shdata; ehdr_t header{}; std::vector progs{}; - std::vector shdrs{}; + std::vector shdrs{}; public: elf_object() = default; @@ -238,6 +298,7 @@ public: // Load program headers std::vector _phdrs; + std::vector _shdrs; if (!(opts & elf_opt::no_programs)) { @@ -249,7 +310,7 @@ public: if (!(opts & elf_opt::no_sections)) { stream.seek(offset + header.e_shoff); - if (!stream.read(shdrs, header.e_shnum)) + if (!stream.read(_shdrs, header.e_shnum)) return set_error(elf_error::stream_shdrs); } @@ -257,9 +318,7 @@ public: progs.reserve(_phdrs.size()); for (const auto& hdr : _phdrs) { - progs.emplace_back(); - - static_cast(progs.back()) = hdr; + static_cast(progs.emplace_back()) = hdr; if (!(opts & elf_opt::no_data)) { @@ -269,6 +328,20 @@ public: } } + shdrs.clear(); + shdrs.reserve(_shdrs.size()); + for (const auto& shdr : _shdrs) + { + static_cast(shdrs.emplace_back()) = shdr; + + if (!(opts & elf_opt::no_data) && is_memorizable_section(shdr.sh_type)) + { + stream.seek(offset + shdr.sh_offset); + if (!stream.read(shdrs.back().bin, shdr.sh_size)) + return set_error(elf_error::stream_data); + } + } + shdrs.shrink_to_fit(); progs.shrink_to_fit(); @@ -279,6 +352,8 @@ public: { fs::file stream = fs::make_stream>(std::move(init)); + const bool fixup_shdrs = shdrs.empty() || shdrs[0].sh_type != sec_type::sht_null; + // Write header ehdr_t header{}; header.e_magic = "\177ELF"_u32; @@ -298,7 +373,7 @@ public: header.e_phentsize = u32{sizeof(phdr_t)}; header.e_phnum = ::size32(progs); header.e_shentsize = u32{sizeof(shdr_t)}; - header.e_shnum = ::size32(shdrs); + header.e_shnum = ::size32(shdrs) + u32{fixup_shdrs}; header.e_shstrndx = this->header.e_shstrndx; stream.write(header); @@ -310,9 +385,19 @@ public: stream.write(phdr); } + if (fixup_shdrs) + { + // Insert a must-have empty section at the start + stream.write(shdr_t{}); + } + for (shdr_t shdr : shdrs) { - // TODO? + if (is_memorizable_section(shdr.sh_type)) + { + shdr.sh_offset = std::exchange(off, off + shdr.sh_size); + } + stream.write(shdr); } @@ -322,6 +407,16 @@ public: stream.write(prog.bin); } + for (const auto& shdr : shdrs) + { + if (!is_memorizable_section(shdr.sh_type)) + { + continue; + } + + stream.write(shdr.bin); + } + return std::move(static_cast>*>(stream.release().get())->obj); } @@ -364,5 +459,7 @@ public: using ppu_exec_object = elf_object; using ppu_prx_object = elf_object; +using ppu_rel_object = elf_object; using spu_exec_object = elf_object; +using spu_rel_object = elf_object; using arm_exec_object = elf_object;