diff --git a/Utilities/JIT.cpp b/Utilities/JITASM.cpp similarity index 100% rename from Utilities/JIT.cpp rename to Utilities/JITASM.cpp diff --git a/Utilities/JITLLVM.cpp b/Utilities/JITLLVM.cpp new file mode 100644 index 0000000000..94d1ca5fe9 --- /dev/null +++ b/Utilities/JITLLVM.cpp @@ -0,0 +1,1492 @@ +#include "util/types.hpp" +#include "util/sysinfo.hpp" +#include "JIT.h" +#include "StrFmt.h" +#include "File.h" +#include "util/logs.hpp" +#include "mutex.h" +#include "util/vm.hpp" +#include "util/asm.hpp" +#include "util/v128.hpp" +#include "util/simd.hpp" +#include "Crypto/unzip.h" + +#include + +#ifdef __linux__ +#define CAN_OVERCOMMIT +#endif + +LOG_CHANNEL(jit_log, "JIT"); + +void jit_announce(uptr func, usz size, std::string_view name) +{ +#ifdef __linux__ + static const struct tmp_perf_map + { + std::string name{fmt::format("/tmp/perf-%d.map", getpid())}; + fs::file data{name, fs::rewrite + fs::append}; + + tmp_perf_map() = default; + tmp_perf_map(const tmp_perf_map&) = delete; + tmp_perf_map& operator=(const tmp_perf_map&) = delete; + + ~tmp_perf_map() + { + fs::remove_file(name); + } + } s_map; + + if (size && name.size()) + { + s_map.data.write(fmt::format("%x %x %s\n", func, size, name)); + } + + if (!func && !size && !name.size()) + { + fs::remove_file(s_map.name); + return; + } +#endif + + if (!size) + { + jit_log.error("Empty function announced: %s (%p)", name, func); + return; + } + + // If directory ASMJIT doesn't exist, nothing will be written + static constexpr u64 c_dump_size = 0x1'0000'0000; + static constexpr u64 c_index_size = c_dump_size / 16; + static atomic_t g_index_off = 0; + static atomic_t g_data_off = c_index_size; + + static void* g_asm = []() -> void* + { + fs::remove_all(fs::get_cache_dir() + "/ASMJIT/", false); + + fs::file objs(fmt::format("%s/ASMJIT/.objects", fs::get_cache_dir()), fs::read + fs::rewrite); + + if (!objs || !objs.trunc(c_dump_size)) + { + return nullptr; + } + + return utils::memory_map_fd(objs.get_handle(), c_dump_size, utils::protection::rw); + }(); + + if (g_asm && size < c_index_size) + { + struct entry + { + u64 addr; // RPCS3 process address + u32 size; // Function size + u32 off; // Function offset + }; + + // Write index entry at the beginning of file, and data + NTS name at fixed offset + const u64 index_off = g_index_off.fetch_add(1); + const u64 size_all = size + name.size() + 1; + const u64 data_off = g_data_off.fetch_add(size_all); + + // If either index or data area is exhausted, nothing will be written + if (index_off < c_index_size / sizeof(entry) && data_off + size_all < c_dump_size) + { + entry& index = static_cast(g_asm)[index_off]; + + std::memcpy(static_cast(g_asm) + data_off, reinterpret_cast(func), size); + std::memcpy(static_cast(g_asm) + data_off + size, name.data(), name.size()); + index.size = static_cast(size); + index.off = static_cast(data_off); + atomic_storage::store(index.addr, func); + } + } + + if (g_asm && !name.empty() && name[0] != '_') + { + // Save some objects separately + fs::file dump(fmt::format("%s/ASMJIT/%s", fs::get_cache_dir(), name), fs::rewrite); + + if (dump) + { + dump.write(reinterpret_cast(func), size); + } + } +} + +static u8* get_jit_memory() +{ + // Reserve 2G memory (magic static) + static void* const s_memory2 = []() -> void* + { + void* ptr = utils::memory_reserve(0x80000000); +#ifdef CAN_OVERCOMMIT + utils::memory_commit(ptr, 0x80000000); + utils::memory_protect(ptr, 0x40000000, utils::protection::wx); +#endif + return ptr; + }(); + + return static_cast(s_memory2); +} + +// Allocation counters (1G code, 1G data subranges) +static atomic_t s_code_pos{0}, s_data_pos{0}; + +// Snapshot of code generated before main() +static std::vector s_code_init, s_data_init; + +template & Ctr, uint Off, utils::protection Prot> +static u8* add_jit_memory(usz size, usz align) +{ + // Select subrange + u8* pointer = get_jit_memory() + Off; + + if (!size && !align) [[unlikely]] + { + // Return subrange info + return pointer; + } + + u64 olda, newa; + + // Simple allocation by incrementing pointer to the next free data + const u64 pos = Ctr.atomic_op([&](u64& ctr) -> u64 + { + const u64 _pos = utils::align(ctr & 0xffff'ffff, align); + const u64 _new = utils::align(_pos + size, align); + + if (_new > 0x40000000) [[unlikely]] + { + // Sorry, we failed, and further attempts should fail too. + ctr |= 0x40000000; + return -1; + } + + // Last allocation is stored in highest bits + olda = ctr >> 32; + newa = olda; + + // Check the necessity to commit more memory + if (_new > olda) [[unlikely]] + { + newa = utils::align(_new, 0x200000); + } + + ctr += _new - (ctr & 0xffff'ffff); + return _pos; + }); + + if (pos == umax) [[unlikely]] + { + jit_log.error("Out of memory (size=0x%x, align=0x%x, off=0x%x)", size, align, Off); + return nullptr; + } + + if (olda != newa) [[unlikely]] + { +#ifndef CAN_OVERCOMMIT + // Commit more memory + utils::memory_commit(pointer + olda, newa - olda, Prot); +#endif + // Acknowledge committed memory + Ctr.atomic_op([&](u64& ctr) + { + if ((ctr >> 32) < newa) + { + ctr += (newa - (ctr >> 32)) << 32; + } + }); + } + + ensure(pointer + pos >= get_jit_memory() + Off); + ensure(pointer + pos < get_jit_memory() + Off + 0x40000000); + + return pointer + pos; +} + +const asmjit::Environment& jit_runtime_base::environment() const noexcept +{ + static const asmjit::Environment g_env = asmjit::Environment::host(); + + return g_env; +} + +void* jit_runtime_base::_add(asmjit::CodeHolder* code) noexcept +{ + ensure(!code->flatten()); + ensure(!code->resolveUnresolvedLinks()); + usz codeSize = code->codeSize(); + if (!codeSize) + return nullptr; + + auto p = ensure(this->_alloc(codeSize, 64)); + ensure(!code->relocateToBase(uptr(p))); + + { + // We manage rw <-> rx transitions manually on Apple + // because it's easier to keep track of when and where we need to toggle W^X +#if !(defined(ARCH_ARM64) && defined(__APPLE__)) + asmjit::VirtMem::ProtectJitReadWriteScope rwScope(p, codeSize); +#endif + + for (asmjit::Section* section : code->_sections) + { + std::memcpy(p + section->offset(), section->data(), section->bufferSize()); + } + } + + return p; +} + +jit_runtime::jit_runtime() +{ +} + +jit_runtime::~jit_runtime() +{ +} + +uchar* jit_runtime::_alloc(usz size, usz align) noexcept +{ + return jit_runtime::alloc(size, align, true); +} + +u8* jit_runtime::alloc(usz size, usz align, bool exec) noexcept +{ + if (exec) + { + return add_jit_memory(size, align); + } + else + { + return add_jit_memory(size, align); + } +} + +void jit_runtime::initialize() +{ + if (!s_code_init.empty() || !s_data_init.empty()) + { + return; + } + + // Create code/data snapshot + s_code_init.resize(s_code_pos & 0xffff'ffff); + std::memcpy(s_code_init.data(), alloc(0, 0, true), s_code_init.size()); + s_data_init.resize(s_data_pos & 0xffff'ffff); + std::memcpy(s_data_init.data(), alloc(0, 0, false), s_data_init.size()); +} + +void jit_runtime::finalize() noexcept +{ +#ifdef __APPLE__ + pthread_jit_write_protect_np(false); +#endif + // Reset JIT memory +#ifdef CAN_OVERCOMMIT + utils::memory_reset(get_jit_memory(), 0x80000000); + utils::memory_protect(get_jit_memory(), 0x40000000, utils::protection::wx); +#else + utils::memory_decommit(get_jit_memory(), 0x80000000); +#endif + + s_code_pos = 0; + s_data_pos = 0; + + // Restore code/data snapshot + std::memcpy(alloc(s_code_init.size(), 1, true), s_code_init.data(), s_code_init.size()); + std::memcpy(alloc(s_data_init.size(), 1, false), s_data_init.data(), s_data_init.size()); + +#ifdef __APPLE__ + pthread_jit_write_protect_np(true); +#endif +#ifdef ARCH_ARM64 + // Flush all cache lines after potentially writing executable code + asm("ISB"); + asm("DSB ISH"); +#endif +} + +jit_runtime_base& asmjit::get_global_runtime() +{ + // 16 MiB for internal needs + static constexpr u64 size = 1024 * 1024 * 16; + + struct custom_runtime final : jit_runtime_base + { + custom_runtime() noexcept + { + // Search starting in first 2 GiB of memory + for (u64 addr = size;; addr += size) + { + if (auto ptr = utils::memory_reserve(size, reinterpret_cast(addr))) + { + m_pos.raw() = static_cast(ptr); + break; + } + } + + // Initialize "end" pointer + m_max = m_pos + size; + + // Make memory writable + executable + utils::memory_commit(m_pos, size, utils::protection::wx); + } + + uchar* _alloc(usz size, usz align) noexcept override + { + return m_pos.atomic_op([&](uchar*& pos) -> uchar* + { + const auto r = reinterpret_cast(utils::align(uptr(pos), align)); + + if (r >= pos && r + size > pos && r + size <= m_max) + { + pos = r + size; + return r; + } + + return nullptr; + }); + } + + private: + atomic_t m_pos{}; + + uchar* m_max{}; + }; + + // Magic static + static custom_runtime g_rt; + return g_rt; +} + +asmjit::inline_runtime::inline_runtime(uchar* data, usz size) + : m_data(data) + , m_size(size) +{ +} + +uchar* asmjit::inline_runtime::_alloc(usz size, usz align) noexcept +{ + ensure(align <= 4096); + + return size <= m_size ? m_data : nullptr; +} + +asmjit::inline_runtime::~inline_runtime() +{ + utils::memory_protect(m_data, m_size, utils::protection::rx); +} + +#if defined(ARCH_X64) +asmjit::simd_builder::simd_builder(CodeHolder* ch) noexcept + : native_asm(ch) +{ + _init(0); + consts[~v128()] = this->newLabel(); +} + +asmjit::simd_builder::~simd_builder() +{ +} + +void asmjit::simd_builder::_init(uint new_vsize) +{ + if ((!new_vsize && utils::has_avx512_icl()) || new_vsize == 64) + { + v0 = x86::zmm0; + v1 = x86::zmm1; + v2 = x86::zmm2; + v3 = x86::zmm3; + v4 = x86::zmm4; + v5 = x86::zmm5; + vsize = 64; + } + else if ((!new_vsize && utils::has_avx2()) || new_vsize == 32) + { + v0 = x86::ymm0; + v1 = x86::ymm1; + v2 = x86::ymm2; + v3 = x86::ymm3; + v4 = x86::ymm4; + v5 = x86::ymm5; + vsize = 32; + } + else + { + v0 = x86::xmm0; + v1 = x86::xmm1; + v2 = x86::xmm2; + v3 = x86::xmm3; + v4 = x86::xmm4; + v5 = x86::xmm5; + vsize = new_vsize ? new_vsize : 16; + } + + if (utils::has_avx512()) + { + if (!new_vsize) + vmask = -1; + } + else + { + vmask = 0; + } +} + +void asmjit::simd_builder::operator()() noexcept +{ + for (auto&& [x, y] : consts) + { + this->align(AlignMode::kData, 16); + this->bind(y); + this->embed(&x, 16); + } +} + +void asmjit::simd_builder::vec_cleanup_ret() +{ + if (utils::has_avx() && vsize > 16) + this->vzeroupper(); + this->ret(); +} + +void asmjit::simd_builder::vec_set_all_zeros(const Operand& v) +{ + x86::Xmm reg(v.id()); + if (utils::has_avx()) + this->vpxor(reg, reg, reg); + else + this->xorps(reg, reg); +} + +void asmjit::simd_builder::vec_set_all_ones(const Operand& v) +{ + x86::Xmm reg(v.id()); + if (x86::Zmm zr(v.id()); zr == v) + this->vpternlogd(zr, zr, zr, 0xff); + else if (x86::Ymm yr(v.id()); yr == v) + this->vpcmpeqd(yr, yr, yr); + else if (utils::has_avx()) + this->vpcmpeqd(reg, reg, reg); + else + this->pcmpeqd(reg, reg); +} + +void asmjit::simd_builder::vec_set_const(const Operand& v, const v128& val) +{ + if (!val._u) + return vec_set_all_zeros(v); + if (!~val._u) + return vec_set_all_ones(v); + else + { + Label co = consts[val]; + if (!co.isValid()) + co = consts[val] = this->newLabel(); + if (x86::Zmm zr(v.id()); zr == v) + this->vbroadcasti32x4(zr, x86::oword_ptr(co)); + else if (x86::Ymm yr(v.id()); yr == v) + this->vbroadcasti128(yr, x86::oword_ptr(co)); + else if (utils::has_avx()) + this->vmovaps(x86::Xmm(v.id()), x86::oword_ptr(co)); + else + this->movaps(x86::Xmm(v.id()), x86::oword_ptr(co)); + } +} + +void asmjit::simd_builder::vec_clobbering_test(u32 esize, const Operand& v, const Operand& rhs) +{ + if (esize == 64) + { + this->emit(x86::Inst::kIdVptestmd, x86::k0, v, rhs); + this->ktestw(x86::k0, x86::k0); + } + else if (esize == 32) + { + this->emit(x86::Inst::kIdVptest, v, rhs); + } + else if (esize == 16 && utils::has_avx()) + { + this->emit(x86::Inst::kIdVptest, v, rhs); + } + else if (esize == 16 && utils::has_sse41()) + { + this->emit(x86::Inst::kIdPtest, v, rhs); + } + else + { + if (v != rhs) + this->emit(x86::Inst::kIdPand, v, rhs); + if (esize == 16) + this->emit(x86::Inst::kIdPacksswb, v, v); + this->emit(x86::Inst::kIdMovq, x86::rax, v); + if (esize == 16 || esize == 8) + this->test(x86::rax, x86::rax); + else if (esize == 4) + this->test(x86::eax, x86::eax); + else if (esize == 2) + this->test(x86::ax, x86::ax); + else if (esize == 1) + this->test(x86::al, x86::al); + else + fmt::throw_exception("Unimplemented"); + } +} + +void asmjit::simd_builder::vec_broadcast_gpr(u32 esize, const Operand& v, const x86::Gp& r) +{ + if (esize == 2) + { + if (utils::has_avx512()) + this->emit(x86::Inst::kIdVpbroadcastw, v, r.r32()); + else if (utils::has_avx()) + { + this->emit(x86::Inst::kIdVmovd, v, r.r32()); + if (utils::has_avx2()) + this->emit(x86::Inst::kIdVpbroadcastw, v, v); + else + { + this->emit(x86::Inst::kIdVpunpcklwd, v, v, v); + this->emit(x86::Inst::kIdVpshufd, v, v, Imm(0)); + } + } + else + { + this->emit(x86::Inst::kIdMovd, v, r.r32()); + this->emit(x86::Inst::kIdPunpcklwd, v, v); + this->emit(x86::Inst::kIdPshufd, v, v, Imm(0)); + } + } + else if (esize == 4) + { + if (utils::has_avx512()) + this->emit(x86::Inst::kIdVpbroadcastd, v, r.r32()); + else if (utils::has_avx()) + { + this->emit(x86::Inst::kIdVmovd, v, r.r32()); + if (utils::has_avx2()) + this->emit(x86::Inst::kIdVpbroadcastd, v, v); + else + this->emit(x86::Inst::kIdVpshufd, v, v, Imm(0)); + } + else + { + this->emit(x86::Inst::kIdMovd, v, r.r32()); + this->emit(x86::Inst::kIdPshufd, v, v, Imm(0)); + } + } + else + { + fmt::throw_exception("Unimplemented"); + } +} + +asmjit::x86::Mem asmjit::simd_builder::ptr_scale_for_vec(u32 esize, const x86::Gp& base, const x86::Gp& index) +{ + switch (ensure(esize)) + { + case 1: return x86::ptr(base, index, 0, 0); + case 2: return x86::ptr(base, index, 1, 0); + case 4: return x86::ptr(base, index, 2, 0); + case 8: return x86::ptr(base, index, 3, 0); + default: fmt::throw_exception("Bad esize"); + } +} + +void asmjit::simd_builder::vec_load_unaligned(u32 esize, const Operand& v, const x86::Mem& src) +{ + ensure(std::has_single_bit(esize)); + ensure(std::has_single_bit(vsize)); + + if (esize == 2) + { + ensure(vsize >= 2); + if (vsize == 2) + vec_set_all_zeros(v); + if (vsize == 2 && utils::has_avx()) + this->emit(x86::Inst::kIdVpinsrw, x86::Xmm(v.id()), x86::Xmm(v.id()), src, Imm(0)); + else if (vsize == 2) + this->emit(x86::Inst::kIdPinsrw, v, src, Imm(0)); + else if ((vmask && vmask < 8) || vsize >= 64) + this->emit(x86::Inst::kIdVmovdqu16, v, src); + else + return vec_load_unaligned(vsize, v, src); + } + else if (esize == 4) + { + ensure(vsize >= 4); + if (vsize == 4 && utils::has_avx()) + this->emit(x86::Inst::kIdVmovd, x86::Xmm(v.id()), src); + else if (vsize == 4) + this->emit(x86::Inst::kIdMovd, v, src); + else if ((vmask && vmask < 8) || vsize >= 64) + this->emit(x86::Inst::kIdVmovdqu32, v, src); + else + return vec_load_unaligned(vsize, v, src); + } + else if (esize == 8) + { + ensure(vsize >= 8); + if (vsize == 8 && utils::has_avx()) + this->emit(x86::Inst::kIdVmovq, x86::Xmm(v.id()), src); + else if (vsize == 8) + this->emit(x86::Inst::kIdMovq, v, src); + else if ((vmask && vmask < 8) || vsize >= 64) + this->emit(x86::Inst::kIdVmovdqu64, v, src); + else + return vec_load_unaligned(vsize, v, src); + } + else if (esize >= 16) + { + ensure(vsize >= 16); + if ((vmask && vmask < 8) || vsize >= 64) + this->emit(x86::Inst::kIdVmovdqu64, v, src); // Not really needed + else if (utils::has_avx()) + this->emit(x86::Inst::kIdVmovdqu, v, src); + else + this->emit(x86::Inst::kIdMovups, v, src); + } + else + { + fmt::throw_exception("Unimplemented"); + } +} + +void asmjit::simd_builder::vec_store_unaligned(u32 esize, const Operand& v, const x86::Mem& dst) +{ + ensure(std::has_single_bit(esize)); + ensure(std::has_single_bit(vsize)); + + if (esize == 2) + { + ensure(vsize >= 2); + if (vsize == 2 && utils::has_avx()) + this->emit(x86::Inst::kIdVpextrw, dst, x86::Xmm(v.id()), Imm(0)); + else if (vsize == 2 && utils::has_sse41()) + this->emit(x86::Inst::kIdPextrw, dst, v, Imm(0)); + else if (vsize == 2) + this->push(x86::rax), this->pextrw(x86::eax, x86::Xmm(v.id()), 0), this->mov(dst, x86::ax), this->pop(x86::rax); + else if ((vmask && vmask < 8) || vsize >= 64) + this->emit(x86::Inst::kIdVmovdqu16, dst, v); + else + return vec_store_unaligned(vsize, v, dst); + } + else if (esize == 4) + { + ensure(vsize >= 4); + if (vsize == 4 && utils::has_avx()) + this->emit(x86::Inst::kIdVmovd, dst, x86::Xmm(v.id())); + else if (vsize == 4) + this->emit(x86::Inst::kIdMovd, dst, v); + else if ((vmask && vmask < 8) || vsize >= 64) + this->emit(x86::Inst::kIdVmovdqu32, dst, v); + else + return vec_store_unaligned(vsize, v, dst); + } + else if (esize == 8) + { + ensure(vsize >= 8); + if (vsize == 8 && utils::has_avx()) + this->emit(x86::Inst::kIdVmovq, dst, x86::Xmm(v.id())); + else if (vsize == 8) + this->emit(x86::Inst::kIdMovq, dst, v); + else if ((vmask && vmask < 8) || vsize >= 64) + this->emit(x86::Inst::kIdVmovdqu64, dst, v); + else + return vec_store_unaligned(vsize, v, dst); + } + else if (esize >= 16) + { + ensure(vsize >= 16); + if ((vmask && vmask < 8) || vsize >= 64) + this->emit(x86::Inst::kIdVmovdqu64, dst, v); // Not really needed + else if (utils::has_avx()) + this->emit(x86::Inst::kIdVmovdqu, dst, v); + else + this->emit(x86::Inst::kIdMovups, dst, v); + } + else + { + fmt::throw_exception("Unimplemented"); + } +} + +void asmjit::simd_builder::_vec_binary_op(x86::Inst::Id sse_op, x86::Inst::Id vex_op, x86::Inst::Id evex_op, const Operand& dst, const Operand& lhs, const Operand& rhs) +{ + if (utils::has_avx()) + { + if (evex_op != x86::Inst::kIdNone && (vex_op == x86::Inst::kIdNone || this->_extraReg.isReg() || vsize >= 64)) + { + this->evex().emit(evex_op, dst, lhs, rhs); + } + else + { + this->emit(vex_op, dst, lhs, rhs); + } + } + else if (dst == lhs) + { + this->emit(sse_op, dst, rhs); + } + else if (dst == rhs) + { + fmt::throw_exception("Unimplemented"); + } + else + { + this->emit(x86::Inst::kIdMovaps, dst, lhs); + this->emit(sse_op, dst, rhs); + } +} + +void asmjit::simd_builder::vec_umin(u32 esize, const Operand& dst, const Operand& lhs, const Operand& rhs) +{ + using enum x86::Inst::Id; + if (esize == 2) + { + if (utils::has_sse41()) + return _vec_binary_op(kIdPminuw, kIdVpminuw, kIdVpminuw, dst, lhs, rhs); + } + else if (esize == 4) + { + if (utils::has_sse41()) + return _vec_binary_op(kIdPminud, kIdVpminud, kIdVpminud, dst, lhs, rhs); + } + + fmt::throw_exception("Unimplemented"); +} + +void asmjit::simd_builder::vec_umax(u32 esize, const Operand& dst, const Operand& lhs, const Operand& rhs) +{ + using enum x86::Inst::Id; + if (esize == 2) + { + if (utils::has_sse41()) + return _vec_binary_op(kIdPmaxuw, kIdVpmaxuw, kIdVpmaxuw, dst, lhs, rhs); + } + else if (esize == 4) + { + if (utils::has_sse41()) + return _vec_binary_op(kIdPmaxud, kIdVpmaxud, kIdVpmaxud, dst, lhs, rhs); + } + + fmt::throw_exception("Unimplemented"); +} + +void asmjit::simd_builder::vec_cmp_eq(u32 esize, const Operand& dst, const Operand& lhs, const Operand& rhs) +{ + using enum x86::Inst::Id; + if (esize == 2) + { + if (vsize == 64) + { + this->evex().emit(kIdVpcmpeqw, x86::k0, lhs, rhs); + this->evex().emit(kIdVpmovm2w, dst, x86::k0); + } + else + { + _vec_binary_op(kIdPcmpeqw, kIdVpcmpeqw, kIdNone, dst, lhs, rhs); + } + } + else if (esize == 4) + { + if (vsize == 64) + { + this->evex().emit(kIdVpcmpeqd, x86::k0, lhs, rhs); + this->evex().emit(kIdVpmovm2d, dst, x86::k0); + } + else + { + _vec_binary_op(kIdPcmpeqd, kIdVpcmpeqd, kIdNone, dst, lhs, rhs); + } + } + else + { + fmt::throw_exception("Unimplemented"); + } +} + +void asmjit::simd_builder::vec_extract_high(u32, const Operand& dst, const Operand& src) +{ + if (vsize == 32) + this->vextracti32x8(x86::Ymm(dst.id()), x86::Zmm(src.id()), 1); + else if (vsize == 16) + this->vextracti128(x86::Xmm(dst.id()), x86::Ymm(src.id()), 1); + else + { + if (utils::has_avx()) + this->vpsrldq(x86::Xmm(dst.id()), x86::Xmm(src.id()), vsize); + else + { + this->movdqa(x86::Xmm(dst.id()), x86::Xmm(src.id())); + this->psrldq(x86::Xmm(dst.id()), vsize); + } + } +} + +void asmjit::simd_builder::vec_extract_gpr(u32 esize, const x86::Gp& dst, const Operand& src) +{ + if (esize == 8 && utils::has_avx()) + this->vmovq(dst.r64(), x86::Xmm(src.id())); + else if (esize == 8) + this->movq(dst.r64(), x86::Xmm(src.id())); + else if (esize == 4 && utils::has_avx()) + this->vmovd(dst.r32(), x86::Xmm(src.id())); + else if (esize == 4) + this->movd(dst.r32(), x86::Xmm(src.id())); + else if (esize == 2 && utils::has_avx()) + this->vpextrw(dst.r32(), x86::Xmm(src.id()), 0); + else if (esize == 2) + this->pextrw(dst.r32(), x86::Xmm(src.id()), 0); + else + fmt::throw_exception("Unimplemented"); +} + +#endif /* X86 */ + +#ifdef LLVM_AVAILABLE + +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(push, 0) +#else +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#pragma GCC diagnostic ignored "-Wextra" +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#pragma GCC diagnostic ignored "-Wredundant-decls" +#pragma GCC diagnostic ignored "-Weffc++" +#pragma GCC diagnostic ignored "-Wmissing-noreturn" +#endif +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/TargetParser/Host.h" +#include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/SymbolSize.h" +#ifdef _MSC_VER +#pragma warning(pop) +#else +#pragma GCC diagnostic pop +#endif + +const bool jit_initialize = []() -> bool +{ + llvm::InitializeNativeTarget(); + llvm::InitializeNativeTargetAsmPrinter(); + llvm::InitializeNativeTargetAsmParser(); + LLVMLinkInMCJIT(); + return true; +}(); + +[[noreturn]] static void null(const char* name) +{ + fmt::throw_exception("Null function: %s", name); +} + +namespace vm +{ + extern u8* const g_sudo_addr; +} + +static shared_mutex null_mtx; + +static std::unordered_map null_funcs; + +static u64 make_null_function(const std::string& name) +{ + if (name.starts_with("__0x")) + { + u32 addr = -1; + auto res = std::from_chars(name.c_str() + 4, name.c_str() + name.size(), addr, 16); + + if (res.ec == std::errc() && res.ptr == name.c_str() + name.size() && addr < 0x8000'0000) + { + // Point the garbage to reserved, non-executable memory + return reinterpret_cast(vm::g_sudo_addr + addr); + } + } + + std::lock_guard lock(null_mtx); + + if (u64& func_ptr = null_funcs[name]) [[likely]] + { + // Already exists + return func_ptr; + } + else + { + using namespace asmjit; + + // Build a "null" function that contains its name + const auto func = build_function_asm("NULL", [&](native_asm& c, auto& args) + { +#if defined(ARCH_X64) + Label data = c.newLabel(); + c.lea(args[0], x86::qword_ptr(data, 0)); + c.jmp(Imm(&null)); + c.align(AlignMode::kCode, 16); + c.bind(data); + + // Copy function name bytes + for (char ch : name) + c.db(ch); + c.db(0); + c.align(AlignMode::kData, 16); +#else + // AArch64 implementation + Label jmp_address = c.newLabel(); + Label data = c.newLabel(); + // Force absolute jump to prevent out of bounds PC-rel jmp + c.ldr(args[0], arm::ptr(jmp_address)); + c.br(args[0]); + c.align(AlignMode::kCode, 16); + + c.bind(data); + c.embed(name.c_str(), name.size()); + c.embedUInt8(0U); + c.bind(jmp_address); + c.embedUInt64(reinterpret_cast(&null)); + c.align(AlignMode::kData, 16); +#endif + }); + + func_ptr = reinterpret_cast(func); + return func_ptr; + } +} + +struct JITAnnouncer : llvm::JITEventListener +{ + void notifyObjectLoaded(u64, const llvm::object::ObjectFile& obj, const llvm::RuntimeDyld::LoadedObjectInfo& info) override + { + using namespace llvm; + + object::OwningBinary debug_obj_ = info.getObjectForDebug(obj); + if (!debug_obj_.getBinary()) + { +#ifdef __linux__ + jit_log.error("LLVM: Failed to announce JIT events (no debug object)"); +#endif + return; + } + + const object::ObjectFile& debug_obj = *debug_obj_.getBinary(); + + for (const auto& [sym, size] : computeSymbolSizes(debug_obj)) + { + Expected type_ = sym.getType(); + if (!type_ || *type_ != object::SymbolRef::ST_Function) + continue; + + Expected name = sym.getName(); + if (!name) + continue; + + Expected addr = sym.getAddress(); + if (!addr) + continue; + + jit_announce(*addr, size, {name->data(), name->size()}); + } + } +}; + +// Simple memory manager +struct MemoryManager1 : llvm::RTDyldMemoryManager +{ + // 256 MiB for code or data + static constexpr u64 c_max_size = 0x20000000 / 2; + + // Allocation unit (2M) + static constexpr u64 c_page_size = 2 * 1024 * 1024; + + // Reserve 512 MiB + u8* const ptr = static_cast(utils::memory_reserve(c_max_size * 2)); + + u64 code_ptr = 0; + u64 data_ptr = c_max_size; + + MemoryManager1() = default; + + MemoryManager1(const MemoryManager1&) = delete; + + MemoryManager1& operator=(const MemoryManager1&) = delete; + + ~MemoryManager1() override + { + // Hack: don't release to prevent reuse of address space, see jit_announce + utils::memory_decommit(ptr, c_max_size * 2); + } + + llvm::JITSymbol findSymbol(const std::string& name) override + { + u64 addr = RTDyldMemoryManager::getSymbolAddress(name); + + if (!addr) + { + addr = make_null_function(name); + + if (!addr) + { + fmt::throw_exception("Failed to link '%s'", name); + } + } + + return {addr, llvm::JITSymbolFlags::Exported}; + } + + u8* allocate(u64& oldp, uptr size, uint align, utils::protection prot) + { + if (align > c_page_size) + { + jit_log.fatal("Unsupported alignment (size=0x%x, align=0x%x)", size, align); + return nullptr; + } + + const u64 olda = utils::align(oldp, align); + const u64 newp = utils::align(olda + size, align); + + if ((newp - 1) / c_max_size != oldp / c_max_size) + { + jit_log.fatal("Out of memory (size=0x%x, align=0x%x)", size, align); + return nullptr; + } + + if ((oldp - 1) / c_page_size != (newp - 1) / c_page_size) + { + // Allocate pages on demand + const u64 pagea = utils::align(oldp, c_page_size); + const u64 psize = utils::align(newp - pagea, c_page_size); + utils::memory_commit(this->ptr + pagea, psize, prot); + } + + // Update allocation counter + oldp = newp; + + return this->ptr + olda; + } + + u8* allocateCodeSection(uptr size, uint align, uint /*sec_id*/, llvm::StringRef /*sec_name*/) override + { + return allocate(code_ptr, size, align, utils::protection::wx); + } + + u8* allocateDataSection(uptr size, uint align, uint /*sec_id*/, llvm::StringRef /*sec_name*/, bool /*is_ro*/) override + { + return allocate(data_ptr, size, align, utils::protection::rw); + } + + bool finalizeMemory(std::string* = nullptr) override + { + return false; + } + + void registerEHFrames(u8*, u64, usz) override + { + } + + void deregisterEHFrames() override + { + } +}; + +// Simple memory manager +struct MemoryManager2 : llvm::RTDyldMemoryManager +{ + MemoryManager2() = default; + + ~MemoryManager2() override + { + } + + llvm::JITSymbol findSymbol(const std::string& name) override + { + u64 addr = RTDyldMemoryManager::getSymbolAddress(name); + + if (!addr) + { + addr = make_null_function(name); + + if (!addr) + { + fmt::throw_exception("Failed to link '%s' (MM2)", name); + } + } + + return {addr, llvm::JITSymbolFlags::Exported}; + } + + u8* allocateCodeSection(uptr size, uint align, uint /*sec_id*/, llvm::StringRef /*sec_name*/) override + { + return jit_runtime::alloc(size, align, true); + } + + u8* allocateDataSection(uptr size, uint align, uint /*sec_id*/, llvm::StringRef /*sec_name*/, bool /*is_ro*/) override + { + return jit_runtime::alloc(size, align, false); + } + + bool finalizeMemory(std::string* = nullptr) override + { + return false; + } + + void registerEHFrames(u8*, u64, usz) override + { + } + + void deregisterEHFrames() override + { + } +}; + +// Helper class +class ObjectCache final : public llvm::ObjectCache +{ + const std::string& m_path; + +public: + ObjectCache(const std::string& path) + : m_path(path) + { + } + + ~ObjectCache() override = default; + + void notifyObjectCompiled(const llvm::Module* _module, llvm::MemoryBufferRef obj) override + { + std::string name = m_path; + name.append(_module->getName().data()); + //fs::file(name, fs::rewrite).write(obj.getBufferStart(), obj.getBufferSize()); + name.append(".gz"); + + fs::file module_file(name, fs::rewrite); + + if (!module_file) + { + jit_log.error("LLVM: Failed to create module file: %s (%s)", name, fs::g_tls_error); + return; + } + + if (!zip(obj.getBufferStart(), obj.getBufferSize(), module_file)) + { + jit_log.error("LLVM: Failed to compress module: %s", _module->getName().data()); + module_file.close(); + fs::remove_file(name); + return; + } + + jit_log.notice("LLVM: Created module: %s", _module->getName().data()); + } + + static std::unique_ptr load(const std::string& path) + { + if (fs::file cached{path + ".gz", fs::read}) + { + const std::vector cached_data = cached.to_vector(); + + if (cached_data.empty()) [[unlikely]] + { + return nullptr; + } + + const std::vector out = unzip(cached_data); + + if (out.empty()) + { + jit_log.error("LLVM: Failed to unzip module: '%s'", path); + return nullptr; + } + + auto buf = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(out.size()); + std::memcpy(buf->getBufferStart(), out.data(), out.size()); + return buf; + } + + if (fs::file cached{path, fs::read}) + { + if (cached.size() == 0) [[unlikely]] + { + return nullptr; + } + + auto buf = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(cached.size()); + cached.read(buf->getBufferStart(), buf->getBufferSize()); + return buf; + } + + return nullptr; + } + + std::unique_ptr getObject(const llvm::Module* _module) override + { + std::string path = m_path; + path.append(_module->getName().data()); + + if (auto buf = load(path)) + { + jit_log.notice("LLVM: Loaded module: %s", _module->getName().data()); + return buf; + } + + return nullptr; + } +}; + +std::string jit_compiler::cpu(const std::string& _cpu) +{ + std::string m_cpu = _cpu; + + if (m_cpu.empty()) + { + m_cpu = llvm::sys::getHostCPUName().str(); + + if (m_cpu == "sandybridge" || + m_cpu == "ivybridge" || + m_cpu == "haswell" || + m_cpu == "broadwell" || + m_cpu == "skylake" || + m_cpu == "skylake-avx512" || + m_cpu == "cascadelake" || + m_cpu == "cooperlake" || + m_cpu == "cannonlake" || + m_cpu == "icelake" || + m_cpu == "icelake-client" || + m_cpu == "icelake-server" || + m_cpu == "tigerlake" || + m_cpu == "rocketlake" || + m_cpu == "alderlake" || + m_cpu == "raptorlake" || + m_cpu == "meteorlake") + { + // Downgrade if AVX is not supported by some chips + if (!utils::has_avx()) + { + m_cpu = "nehalem"; + } + } + + if (m_cpu == "skylake-avx512" || + m_cpu == "cascadelake" || + m_cpu == "cooperlake" || + m_cpu == "cannonlake" || + m_cpu == "icelake" || + m_cpu == "icelake-client" || + m_cpu == "icelake-server" || + m_cpu == "tigerlake" || + m_cpu == "rocketlake") + { + // Downgrade if AVX-512 is disabled or not supported + if (!utils::has_avx512()) + { + m_cpu = "skylake"; + } + } + + if (m_cpu == "znver1" && utils::has_clwb()) + { + // Upgrade + m_cpu = "znver2"; + } + + if ((m_cpu == "znver3" || m_cpu == "goldmont" || m_cpu == "alderlake" || m_cpu == "raptorlake" || m_cpu == "meteorlake") && utils::has_avx512_icl()) + { + // Upgrade + m_cpu = "icelake-client"; + } + + if (m_cpu == "goldmont" && utils::has_avx2()) + { + // Upgrade + m_cpu = "alderlake"; + } + } + + return m_cpu; +} + +std::string jit_compiler::triple1() +{ +#if defined(_WIN32) + return llvm::Triple::normalize(llvm::sys::getProcessTriple()); +#elif defined(__APPLE__) && defined(ARCH_X64) + return llvm::Triple::normalize("x86_64-unknown-linux-gnu"); +#elif defined(__APPLE__) && defined(ARCH_ARM64) + return llvm::Triple::normalize("aarch64-unknown-linux-gnu"); +#else + return llvm::Triple::normalize(llvm::sys::getProcessTriple()); +#endif +} + +std::string jit_compiler::triple2() +{ +#if defined(_WIN32) && defined(ARCH_X64) + return llvm::Triple::normalize("x86_64-unknown-linux-gnu"); +#elif defined(_WIN32) && defined(ARCH_ARM64) + return llvm::Triple::normalize("aarch64-unknown-linux-gnu"); +#elif defined(__APPLE__) && defined(ARCH_X64) + return llvm::Triple::normalize("x86_64-unknown-linux-gnu"); +#elif defined(__APPLE__) && defined(ARCH_ARM64) + return llvm::Triple::normalize("aarch64-unknown-linux-gnu"); +#else + return llvm::Triple::normalize(llvm::sys::getProcessTriple()); +#endif +} + +jit_compiler::jit_compiler(const std::unordered_map& _link, const std::string& _cpu, u32 flags) + : m_context(new llvm::LLVMContext) + , m_cpu(cpu(_cpu)) +{ + std::string result; + + auto null_mod = std::make_unique ("null_", *m_context); + null_mod->setTargetTriple(jit_compiler::triple1()); + + std::unique_ptr mem; + + if (_link.empty()) + { + // Auxiliary JIT (does not use custom memory manager, only writes the objects) + if (flags & 0x1) + { + mem = std::make_unique(); + } + else + { + mem = std::make_unique(); + null_mod->setTargetTriple(jit_compiler::triple2()); + } + } + else + { + mem = std::make_unique(); + } + + { + m_engine.reset(llvm::EngineBuilder(std::move(null_mod)) + .setErrorStr(&result) + .setEngineKind(llvm::EngineKind::JIT) + .setMCJITMemoryManager(std::move(mem)) + .setOptLevel(llvm::CodeGenOpt::Aggressive) + .setCodeModel(flags & 0x2 ? llvm::CodeModel::Large : llvm::CodeModel::Small) +#ifdef __APPLE__ + //.setCodeModel(llvm::CodeModel::Large) +#endif + .setRelocationModel(llvm::Reloc::Model::PIC_) + .setMCPU(m_cpu) + .create()); + } + + if (!_link.empty()) + { + for (auto&& [name, addr] : _link) + { + m_engine->updateGlobalMapping(name, addr); + } + } + + if (!_link.empty() || !(flags & 0x1)) + { + m_engine->RegisterJITEventListener(llvm::JITEventListener::createIntelJITEventListener()); + m_engine->RegisterJITEventListener(new JITAnnouncer); + } + + if (!m_engine) + { + fmt::throw_exception("LLVM: Failed to create ExecutionEngine: %s", result); + } +} + +jit_compiler::~jit_compiler() +{ +} + +void jit_compiler::add(std::unique_ptr _module, const std::string& path) +{ + ObjectCache cache{path}; + m_engine->setObjectCache(&cache); + + const auto ptr = _module.get(); + m_engine->addModule(std::move(_module)); + m_engine->generateCodeForModule(ptr); + m_engine->setObjectCache(nullptr); + + for (auto& func : ptr->functions()) + { + // Delete IR to lower memory consumption + func.deleteBody(); + } +} + +void jit_compiler::add(std::unique_ptr _module) +{ + const auto ptr = _module.get(); + m_engine->addModule(std::move(_module)); + m_engine->generateCodeForModule(ptr); + + for (auto& func : ptr->functions()) + { + // Delete IR to lower memory consumption + func.deleteBody(); + } +} + +void jit_compiler::add(const std::string& path) +{ + auto cache = ObjectCache::load(path); + + if (auto object_file = llvm::object::ObjectFile::createObjectFile(*cache)) + { + m_engine->addObjectFile(llvm::object::OwningBinary(std::move(*object_file), std::move(cache))); + } + else + { + jit_log.error("ObjectCache: Adding failed: %s", path); + } +} + +bool jit_compiler::check(const std::string& path) +{ + if (auto cache = ObjectCache::load(path)) + { + if (auto object_file = llvm::object::ObjectFile::createObjectFile(*cache)) + { + return true; + } + + if (fs::remove_file(path)) + { + jit_log.error("ObjectCache: Removed damaged file: %s", path); + } + } + + return false; +} + +void jit_compiler::update_global_mapping(const std::string& name, u64 addr) +{ + m_engine->updateGlobalMapping(name, addr); +} + +void jit_compiler::fin() +{ + m_engine->finalizeObject(); +} + +u64 jit_compiler::get(const std::string& name) +{ + return m_engine->getGlobalValueAddress(name); +} + +#endif // LLVM_AVAILABLE diff --git a/rpcs3/Emu/CMakeLists.txt b/rpcs3/Emu/CMakeLists.txt index d8c33ae693..ea546e40d4 100644 --- a/rpcs3/Emu/CMakeLists.txt +++ b/rpcs3/Emu/CMakeLists.txt @@ -56,7 +56,8 @@ target_sources(rpcs3_emu PRIVATE ../../Utilities/cond.cpp ../../Utilities/Config.cpp ../../Utilities/File.cpp - ../../Utilities/JIT.cpp + ../../Utilities/JITASM.cpp + ../../Utilities/JITLLVM.cpp ../../Utilities/LUrlParser.cpp ../../Utilities/mutex.cpp ../../Utilities/rXml.cpp @@ -199,7 +200,8 @@ target_sources(rpcs3_emu PRIVATE Cell/SPUASMJITRecompiler.cpp Cell/SPUDisAsm.cpp Cell/SPUInterpreter.cpp - Cell/SPURecompiler.cpp + Cell/SPUCommonRecompiler.cpp + Cell/SPULLVMRecompiler.cpp Cell/SPUThread.cpp Cell/lv2/lv2.cpp Cell/lv2/sys_bdemu.cpp diff --git a/rpcs3/Emu/Cell/SPURecompiler.cpp b/rpcs3/Emu/Cell/SPUCommonRecompiler.cpp similarity index 100% rename from rpcs3/Emu/Cell/SPURecompiler.cpp rename to rpcs3/Emu/Cell/SPUCommonRecompiler.cpp diff --git a/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp b/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp new file mode 100644 index 0000000000..396412dc7a --- /dev/null +++ b/rpcs3/Emu/Cell/SPULLVMRecompiler.cpp @@ -0,0 +1,12409 @@ +#include "stdafx.h" +#include "SPURecompiler.h" + +#include "Emu/System.h" +#include "Emu/system_config.h" +#include "Emu/system_progress.hpp" +#include "Emu/system_utils.hpp" +#include "Emu/cache_utils.hpp" +#include "Emu/IdManager.h" +#include "Emu/Cell/timers.hpp" +#include "Crypto/sha1.h" +#include "Utilities/StrUtil.h" +#include "Utilities/JIT.h" +#include "util/init_mutex.hpp" +#include "util/shared_ptr.hpp" + +#include "SPUThread.h" +#include "SPUAnalyser.h" +#include "SPUInterpreter.h" +#include "SPUDisAsm.h" +#include +#include +#include +#include +#include + +#include "util/v128.hpp" +#include "util/simd.hpp" +#include "util/sysinfo.hpp" + +const extern spu_decoder g_spu_itype; +const extern spu_decoder g_spu_iname; +const extern spu_decoder g_spu_iflag; + +// Move 4 args for calling native function from a GHC calling convention function +#if defined(ARCH_X64) +static u8* move_args_ghc_to_native(u8* raw) +{ +#ifdef _WIN32 + // mov rcx, r13 + // mov rdx, rbp + // mov r8, r12 + // mov r9, rbx + std::memcpy(raw, "\x4C\x89\xE9\x48\x89\xEA\x4D\x89\xE0\x49\x89\xD9", 12); +#else + // mov rdi, r13 + // mov rsi, rbp + // mov rdx, r12 + // mov rcx, rbx + std::memcpy(raw, "\x4C\x89\xEF\x48\x89\xEE\x4C\x89\xE2\x48\x89\xD9", 12); +#endif + + return raw + 12; +} +#elif defined(ARCH_ARM64) +static void ghc_cpp_trampoline(u64 fn_target, native_asm& c, auto& args) +{ + using namespace asmjit; + + Label target = c.newLabel(); + c.mov(args[0], a64::x19); + c.mov(args[1], a64::x20); + c.mov(args[2], a64::x21); + c.mov(args[3], a64::x22); + + c.ldr(a64::x15, arm::Mem(target)); + c.br(a64::x15); + + c.brk(Imm(0x42)); // Unreachable + + c.bind(target); + c.embedUInt64(fn_target); +} +#endif + +DECLARE(spu_runtime::tr_dispatch) = [] +{ +#ifdef __APPLE__ + pthread_jit_write_protect_np(false); +#endif +#if defined(ARCH_X64) + // Generate a special trampoline to spu_recompiler_base::dispatch with pause instruction + u8* const trptr = jit_runtime::alloc(32, 16); + u8* raw = move_args_ghc_to_native(trptr); + *raw++ = 0xf3; // pause + *raw++ = 0x90; + *raw++ = 0xff; // jmp [rip] + *raw++ = 0x25; + std::memset(raw, 0, 4); + const u64 target = reinterpret_cast(&spu_recompiler_base::dispatch); + std::memcpy(raw + 4, &target, 8); + return reinterpret_cast(trptr); +#elif defined(ARCH_ARM64) + auto trptr = build_function_asm("tr_dispatch", + [](native_asm& c, auto& args) + { + c.yield(); + ghc_cpp_trampoline(reinterpret_cast(&spu_recompiler_base::dispatch), c, args); + + c.embed("tr_dispatch", 11); + }); + return trptr; +#else +#error "Unimplemented" +#endif +}(); + +DECLARE(spu_runtime::tr_branch) = [] +{ +#if defined(ARCH_X64) + // Generate a trampoline to spu_recompiler_base::branch + u8* const trptr = jit_runtime::alloc(32, 16); + u8* raw = move_args_ghc_to_native(trptr); + *raw++ = 0xff; // jmp [rip] + *raw++ = 0x25; + std::memset(raw, 0, 4); + const u64 target = reinterpret_cast(&spu_recompiler_base::branch); + std::memcpy(raw + 4, &target, 8); + return reinterpret_cast(trptr); + +#elif defined(ARCH_ARM64) + auto trptr = build_function_asm("tr_branch", + [](native_asm& c, auto& args) + { + ghc_cpp_trampoline(reinterpret_cast(&spu_recompiler_base::branch), c, args); + + c.embed("tr_branch", 9); + }); + return trptr; +#else +#error "Unimplemented" +#endif +}(); + +DECLARE(spu_runtime::tr_interpreter) = [] +{ +#if defined(ARCH_X64) + u8* const trptr = jit_runtime::alloc(32, 16); + u8* raw = move_args_ghc_to_native(trptr); + *raw++ = 0xff; // jmp [rip] + *raw++ = 0x25; + std::memset(raw, 0, 4); + const u64 target = reinterpret_cast(&spu_recompiler_base::old_interpreter); + std::memcpy(raw + 4, &target, 8); + return reinterpret_cast(trptr); +#elif defined(ARCH_ARM64) + auto trptr = build_function_asm("tr_interpreter", + [](native_asm& c, auto& args) + { + ghc_cpp_trampoline(reinterpret_cast(&spu_recompiler_base::old_interpreter), c, args); + + c.embed("tr_interpreter", 14); + }); + return trptr; +#endif +}(); + +DECLARE(spu_runtime::g_dispatcher) = [] +{ + // Allocate 2^20 positions in data area + const auto ptr = reinterpret_cast>(jit_runtime::alloc(sizeof(*g_dispatcher), 64, false)); + + for (auto& x : *ptr) + { + x.raw() = tr_dispatch; + } + + return ptr; +}(); + +DECLARE(spu_runtime::tr_all) = [] +{ +#if defined(ARCH_X64) + u8* const trptr = jit_runtime::alloc(32, 16); + u8* raw = trptr; + + // Load PC: mov eax, [r13 + spu_thread::pc] + *raw++ = 0x41; + *raw++ = 0x8b; + *raw++ = 0x45; + *raw++ = ::narrow(::offset32(&spu_thread::pc)); + + // Get LS address starting from PC: lea rcx, [rbp + rax] + *raw++ = 0x48; + *raw++ = 0x8d; + *raw++ = 0x4c; + *raw++ = 0x05; + *raw++ = 0x00; + + // mov eax, [rcx] + *raw++ = 0x8b; + *raw++ = 0x01; + + // shr eax, (32 - 20) + *raw++ = 0xc1; + *raw++ = 0xe8; + *raw++ = 0x0c; + + // Load g_dispatcher to rdx + *raw++ = 0x48; + *raw++ = 0x8d; + *raw++ = 0x15; + const s32 r32 = ::narrow(reinterpret_cast(g_dispatcher) - reinterpret_cast(raw) - 4); + std::memcpy(raw, &r32, 4); + raw += 4; + + // Update block_hash (set zero): mov [r13 + spu_thread::m_block_hash], 0 + *raw++ = 0x49; + *raw++ = 0xc7; + *raw++ = 0x45; + *raw++ = ::narrow(::offset32(&spu_thread::block_hash)); + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x00; + + // jmp [rdx + rax * 8] + *raw++ = 0xff; + *raw++ = 0x24; + *raw++ = 0xc2; + + return reinterpret_cast(trptr); + +#elif defined(ARCH_ARM64) + auto trptr = build_function_asm("tr_all", + [](native_asm& c, auto& args) + { + using namespace asmjit; + + // w1: PC (eax in x86 SPU) + // x7: lsa (rcx in x86 SPU) + + // Load PC + Label pc_offset = c.newLabel(); + c.ldr(a64::x0, arm::Mem(pc_offset)); + c.ldr(a64::w1, arm::Mem(a64::x19, a64::x0)); // REG_Base + offset(spu_thread::pc) + // Compute LS address = REG_Sp + PC, store into x7 (use later) + c.add(a64::x7, a64::x20, a64::x1); + // Load 32b from LS address + c.ldr(a64::w3, arm::Mem(a64::x7)); + // shr (32 - 20) + c.lsr(a64::w3, a64::w3, Imm(32 - 20)); + // Load g_dispatcher + Label g_dispatcher_offset = c.newLabel(); + c.ldr(a64::x4, arm::Mem(g_dispatcher_offset)); + // Update block hash + Label block_hash_offset = c.newLabel(); + c.mov(a64::x5, Imm(0)); + c.ldr(a64::x6, arm::Mem(block_hash_offset)); + c.str(a64::x5, arm::Mem(a64::x19, a64::x6)); // REG_Base + offset(spu_thread::block_hash) + // Jump to [g_dispatcher + idx * 8] + c.mov(a64::x6, Imm(8)); + c.mul(a64::x6, a64::x3, a64::x6); + c.add(a64::x4, a64::x4, a64::x6); + c.ldr(a64::x4, arm::Mem(a64::x4)); + c.br(a64::x4); + + c.bind(pc_offset); + c.embedUInt64(::offset32(&spu_thread::pc)); + c.bind(g_dispatcher_offset); + c.embedUInt64(reinterpret_cast(g_dispatcher)); + c.bind(block_hash_offset); + c.embedUInt64(::offset32(&spu_thread::block_hash)); + c.embed("tr_all", 6); + }); + return trptr; +#else +#error "Unimplemented" +#endif +}(); + +DECLARE(spu_runtime::g_gateway) = build_function_asm("spu_gateway", [](native_asm& c, auto& args) +{ + // Gateway for SPU dispatcher, converts from native to GHC calling convention, also saves RSP value for spu_escape + using namespace asmjit; + +#if defined(ARCH_X64) +#ifdef _WIN32 + c.push(x86::r15); + c.push(x86::r14); + c.push(x86::r13); + c.push(x86::r12); + c.push(x86::rsi); + c.push(x86::rdi); + c.push(x86::rbp); + c.push(x86::rbx); + c.sub(x86::rsp, 0xa8); + c.movaps(x86::oword_ptr(x86::rsp, 0x90), x86::xmm15); + c.movaps(x86::oword_ptr(x86::rsp, 0x80), x86::xmm14); + c.movaps(x86::oword_ptr(x86::rsp, 0x70), x86::xmm13); + c.movaps(x86::oword_ptr(x86::rsp, 0x60), x86::xmm12); + c.movaps(x86::oword_ptr(x86::rsp, 0x50), x86::xmm11); + c.movaps(x86::oword_ptr(x86::rsp, 0x40), x86::xmm10); + c.movaps(x86::oword_ptr(x86::rsp, 0x30), x86::xmm9); + c.movaps(x86::oword_ptr(x86::rsp, 0x20), x86::xmm8); + c.movaps(x86::oword_ptr(x86::rsp, 0x10), x86::xmm7); + c.movaps(x86::oword_ptr(x86::rsp, 0), x86::xmm6); +#else + c.push(x86::rbp); + c.push(x86::r15); + c.push(x86::r14); + c.push(x86::r13); + c.push(x86::r12); + c.push(x86::rbx); + c.push(x86::rax); +#endif + + // Save native stack pointer for longjmp emulation + c.mov(x86::qword_ptr(args[0], ::offset32(&spu_thread::saved_native_sp)), x86::rsp); + + // Move 4 args (despite spu_function_t def) + c.mov(x86::r13, args[0]); + c.mov(x86::rbp, args[1]); + c.mov(x86::r12, args[2]); + c.mov(x86::rbx, args[3]); + + if (utils::has_avx()) + { + c.vzeroupper(); + } + + c.call(spu_runtime::tr_all); + + if (utils::has_avx()) + { + c.vzeroupper(); + } + +#ifdef _WIN32 + c.movaps(x86::xmm6, x86::oword_ptr(x86::rsp, 0)); + c.movaps(x86::xmm7, x86::oword_ptr(x86::rsp, 0x10)); + c.movaps(x86::xmm8, x86::oword_ptr(x86::rsp, 0x20)); + c.movaps(x86::xmm9, x86::oword_ptr(x86::rsp, 0x30)); + c.movaps(x86::xmm10, x86::oword_ptr(x86::rsp, 0x40)); + c.movaps(x86::xmm11, x86::oword_ptr(x86::rsp, 0x50)); + c.movaps(x86::xmm12, x86::oword_ptr(x86::rsp, 0x60)); + c.movaps(x86::xmm13, x86::oword_ptr(x86::rsp, 0x70)); + c.movaps(x86::xmm14, x86::oword_ptr(x86::rsp, 0x80)); + c.movaps(x86::xmm15, x86::oword_ptr(x86::rsp, 0x90)); + c.add(x86::rsp, 0xa8); + c.pop(x86::rbx); + c.pop(x86::rbp); + c.pop(x86::rdi); + c.pop(x86::rsi); + c.pop(x86::r12); + c.pop(x86::r13); + c.pop(x86::r14); + c.pop(x86::r15); +#else + c.add(x86::rsp, +8); + c.pop(x86::rbx); + c.pop(x86::r12); + c.pop(x86::r13); + c.pop(x86::r14); + c.pop(x86::r15); + c.pop(x86::rbp); +#endif + + c.ret(); +#elif defined(ARCH_ARM64) + // Push callee saved registers to the stack + // We need to save x18-x30 = 13 x 8B each + 8 bytes for 16B alignment = 112B + c.sub(a64::sp, a64::sp, Imm(112)); + c.stp(a64::x18, a64::x19, arm::Mem(a64::sp)); + c.stp(a64::x20, a64::x21, arm::Mem(a64::sp, 16)); + c.stp(a64::x22, a64::x23, arm::Mem(a64::sp, 32)); + c.stp(a64::x24, a64::x25, arm::Mem(a64::sp, 48)); + c.stp(a64::x26, a64::x27, arm::Mem(a64::sp, 64)); + c.stp(a64::x28, a64::x29, arm::Mem(a64::sp, 80)); + c.str(a64::x30, arm::Mem(a64::sp, 96)); + + // Save native stack pointer for longjmp emulation + Label sp_offset = c.newLabel(); + c.ldr(a64::x26, arm::Mem(sp_offset)); + // sp not allowed to be used in load/stores directly + c.mov(a64::x15, a64::sp); + c.str(a64::x15, arm::Mem(args[0], a64::x26)); + + // Move 4 args (despite spu_function_t def) + c.mov(a64::x19, args[0]); + c.mov(a64::x20, args[1]); + c.mov(a64::x21, args[2]); + c.mov(a64::x22, args[3]); + + // Save ret address to stack + // since non-tail calls to cpp fns may corrupt lr and + // g_tail_escape may jump out of a fn before the epilogue can restore lr + Label ret_addr = c.newLabel(); + c.adr(a64::x0, ret_addr); + c.str(a64::x0, arm::Mem(a64::sp, 104)); + + Label call_target = c.newLabel(); + c.ldr(a64::x0, arm::Mem(call_target)); + c.blr(a64::x0); + + c.bind(ret_addr); + + // Restore stack ptr + c.ldr(a64::x26, arm::Mem(sp_offset)); + c.ldr(a64::x15, arm::Mem(a64::x19, a64::x26)); + c.mov(a64::sp, a64::x15); + + // Restore registers from the stack + c.ldp(a64::x18, a64::x19, arm::Mem(a64::sp)); + c.ldp(a64::x20, a64::x21, arm::Mem(a64::sp, 16)); + c.ldp(a64::x22, a64::x23, arm::Mem(a64::sp, 32)); + c.ldp(a64::x24, a64::x25, arm::Mem(a64::sp, 48)); + c.ldp(a64::x26, a64::x27, arm::Mem(a64::sp, 64)); + c.ldp(a64::x28, a64::x29, arm::Mem(a64::sp, 80)); + c.ldr(a64::x30, arm::Mem(a64::sp, 96)); + // Restore stack ptr + c.add(a64::sp, a64::sp, Imm(112)); + // Return + c.ret(a64::x30); + + c.bind(sp_offset); + c.embedUInt64(::offset32(&spu_thread::saved_native_sp)); + c.bind(call_target); + c.embedUInt64(reinterpret_cast(spu_runtime::tr_all)); + c.embed("spu_gateway", 11); +#else +#error "Unimplemented" +#endif +}); + +DECLARE(spu_runtime::g_escape) = build_function_asm("spu_escape", [](native_asm& c, auto& args) +{ + using namespace asmjit; + +#if defined(ARCH_X64) + // Restore native stack pointer (longjmp emulation) + c.mov(x86::rsp, x86::qword_ptr(args[0], ::offset32(&spu_thread::saved_native_sp))); + + // Return to the return location + c.sub(x86::rsp, 8); + c.ret(); +#elif defined(ARCH_ARM64) + // Restore native stack pointer (longjmp emulation) + Label sp_offset = c.newLabel(); + c.ldr(a64::x15, arm::Mem(sp_offset)); + c.ldr(a64::x15, arm::Mem(args[0], a64::x15)); + c.mov(a64::sp, a64::x15); + + c.ldr(a64::x30, arm::Mem(a64::sp, 104)); + c.ret(a64::x30); + + c.bind(sp_offset); + c.embedUInt64(::offset32(&spu_thread::saved_native_sp)); + + c.embed("spu_escape", 10); +#else +#error "Unimplemented" +#endif +}); + +DECLARE(spu_runtime::g_tail_escape) = build_function_asm("spu_tail_escape", [](native_asm& c, auto& args) +{ + using namespace asmjit; + +#if defined(ARCH_X64) + // Restore native stack pointer (longjmp emulation) + c.mov(x86::rsp, x86::qword_ptr(args[0], ::offset32(&spu_thread::saved_native_sp))); + + // Adjust stack for initial call instruction in the gateway + c.sub(x86::rsp, 16); + + // Tail call, GHC CC (second arg) + c.mov(x86::r13, args[0]); + c.mov(x86::rbp, x86::qword_ptr(args[0], ::offset32(&spu_thread::ls))); + c.mov(x86::r12, args[2]); + c.xor_(x86::ebx, x86::ebx); + c.mov(x86::qword_ptr(x86::rsp), args[1]); + c.ret(); +#elif defined(ARCH_ARM64) + // Restore native stack pointer (longjmp emulation) + Label sp_offset = c.newLabel(); + c.ldr(a64::x15, arm::Mem(sp_offset)); + c.ldr(a64::x15, arm::Mem(args[0], a64::x15)); + c.mov(a64::sp, a64::x15); + + // Reload lr, since it might've been clobbered by a cpp fn + // and g_tail_escape runs before epilogue + c.ldr(a64::x30, arm::Mem(a64::sp, 104)); + + // Tail call, GHC CC + c.mov(a64::x19, args[0]); // REG_Base + Label ls_offset = c.newLabel(); + c.ldr(a64::x20, arm::Mem(ls_offset)); + c.ldr(a64::x20, arm::Mem(args[0], a64::x20)); // REG_Sp + c.mov(a64::x21, args[2]); // REG_Hp + c.eor(a64::w22, a64::w22, a64::w22); // REG_R1 + + c.br(args[1]); + + c.bind(ls_offset); + c.embedUInt64(::offset32(&spu_thread::ls)); + c.bind(sp_offset); + c.embedUInt64(::offset32(&spu_thread::saved_native_sp)); + + c.embed("spu_tail_escape", 15); +#else +#error "Unimplemented" +#endif +}); + +DECLARE(spu_runtime::g_interpreter_table) = {}; + +DECLARE(spu_runtime::g_interpreter) = nullptr; + +spu_cache::spu_cache(const std::string& loc) + : m_file(loc, fs::read + fs::write + fs::create + fs::append) +{ +} + +spu_cache::~spu_cache() +{ +} + +extern void utilize_spu_data_segment(u32 vaddr, const void* ls_data_vaddr, u32 size) +{ + if (vaddr % 4) + { + return; + } + + size &= -4; + + if (!size || vaddr + size > SPU_LS_SIZE) + { + return; + } + + if (!g_cfg.core.llvm_precompilation) + { + return; + } + + g_fxo->need(); + + if (!g_fxo->get().collect_funcs_to_precompile) + { + return; + } + + std::basic_string data(size / 4, 0); + std::memcpy(data.data(), ls_data_vaddr, size); + + spu_cache::precompile_data_t obj{vaddr, std::move(data)}; + + obj.funcs = spu_thread::discover_functions(vaddr, { reinterpret_cast(ls_data_vaddr), size }, vaddr != 0, umax); + + if (obj.funcs.empty()) + { + // Nothing to add + return; + } + + for (u32 addr : obj.funcs) + { + spu_log.notice("Found SPU function at: 0x%05x", addr); + } + + spu_log.notice("Found %u SPU functions", obj.funcs.size()); + + g_fxo->get().precompile_funcs.push(std::move(obj)); +} + +// For SPU cache validity check +static u16 calculate_crc16(const uchar* data, usz length) +{ + u16 crc = umax; + + while (length--) + { + u8 x = (crc >> 8) ^ *data++; + x ^= (x >> 4); + crc = static_cast((crc << 8) ^ (x << 12) ^ (x << 5) ^ x); + } + + return crc; +} + +std::deque spu_cache::get() +{ + std::deque result; + + if (!m_file) + { + return result; + } + + m_file.seek(0); + + // TODO: signal truncated or otherwise broken file + while (true) + { + struct block_info_t + { + be_t crc; + be_t size; + be_t addr; + } block_info{}; + + if (!m_file.read(block_info)) + { + break; + } + + const u32 crc = block_info.crc; + const u32 size = block_info.size; + const u32 addr = block_info.addr; + + if (utils::add_saturate(addr, size * 4) > SPU_LS_SIZE) + { + break; + } + + std::vector func; + + if (!m_file.read(func, size)) + { + break; + } + + if (!size || !func[0]) + { + // Skip old format Giga entries + continue; + } + + // CRC check is optional to be compatible with old format + if (crc && std::max(calculate_crc16(reinterpret_cast(func.data()), size * 4), 1) != crc) + { + // Invalid, but continue anyway + continue; + } + + spu_program res; + res.entry_point = addr; + res.lower_bound = addr; + res.data = std::move(func); + result.emplace_front(std::move(res)); + } + + return result; +} + +void spu_cache::add(const spu_program& func) +{ + if (!m_file) + { + return; + } + + be_t size = ::size32(func.data); + be_t addr = func.entry_point; + + // Add CRC (forced non-zero) + size |= std::max(calculate_crc16(reinterpret_cast(func.data.data()), size * 4), 1) << 16; + + const fs::iovec_clone gather[3] + { + {&size, sizeof(size)}, + {&addr, sizeof(addr)}, + {func.data.data(), func.data.size() * 4} + }; + + // Append data + m_file.write_gather(gather, 3); +} + +void spu_cache::initialize(bool build_existing_cache) +{ + spu_runtime::g_interpreter = spu_runtime::g_gateway; + + if (g_cfg.core.spu_decoder == spu_decoder_type::_static || g_cfg.core.spu_decoder == spu_decoder_type::dynamic) + { + for (auto& x : *spu_runtime::g_dispatcher) + { + x.raw() = spu_runtime::tr_interpreter; + } + } + + const std::string ppu_cache = rpcs3::cache::get_ppu_cache(); + + if (ppu_cache.empty()) + { + return; + } + + // SPU cache file (version + block size type) + const std::string loc = ppu_cache + "spu-" + fmt::to_lower(g_cfg.core.spu_block_size.to_string()) + "-v1-tane.dat"; + + spu_cache cache(loc); + + if (!cache) + { + spu_log.error("Failed to initialize SPU cache at: %s", loc); + return; + } + + // Read cache + auto func_list = cache.get(); + atomic_t fnext{}; + atomic_t fail_flag{0}; + + auto data_list = g_fxo->get().precompile_funcs.pop_all(); + g_fxo->get().collect_funcs_to_precompile = false; + + u32 total_precompile = 0; + + for (auto& sec : data_list) + { + total_precompile += sec.funcs.size(); + } + + const bool spu_precompilation_enabled = func_list.empty() && g_cfg.core.spu_cache && g_cfg.core.llvm_precompilation; + + if (spu_precompilation_enabled) + { + // What compiles in this case goes straight to disk + g_fxo->get() = std::move(cache); + } + else if (!build_existing_cache) + { + return; + } + else + { + total_precompile = 0; + data_list = {}; + } + + atomic_t data_indexer = 0; + + if (g_cfg.core.spu_decoder == spu_decoder_type::dynamic || g_cfg.core.spu_decoder == spu_decoder_type::llvm) + { + if (auto compiler = spu_recompiler_base::make_llvm_recompiler(11)) + { + compiler->init(); + + if (compiler->compile({}) && spu_runtime::g_interpreter) + { + spu_log.success("SPU Runtime: Built the interpreter."); + + if (g_cfg.core.spu_decoder != spu_decoder_type::llvm) + { + return; + } + } + else + { + spu_log.fatal("SPU Runtime: Failed to build the interpreter."); + } + } + } + + u32 worker_count = 0; + + std::optional progr; + + u32 total_funcs = 0; + + if (g_cfg.core.spu_decoder == spu_decoder_type::asmjit || g_cfg.core.spu_decoder == spu_decoder_type::llvm) + { + const u32 add_count = ::size32(func_list) + total_precompile; + + if (add_count) + { + total_funcs = build_existing_cache ? add_count : 0; + } + + worker_count = std::min(rpcs3::utils::get_max_threads(), add_count); + } + + atomic_t pending_progress = 0; + atomic_t showing_progress = false; + + if (!g_progr_ptotal) + { + g_progr_ptotal += total_funcs; + showing_progress.release(true); + progr.emplace("Building SPU cache..."); + } + + named_thread_group workers("SPU Worker ", worker_count, [&]() -> uint + { +#ifdef __APPLE__ + pthread_jit_write_protect_np(false); +#endif + // Set low priority + thread_ctrl::scoped_priority low_prio(-1); + + // Initialize compiler instances for parallel compilation + std::unique_ptr compiler; + + if (g_cfg.core.spu_decoder == spu_decoder_type::asmjit) + { + compiler = spu_recompiler_base::make_asmjit_recompiler(); + } + else if (g_cfg.core.spu_decoder == spu_decoder_type::llvm) + { + compiler = spu_recompiler_base::make_llvm_recompiler(); + } + + compiler->init(); + + // How much every thread compiled + uint result = 0; + + // Fake LS + std::vector> ls(0x10000); + + usz func_i = fnext++; + + // Ensure some actions are performed on a single thread + const bool is_first_thread = func_i == 0; + + // Build functions + for (; func_i < func_list.size(); func_i = fnext++, (showing_progress ? g_progr_pdone : pending_progress) += build_existing_cache ? 1 : 0) + { + const spu_program& func = std::as_const(func_list)[func_i]; + + if (Emu.IsStopped() || fail_flag) + { + continue; + } + + // Get data start + const u32 start = func.lower_bound; + const u32 size0 = ::size32(func.data); + + be_t hash_start; + { + sha1_context ctx; + u8 output[20]; + + sha1_starts(&ctx); + sha1_update(&ctx, reinterpret_cast(func.data.data()), func.data.size() * 4); + sha1_finish(&ctx, output); + std::memcpy(&hash_start, output, sizeof(hash_start)); + } + + // Check hash against allowed bounds + const bool inverse_bounds = g_cfg.core.spu_llvm_lower_bound > g_cfg.core.spu_llvm_upper_bound; + + if ((!inverse_bounds && (hash_start < g_cfg.core.spu_llvm_lower_bound || hash_start > g_cfg.core.spu_llvm_upper_bound)) || + (inverse_bounds && (hash_start < g_cfg.core.spu_llvm_lower_bound && hash_start > g_cfg.core.spu_llvm_upper_bound))) + { + spu_log.error("[Debug] Skipped function %s", fmt::base57(hash_start)); + result++; + continue; + } + + // Initialize LS with function data only + for (u32 i = 0, pos = start; i < size0; i++, pos += 4) + { + ls[pos / 4] = std::bit_cast>(func.data[i]); + } + + // Call analyser + spu_program func2 = compiler->analyse(ls.data(), func.entry_point); + + if (func2 != func) + { + spu_log.error("[0x%05x] SPU Analyser failed, %u vs %u", func2.entry_point, func2.data.size(), size0); + } + else if (!compiler->compile(std::move(func2))) + { + // Likely, out of JIT memory. Signal to prevent further building. + fail_flag |= 1; + continue; + } + + // Clear fake LS + std::memset(ls.data() + start / 4, 0, 4 * (size0 - 1)); + + result++; + + if (is_first_thread && !showing_progress) + { + if (!g_progr.load() && !g_progr_ptotal && !g_progr_ftotal) + { + showing_progress = true; + g_progr_pdone += pending_progress.exchange(0); + g_progr_ptotal += total_funcs; + progr.emplace("Building SPU cache..."); + } + } + else if (showing_progress && pending_progress) + { + // Cover missing progress due to a race + g_progr_pdone += pending_progress.exchange(0); + } + } + + u32 last_sec_idx = umax; + + for (func_i = data_indexer++;; func_i = data_indexer++, (showing_progress ? g_progr_pdone : pending_progress) += build_existing_cache ? 1 : 0) + { + u32 passed_count = 0; + u32 func_addr = 0; + u32 next_func = 0; + u32 sec_addr = umax; + u32 sec_idx = 0; + std::basic_string_view inst_data; + + // Try to get the data this index points to + for (auto& sec : data_list) + { + if (func_i < passed_count + sec.funcs.size()) + { + const u32 func_idx = func_i - passed_count; + sec_addr = sec.vaddr; + func_addr = ::at32(sec.funcs, func_idx); + inst_data = sec.inst_data; + next_func = sec.funcs.size() >= func_idx ? sec_addr + inst_data.size() * 4 : sec.funcs[func_idx]; + break; + } + + passed_count += sec.funcs.size(); + sec_idx++; + } + + if (sec_addr == umax) + { + // End of compilation for thread + break; + } + + if (Emu.IsStopped() || fail_flag) + { + continue; + } + + if (last_sec_idx != sec_idx) + { + if (last_sec_idx != umax) + { + // Clear fake LS of previous section + auto& sec = data_list[last_sec_idx]; + std::memset(ls.data() + sec.vaddr / 4, 0, sec.inst_data.size() * 4); + } + + // Initialize LS with the entire section data + for (u32 i = 0, pos = sec_addr; i < inst_data.size(); i++, pos += 4) + { + ls[pos / 4] = std::bit_cast>(inst_data[i]); + } + + last_sec_idx = sec_idx; + } + + u32 block_addr = func_addr; + + std::map> targets; + + // Call analyser + spu_program func2 = compiler->analyse(ls.data(), block_addr, &targets); + + while (!func2.data.empty()) + { + const u32 last_inst = std::bit_cast>(func2.data.back()); + const u32 prog_size = func2.data.size(); + + if (!compiler->compile(std::move(func2))) + { + // Likely, out of JIT memory. Signal to prevent further building. + fail_flag |= 1; + break; + } + + result++; + + const u32 start_new = block_addr + prog_size * 4; + + if (start_new >= next_func || (start_new == next_func - 4 && ls[start_new / 4] == 0x200000u)) + { + // Completed + break; + } + + if (auto type = g_spu_itype.decode(last_inst); + type == spu_itype::BRSL || type == spu_itype::BRASL || type == spu_itype::BISL || type == spu_itype::SYNC) + { + if (ls[start_new / 4] && g_spu_itype.decode(ls[start_new / 4]) != spu_itype::UNK) + { + spu_log.notice("Precompiling fallthrough to 0x%05x", start_new); + func2 = compiler->analyse(ls.data(), start_new, &targets); + block_addr = start_new; + continue; + } + } + + if (targets.empty()) + { + break; + } + + const auto upper = targets.upper_bound(func_addr); + + if (upper == targets.begin()) + { + break; + } + + u32 new_entry = umax; + + // Find the lowest target in the space in-between + for (auto it = std::prev(upper); it != targets.end() && it->first < start_new && new_entry > start_new; it++) + { + for (u32 target : it->second) + { + if (target >= start_new && target < next_func) + { + if (target < new_entry) + { + new_entry = target; + + if (new_entry == start_new) + { + // Cannot go lower + break; + } + } + } + } + } + + if (new_entry != umax && !spu_thread::is_exec_code(new_entry, { reinterpret_cast(ls.data()), SPU_LS_SIZE })) + { + new_entry = umax; + } + + if (new_entry == umax) + { + new_entry = start_new; + + while (new_entry < next_func && (ls[start_new / 4] < 0x3fffc || !spu_thread::is_exec_code(new_entry, { reinterpret_cast(ls.data()), SPU_LS_SIZE }))) + { + new_entry += 4; + } + + if (new_entry >= next_func || (new_entry == next_func - 4 && ls[new_entry / 4] == 0x200000u)) + { + // Completed + break; + } + } + + + spu_log.notice("Precompiling filler space at 0x%05x (next=0x%05x)", new_entry, next_func); + func2 = compiler->analyse(ls.data(), new_entry, &targets); + block_addr = new_entry; + } + + if (is_first_thread && !showing_progress) + { + if (!g_progr.load() && !g_progr_ptotal && !g_progr_ftotal) + { + showing_progress = true; + g_progr_pdone += pending_progress.exchange(0); + g_progr_ptotal += total_funcs; + progr.emplace("Building SPU cache..."); + } + } + else if (showing_progress && pending_progress) + { + // Cover missing progress due to a race + g_progr_pdone += pending_progress.exchange(0); + } + } + + if (showing_progress && pending_progress) + { + // Cover missing progress due to a race + g_progr_pdone += pending_progress.exchange(0); + } + + return result; + }); + + u32 built_total = 0; + + // Join (implicitly) and print individual results + for (u32 i = 0; i < workers.size(); i++) + { + spu_log.notice("SPU Runtime: Worker %u built %u programs.", i + 1, workers[i]); + built_total += workers[i]; + } + + spu_log.notice("SPU Runtime: Workers built %u programs.", built_total); + + if (Emu.IsStopped()) + { + spu_log.error("SPU Runtime: Cache building aborted."); + return; + } + + if (fail_flag) + { + spu_log.fatal("SPU Runtime: Cache building failed (out of memory)."); + return; + } + + if ((g_cfg.core.spu_decoder == spu_decoder_type::asmjit || g_cfg.core.spu_decoder == spu_decoder_type::llvm) && !func_list.empty()) + { + spu_log.success("SPU Runtime: Built %u functions.", func_list.size()); + + if (g_cfg.core.spu_debug) + { + std::string dump; + dump.reserve(10'000'000); + + std::map, spu_program*> sorted; + + for (auto&& f : func_list) + { + // Interpret as a byte string + std::basic_string_view data = {reinterpret_cast(f.data.data()), f.data.size() * sizeof(u32)}; + + sorted[data] = &f; + } + + std::unordered_set depth_n; + + u32 n_max = 0; + + for (auto&& [bytes, f] : sorted) + { + { + sha1_context ctx; + u8 output[20]; + + sha1_starts(&ctx); + sha1_update(&ctx, bytes.data(), bytes.size()); + sha1_finish(&ctx, output); + fmt::append(dump, "\n\t[%s] ", fmt::base57(output)); + } + + u32 depth_m = 0; + + for (auto&& [data, f2] : sorted) + { + u32 depth = 0; + + if (f2 == f) + { + continue; + } + + for (u32 i = 0; i < bytes.size(); i++) + { + if (i < data.size() && data[i] == bytes[i]) + { + depth++; + } + else + { + break; + } + } + + depth_n.emplace(depth); + depth_m = std::max(depth, depth_m); + } + + fmt::append(dump, "c=%06d,d=%06d ", depth_n.size(), depth_m); + + bool sk = false; + + for (u32 i = 0; i < bytes.size(); i++) + { + if (depth_m == i) + { + dump += '|'; + sk = true; + } + + fmt::append(dump, "%02x", bytes[i]); + + if (i % 4 == 3) + { + if (sk) + { + sk = false; + } + else + { + dump += ' '; + } + + dump += ' '; + } + } + + fmt::append(dump, "\n\t%49s", ""); + + for (u32 i = 0; i < f->data.size(); i++) + { + fmt::append(dump, "%-10s", g_spu_iname.decode(std::bit_cast>(f->data[i]))); + } + + n_max = std::max(n_max, ::size32(depth_n)); + + depth_n.clear(); + } + + spu_log.notice("SPU Cache Dump (max_c=%d): %s", n_max, dump); + } + } + + // Initialize global cache instance + if (g_cfg.core.spu_cache && cache) + { + g_fxo->get() = std::move(cache); + } +} + +bool spu_program::operator==(const spu_program& rhs) const noexcept +{ + // TODO + return entry_point - lower_bound == rhs.entry_point - rhs.lower_bound && data == rhs.data; +} + +bool spu_program::operator<(const spu_program& rhs) const noexcept +{ + const u32 lhs_offs = (entry_point - lower_bound) / 4; + const u32 rhs_offs = (rhs.entry_point - rhs.lower_bound) / 4; + + // Select range for comparison + std::basic_string_view lhs_data(data.data() + lhs_offs, data.size() - lhs_offs); + std::basic_string_view rhs_data(rhs.data.data() + rhs_offs, rhs.data.size() - rhs_offs); + const auto cmp0 = lhs_data.compare(rhs_data); + + if (cmp0 < 0) + return true; + else if (cmp0 > 0) + return false; + + // Compare from address 0 to the point before the entry point (TODO: undesirable) + lhs_data = {data.data(), lhs_offs}; + rhs_data = {rhs.data.data(), rhs_offs}; + const auto cmp1 = lhs_data.compare(rhs_data); + + if (cmp1 < 0) + return true; + else if (cmp1 > 0) + return false; + + // TODO + return lhs_offs < rhs_offs; +} + +spu_runtime::spu_runtime() +{ + // Clear LLVM output + m_cache_path = rpcs3::cache::get_ppu_cache(); + + if (m_cache_path.empty()) + { + return; + } + + fs::create_dir(m_cache_path + "llvm/"); + fs::remove_all(m_cache_path + "llvm/", false); + + if (g_cfg.core.spu_debug) + { + fs::file(m_cache_path + "spu.log", fs::rewrite); + fs::file(m_cache_path + "spu-ir.log", fs::rewrite); + } +} + +spu_item* spu_runtime::add_empty(spu_program&& data) +{ + if (data.data.empty()) + { + return nullptr; + } + + // Store previous item if already added + spu_item* prev = nullptr; + + //Try to add item that doesn't exist yet + const auto ret = m_stuff[data.data[0] >> 12].push_if([&](spu_item& _new, spu_item& _old) + { + if (_new.data == _old.data) + { + prev = &_old; + return false; + } + + return true; + }, std::move(data)); + + if (ret) + { + return ret; + } + + return prev; +} + +spu_function_t spu_runtime::rebuild_ubertrampoline(u32 id_inst) +{ + // Prepare sorted list + static thread_local std::vector, spu_function_t>> m_flat_list; + + // Remember top position + auto stuff_it = ::at32(m_stuff, id_inst >> 12).begin(); + auto stuff_end = ::at32(m_stuff, id_inst >> 12).end(); + { + if (stuff_it->trampoline) + { + return stuff_it->trampoline; + } + + m_flat_list.clear(); + + for (auto it = stuff_it; it != stuff_end; ++it) + { + if (const auto ptr = it->compiled.load()) + { + std::basic_string_view range{it->data.data.data(), it->data.data.size()}; + range.remove_prefix((it->data.entry_point - it->data.lower_bound) / 4); + m_flat_list.emplace_back(range, ptr); + } + else + { + // Pull oneself deeper (TODO) + ++stuff_it; + } + } + } + + std::sort(m_flat_list.begin(), m_flat_list.end(), FN(x.first < y.first)); + + struct work + { + u32 size; + u16 from; + u16 level; + u8* rel32; + decltype(m_flat_list)::iterator beg; + decltype(m_flat_list)::iterator end; + }; + + // Scratch vector + static thread_local std::vector workload; + + // Generate a dispatcher (übertrampoline) + const auto beg = m_flat_list.begin(); + const auto _end = m_flat_list.end(); + const u32 size0 = ::size32(m_flat_list); + + auto result = beg->second; + + if (size0 != 1) + { +#if defined(ARCH_ARM64) + // Allocate some writable executable memory + u8* const wxptr = jit_runtime::alloc(size0 * 128 + 16, 16); + + if (!wxptr) + { + return nullptr; + } + + // Raw assembly pointer + u8* raw = wxptr; + + auto make_jump = [&](asmjit::arm::CondCode op, auto target) + { + // 36 bytes + // Fallback to dispatch if no target + const u64 taddr = target ? reinterpret_cast(target) : reinterpret_cast(tr_dispatch); + + // ldr x9, #16 -> ldr x9, taddr + *raw++ = 0x89; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x58; + + if (op == asmjit::arm::CondCode::kAlways) + { + // br x9 + *raw++ = 0x20; + *raw++ = 0x01; + *raw++ = 0x1F; + *raw++ = 0xD6; + + // nop + *raw++ = 0x1F; + *raw++ = 0x20; + *raw++ = 0x03; + *raw++ = 0xD5; + + // nop + *raw++ = 0x1F; + *raw++ = 0x20; + *raw++ = 0x03; + *raw++ = 0xD5; + } + else + { + // b.COND #8 -> b.COND do_branch + switch (op) + { + case asmjit::arm::CondCode::kUnsignedLT: + *raw++ = 0x43; + break; + case asmjit::arm::CondCode::kUnsignedGT: + *raw++ = 0x48; + break; + default: + asm("brk 0x42"); + } + + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x54; + + // b #16 -> b cont + *raw++ = 0x04; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x14; + + // do_branch: br x9 + *raw++ = 0x20; + *raw++ = 0x01; + *raw++ = 0x1f; + *raw++ = 0xD6; + } + + // taddr + std::memcpy(raw, &taddr, 8); + raw += 8; + + // cont: next instruction + }; +#elif defined(ARCH_X64) + // Allocate some writable executable memory + u8* const wxptr = jit_runtime::alloc(size0 * 22 + 14, 16); + + if (!wxptr) + { + return nullptr; + } + + // Raw assembly pointer + u8* raw = wxptr; + + // Write jump instruction with rel32 immediate + auto make_jump = [&](u8 op, auto target) + { + ensure(raw + 8 <= wxptr + size0 * 22 + 16); + + // Fallback to dispatch if no target + const u64 taddr = target ? reinterpret_cast(target) : reinterpret_cast(tr_dispatch); + + // Compute the distance + const s64 rel = taddr - reinterpret_cast(raw) - (op != 0xe9 ? 6 : 5); + + ensure(rel >= s32{smin} && rel <= s32{smax}); + + if (op != 0xe9) + { + // First jcc byte + *raw++ = 0x0f; + ensure((op >> 4) == 0x8); + } + + *raw++ = op; + + const s32 r32 = static_cast(rel); + + std::memcpy(raw, &r32, 4); + raw += 4; + }; +#endif + + workload.clear(); + workload.reserve(size0); + workload.emplace_back(); + workload.back().size = size0; + workload.back().level = 0; + workload.back().from = -1; + workload.back().rel32 = nullptr; + workload.back().beg = beg; + workload.back().end = _end; + + // LS address starting from PC is already loaded into rcx (see spu_runtime::tr_all) + + for (usz i = 0; i < workload.size(); i++) + { + // Get copy of the workload info + auto w = workload[i]; + + // Split range in two parts + auto it = w.beg; + auto it2 = w.beg; + u32 size1 = w.size / 2; + u32 size2 = w.size - size1; + std::advance(it2, w.size / 2); + + while (ensure(w.level < umax)) + { + it = it2; + size1 = w.size - size2; + + if (w.level >= w.beg->first.size()) + { + // Cannot split: smallest function is a prefix of bigger ones (TODO) + break; + } + + const u32 x1 = ::at32(w.beg->first, w.level); + + if (!x1) + { + // Cannot split: some functions contain holes at this level + w.level++; + + // Resort subrange starting from the new level + std::stable_sort(w.beg, w.end, [&](const auto& a, const auto& b) + { + std::basic_string_view lhs = a.first; + std::basic_string_view rhs = b.first; + + lhs.remove_prefix(w.level); + rhs.remove_prefix(w.level); + + return lhs < rhs; + }); + + continue; + } + + // Adjust ranges (forward) + while (it != w.end && x1 == ::at32(it->first, w.level)) + { + it++; + size1++; + } + + if (it == w.end) + { + // Cannot split: words are identical within the range at this level + w.level++; + } + else + { + size2 = w.size - size1; + break; + } + } + + if (w.rel32) + { +#if defined(ARCH_X64) + // Patch rel32 linking it to the current location if necessary + const s32 r32 = ::narrow(raw - w.rel32); + std::memcpy(w.rel32 - 4, &r32, 4); +#elif defined(ARCH_ARM64) + // Rewrite jump address + { + u64 raw64 = reinterpret_cast(raw); + memcpy(w.rel32 - 8, &raw64, 8); + } +#else +#error "Unimplemented" +#endif + } + + if (w.level >= w.beg->first.size() || w.level >= it->first.size()) + { + // If functions cannot be compared, assume smallest function + spu_log.error("Trampoline simplified at ??? (level=%u)", w.level); +#if defined(ARCH_X64) + make_jump(0xe9, w.beg->second); // jmp rel32 +#elif defined(ARCH_ARM64) + u64 branch_target = reinterpret_cast(w.beg->second); + make_jump(asmjit::arm::CondCode::kAlways, branch_target); +#else +#error "Unimplemented" +#endif + continue; + } + + // Value for comparison + const u32 x = ::at32(it->first, w.level); + + // Adjust ranges (backward) + while (it != m_flat_list.begin()) + { + it--; + + if (w.level >= it->first.size()) + { + it = m_flat_list.end(); + break; + } + + if (::at32(it->first, w.level) != x) + { + it++; + break; + } + + ensure(it != w.beg); + size1--; + size2++; + } + + if (it == m_flat_list.end()) + { + spu_log.error("Trampoline simplified (II) at ??? (level=%u)", w.level); +#if defined(ARCH_X64) + make_jump(0xe9, w.beg->second); // jmp rel32 +#elif defined(ARCH_ARM64) + u64 branch_target = reinterpret_cast(w.beg->second); + make_jump(asmjit::arm::CondCode::kAlways, branch_target); +#else +#error "Unimplemented" +#endif + continue; + } + + // Emit 32-bit comparison +#if defined(ARCH_X64) + ensure(raw + 12 <= wxptr + size0 * 22 + 16); // "Asm overflow" +#elif defined(ARCH_ARM64) + ensure(raw + (4 * 4) <= wxptr + size0 * 128 + 16); +#else +#error "Unimplemented" +#endif + + if (w.from != w.level) + { + // If necessary (level has advanced), emit load: mov eax, [rcx + addr] + const u32 cmp_lsa = w.level * 4u; +#if defined(ARCH_X64) + if (cmp_lsa < 0x80) + { + *raw++ = 0x8b; + *raw++ = 0x41; + *raw++ = ::narrow(cmp_lsa); + } + else + { + *raw++ = 0x8b; + *raw++ = 0x81; + std::memcpy(raw, &cmp_lsa, 4); + raw += 4; + } +#elif defined(ARCH_ARM64) + // ldr w9, #8 + *raw++ = 0x49; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x18; + + // b #8 + *raw++ = 0x02; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x14; + + // cmp_lsa + std::memcpy(raw, &cmp_lsa, 4); + raw += 4; + + // ldr w1, [x7, x9] + *raw++ = 0xE1; + *raw++ = 0x68; + *raw++ = 0x69; + *raw++ = 0xB8; +#else +#error "Unimplemented" +#endif + } + + // Emit comparison: cmp eax, imm32 +#if defined(ARCH_X64) + *raw++ = 0x3d; + std::memcpy(raw, &x, 4); + raw += 4; +#elif defined(ARCH_ARM64) + // ldr w9, #8 + *raw++ = 0x49; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x18; + + // b #8 + *raw++ = 0x02; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x14; + + // x + std::memcpy(raw, &x, 4); + raw += 4; + + // cmp w1, w9 + *raw++ = 0x3f; + *raw++ = 0x00; + *raw++ = 0x09; + *raw++ = 0x6B; +#else +#error "Unimplemented" +#endif + + // Low subrange target + if (size1 == 1) + { +#if defined(ARCH_X64) + make_jump(0x82, w.beg->second); // jb rel32 +#elif defined(ARCH_ARM64) + u64 branch_target = reinterpret_cast(w.beg->second); + make_jump(asmjit::arm::CondCode::kUnsignedLT, branch_target); +#else +#error "Unimplemented" +#endif + } + else + { +#if defined(ARCH_X64) + make_jump(0x82, raw); // jb rel32 (stub) +#elif defined(ARCH_ARM64) + make_jump(asmjit::arm::CondCode::kUnsignedLT, raw); +#else +#error "Unimplemented" +#endif + auto& to = workload.emplace_back(w); + to.end = it; + to.size = size1; + to.rel32 = raw; + to.from = w.level; + } + + // Second subrange target + if (size2 == 1) + { +#if defined(ARCH_X64) + make_jump(0xe9, it->second); // jmp rel32 +#elif defined(ARCH_ARM64) + u64 branch_target = reinterpret_cast(it->second); + make_jump(asmjit::arm::CondCode::kAlways, branch_target); +#else +#error "Unimplemented" +#endif + } + else + { + it2 = it; + + // Select additional midrange for equality comparison + while (it2 != w.end && ::at32(it2->first, w.level) == x) + { + size2--; + it2++; + } + + if (it2 != w.end) + { + // High subrange target + if (size2 == 1) + { +#if defined(ARCH_X64) + make_jump(0x87, it2->second); // ja rel32 +#elif defined(ARCH_ARM64) + u64 branch_target = reinterpret_cast(it2->second); + make_jump(asmjit::arm::CondCode::kUnsignedGT, branch_target); +#else +#throw "Unimplemented" +#endif + } + else + { +#if defined(ARCH_X64) + make_jump(0x87, raw); // ja rel32 (stub) +#elif defined(ARCH_ARM64) + make_jump(asmjit::arm::CondCode::kUnsignedGT, raw); +#else +#error "Unimplemented" +#endif + auto& to = workload.emplace_back(w); + to.beg = it2; + to.size = size2; + to.rel32 = raw; + to.from = w.level; + } + + const u32 size3 = w.size - size1 - size2; + + if (size3 == 1) + { +#if defined(ARCH_X64) + make_jump(0xe9, it->second); // jmp rel32 +#elif defined(ARCH_ARM64) + u64 branch_target = reinterpret_cast(it->second); + make_jump(asmjit::arm::CondCode::kAlways, branch_target); +#else +#error "Unimplemented" +#endif + } + else + { +#if defined(ARCH_X64) + make_jump(0xe9, raw); // jmp rel32 (stub) +#elif defined(ARCH_ARM64) + make_jump(asmjit::arm::CondCode::kAlways, raw); +#else +#error "Unimplemented" +#endif + auto& to = workload.emplace_back(w); + to.beg = it; + to.end = it2; + to.size = size3; + to.rel32 = raw; + to.from = w.level; + } + } + else + { +#if defined(ARCH_X64) + make_jump(0xe9, raw); // jmp rel32 (stub) +#elif defined(ARCH_ARM64) + make_jump(asmjit::arm::CondCode::kAlways, raw); +#else +#error "Unimplemented" +#endif + auto& to = workload.emplace_back(w); + to.beg = it; + to.size = w.size - size1; + to.rel32 = raw; + to.from = w.level; + } + } + } + + workload.clear(); + result = reinterpret_cast(reinterpret_cast(wxptr)); + + std::string fname; + fmt::append(fname, "__ub%u", m_flat_list.size()); + jit_announce(wxptr, raw - wxptr, fname); + } + + if (auto _old = stuff_it->trampoline.compare_and_swap(nullptr, result)) + { + return _old; + } + + // Install ubertrampoline + auto& insert_to = ::at32(*spu_runtime::g_dispatcher, id_inst >> 12); + + auto _old = insert_to.load(); + + do + { + // Make sure we are replacing an older ubertrampoline but not newer one + if (_old != tr_dispatch) + { + bool ok = false; + + for (auto it = stuff_it; it != stuff_end; ++it) + { + if (it->trampoline == _old) + { + ok = true; + break; + } + } + + if (!ok) + { + return result; + } + } + } + while (!insert_to.compare_exchange(_old, result)); + + return result; +} + +spu_function_t spu_runtime::find(const u32* ls, u32 addr) const +{ + const u32 index = ls[addr / 4] >> 12; + for (const auto& item : ::at32(m_stuff, index)) + { + if (const auto ptr = item.compiled.load()) + { + std::basic_string_view range{item.data.data.data(), item.data.data.size()}; + range.remove_prefix((item.data.entry_point - item.data.lower_bound) / 4); + + if (addr / 4 + range.size() > 0x10000) + { + continue; + } + + if (range.compare(0, range.size(), ls + addr / 4, range.size()) == 0) + { + return ptr; + } + } + } + + return nullptr; +} + +spu_function_t spu_runtime::make_branch_patchpoint(u16 data) const +{ +#if defined(ARCH_X64) + u8* const raw = jit_runtime::alloc(16, 16); + + if (!raw) + { + return nullptr; + } + + // Save address of the following jmp (GHC CC 3rd argument) + raw[0] = 0x4c; // lea r12, [rip+1] + raw[1] = 0x8d; + raw[2] = 0x25; + raw[3] = 0x01; + raw[4] = 0x00; + raw[5] = 0x00; + raw[6] = 0x00; + + raw[7] = 0x90; // nop + + // Jump to spu_recompiler_base::branch + raw[8] = 0xe9; + // Compute the distance + const s64 rel = reinterpret_cast(tr_branch) - reinterpret_cast(raw + 8) - 5; + std::memcpy(raw + 9, &rel, 4); + raw[13] = 0xcc; + raw[14] = data >> 8; + raw[15] = data & 0xff; + + return reinterpret_cast(raw); +#elif defined(ARCH_ARM64) +#if defined(__APPLE__) + pthread_jit_write_protect_np(false); +#endif + + u8* const patch_fn = ensure(jit_runtime::alloc(36, 16)); + u8* raw = patch_fn; + + // adr x21, #16 + *raw++ = 0x95; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x10; + + // nop x3 + for (int i = 0; i < 3; i++) + { + *raw++ = 0x1F; + *raw++ = 0x20; + *raw++ = 0x03; + *raw++ = 0xD5; + } + + // ldr x9, #8 + *raw++ = 0x49; + *raw++ = 0x00; + *raw++ = 0x00; + *raw++ = 0x58; + + // br x9 + *raw++ = 0x20; + *raw++ = 0x01; + *raw++ = 0x1F; + *raw++ = 0xD6; + + u64 branch_target = reinterpret_cast(tr_branch); + std::memcpy(raw, &branch_target, 8); + raw += 8; + + *raw++ = static_cast(data >> 8); + *raw++ = static_cast(data & 0xff); + +#if defined(__APPLE__) + pthread_jit_write_protect_np(true); +#endif + + // Flush all cache lines after potentially writing executable code + asm("ISB"); + asm("DSB ISH"); + + return reinterpret_cast(patch_fn); +#else +#error "Unimplemented" +#endif +} + +spu_recompiler_base::spu_recompiler_base() +{ +} + +spu_recompiler_base::~spu_recompiler_base() +{ +} + +void spu_recompiler_base::dispatch(spu_thread& spu, void*, u8* rip) +{ + // If code verification failed from a patched patchpoint, clear it with a dispatcher jump + if (rip) + { +#if defined(ARCH_X64) + const s64 rel = reinterpret_cast(spu_runtime::tr_all) - reinterpret_cast(rip - 8) - 5; + + union + { + u8 bytes[8]; + u64 result; + }; + + bytes[0] = 0xe9; // jmp rel32 + std::memcpy(bytes + 1, &rel, 4); + bytes[5] = 0x66; // lnop (2 bytes) + bytes[6] = 0x90; + bytes[7] = 0x90; + + atomic_storage::release(*reinterpret_cast(rip - 8), result); +#elif defined(ARCH_ARM64) + union + { + u8 bytes[16]; + u128 result; + }; + + // ldr x9, #8 + bytes[0] = 0x49; + bytes[1] = 0x00; + bytes[2] = 0x00; + bytes[3] = 0x58; + + // br x9 + bytes[4] = 0x20; + bytes[5] = 0x01; + bytes[6] = 0x1F; + bytes[7] = 0xD6; + + const u64 target = reinterpret_cast(spu_runtime::tr_all); + std::memcpy(bytes + 8, &target, 8); +#if defined(__APPLE__) + pthread_jit_write_protect_np(false); +#endif + atomic_storage::release(*reinterpret_cast(rip), result); +#if defined(__APPLE__) + pthread_jit_write_protect_np(true); +#endif + + // Flush all cache lines after potentially writing executable code + asm("ISB"); + asm("DSB ISH"); +#else +#error "Unimplemented" +#endif + } + + // Second attempt (recover from the recursion after repeated unsuccessful trampoline call) + if (spu.block_counter != spu.block_recover && &dispatch != ::at32(*spu_runtime::g_dispatcher, spu._ref>(spu.pc) >> 12)) + { + spu.block_recover = spu.block_counter; + return; + } + + spu.jit->init(); + + // Compile + if (spu._ref(spu.pc) == 0u) + { + spu_runtime::g_escape(&spu); + return; + } + + const auto func = spu.jit->compile(spu.jit->analyse(spu._ptr(0), spu.pc)); + + if (!func) + { + spu_log.fatal("[0x%05x] Compilation failed.", spu.pc); + return; + } + + // Diagnostic + if (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + const v128 _info = spu.stack_mirror[(spu.gpr[1]._u32[3] & 0x3fff0) >> 4]; + + if (_info._u64[0] + 1) + { + spu_log.trace("Called from 0x%x", _info._u32[2] - 4); + } + } +#if defined(__APPLE__) + pthread_jit_write_protect_np(true); +#endif + +#if defined(ARCH_ARM64) + // Flush all cache lines after potentially writing executable code + asm("ISB"); + asm("DSB ISH"); +#endif + spu_runtime::g_tail_escape(&spu, func, nullptr); +} + +void spu_recompiler_base::branch(spu_thread& spu, void*, u8* rip) +{ +#if defined(ARCH_X64) + if (const u32 ls_off = ((rip[6] << 8) | rip[7]) * 4) +#elif defined(ARCH_ARM64) + if (const u32 ls_off = ((rip[16] << 8) | rip[17]) * 4) // See branch_patchpoint `data` +#else +#error "Unimplemented" +#endif + { + spu_log.todo("Special branch patchpoint hit.\nPlease report to the developer (0x%05x).", ls_off); + } + + // Find function + const auto func = spu.jit->get_runtime().find(static_cast(spu._ptr(0)), spu.pc); + + if (!func) + { + return; + } + +#if defined(ARCH_X64) + // Overwrite jump to this function with jump to the compiled function + const s64 rel = reinterpret_cast(func) - reinterpret_cast(rip) - 5; + + union + { + u8 bytes[8]; + u64 result; + }; + + if (rel >= s32{smin} && rel <= s32{smax}) + { + const s64 rel8 = (rel + 5) - 2; + + if (rel8 >= s8{smin} && rel8 <= s8{smax}) + { + bytes[0] = 0xeb; // jmp rel8 + bytes[1] = static_cast(rel8); + std::memset(bytes + 2, 0xcc, 4); + } + else + { + bytes[0] = 0xe9; // jmp rel32 + std::memcpy(bytes + 1, &rel, 4); + bytes[5] = 0xcc; + } + + bytes[6] = rip[6]; + bytes[7] = rip[7]; + } + else + { + fmt::throw_exception("Impossible far jump: %p -> %p", rip, func); + } + + atomic_storage::release(*reinterpret_cast(rip), result); +#elif defined(ARCH_ARM64) + union + { + u8 bytes[16]; + u128 result; + }; + + // ldr x9, #8 + bytes[0] = 0x49; + bytes[1] = 0x00; + bytes[2] = 0x00; + bytes[3] = 0x58; + + // br x9 + bytes[4] = 0x20; + bytes[5] = 0x01; + bytes[6] = 0x1F; + bytes[7] = 0xD6; + + const u64 target = reinterpret_cast(func); + std::memcpy(bytes + 8, &target, 8); +#if defined(__APPLE__) + pthread_jit_write_protect_np(false); +#endif + atomic_storage::release(*reinterpret_cast(rip), result); +#if defined(__APPLE__) + pthread_jit_write_protect_np(true); +#endif + + // Flush all cache lines after potentially writing executable code + asm("ISB"); + asm("DSB ISH"); +#else +#error "Unimplemented" +#endif + + spu_runtime::g_tail_escape(&spu, func, rip); +} + +void spu_recompiler_base::old_interpreter(spu_thread& spu, void* ls, u8* /*rip*/) +{ + if (g_cfg.core.spu_decoder != spu_decoder_type::_static) + { + fmt::throw_exception("Invalid SPU decoder"); + } + + // Select opcode table + const auto& table = g_fxo->get(); + + // LS pointer + const auto base = static_cast(ls); + + while (true) + { + if (spu.state) [[unlikely]] + { + if (spu.check_state()) + break; + } + + const u32 op = *reinterpret_cast*>(base + spu.pc); + if (table.decode(op)(spu, {op})) + spu.pc += 4; + } +} + +std::vector spu_thread::discover_functions(u32 base_addr, std::span ls, bool is_known_addr, u32 /*entry*/) +{ + std::vector calls; + std::vector branches; + + calls.reserve(100); + + // Discover functions + // Use the most simple method: search for instructions that calls them + // And then filter invalid cases + // TODO: Does not detect jumptables or fixed-addr indirect calls + const v128 brasl_mask = is_known_addr ? v128::from32p(0x62u << 23) : v128::from32p(umax); + + for (u32 i = utils::align(base_addr, 0x10); i < std::min(base_addr + ls.size(), 0x3FFF0); i += 0x10) + { + // Search for BRSL LR and BRASL LR or BR + // TODO: BISL + const v128 inst = read_from_ptr>(ls.data(), i - base_addr); + const v128 cleared_i16 = gv_and32(inst, v128::from32p(utils::rol32(~0xffff, 7))); + const v128 eq_brsl = gv_eq32(cleared_i16, v128::from32p(0x66u << 23)); + const v128 eq_brasl = gv_eq32(cleared_i16, brasl_mask); + const v128 eq_br = gv_eq32(cleared_i16, v128::from32p(0x64u << 23)); + const v128 result = eq_brsl | eq_brasl; + + if (!gv_testz(result)) + { + for (u32 j = 0; j < 4; j++) + { + if (result.u32r[j]) + { + calls.push_back(i + j * 4); + } + } + } + + if (!gv_testz(eq_br)) + { + for (u32 j = 0; j < 4; j++) + { + if (eq_br.u32r[j]) + { + branches.push_back(i + j * 4); + } + } + } + } + + calls.erase(std::remove_if(calls.begin(), calls.end(), [&](u32 caller) + { + // Check the validity of both the callee code and the following caller code + return !is_exec_code(caller, ls, base_addr) || !is_exec_code(caller + 4, ls, base_addr); + }), calls.end()); + + branches.erase(std::remove_if(branches.begin(), branches.end(), [&](u32 caller) + { + // Check the validity of the callee code + return !is_exec_code(caller, ls, base_addr); + }), branches.end()); + + std::vector addrs; + + for (u32 addr : calls) + { + const spu_opcode_t op{read_from_ptr>(ls, addr - base_addr)}; + + const u32 func = op_branch_targets(addr, op)[0]; + + if (func == umax || addr + 4 == func || func == addr || std::count(addrs.begin(), addrs.end(), func)) + { + continue; + } + + addrs.push_back(func); + } + + for (u32 addr : branches) + { + const spu_opcode_t op{read_from_ptr>(ls, addr - base_addr)}; + + const u32 func = op_branch_targets(addr, op)[0]; + + if (func == umax || addr + 4 == func || func == addr || !addr) + { + continue; + } + + // Search for AI R1, +x or OR R3/4, Rx, 0 + // Reasoning: AI R1, +x means stack pointer restoration, branch after that is likely a tail call + // R3 and R4 are common function arguments because they are the first two + for (u32 back = addr - 4, it = 10; it && back >= base_addr && back < std::min(base_addr + ls.size(), 0x3FFF0); it--, back -= 4) + { + const spu_opcode_t test_op{read_from_ptr>(ls, back - base_addr)}; + const auto type = g_spu_itype.decode(test_op.opcode); + + if (type & spu_itype::branch) + { + break; + } + + bool is_tail = false; + + if (type == spu_itype::AI && test_op.rt == 1u && test_op.ra == 1u) + { + if (test_op.si10 <= 0) + { + break; + } + + is_tail = true; + } + else if (!(type & spu_itype::zregmod)) + { + const u32 op_rt = type & spu_itype::_quadrop ? +test_op.rt4 : +test_op.rt; + + if (op_rt >= 80u && (type != spu_itype::LQD || test_op.ra != 1u)) + { + // Modifying non-volatile registers, not a call (and not context restoration) + break; + } + + //is_tail = op_rt == 3u || op_rt == 4u; + } + + if (!is_tail) + { + continue; + } + + if (std::count(addrs.begin(), addrs.end(), func)) + { + break; + } + + addrs.push_back(func); + break; + } + } + + std::sort(addrs.begin(), addrs.end()); + + return addrs; +} + +spu_program spu_recompiler_base::analyse(const be_t* ls, u32 entry_point, std::map>* out_target_list) +{ + // Result: addr + raw instruction data + spu_program result; + result.data.reserve(10000); + result.entry_point = entry_point; + result.lower_bound = entry_point; + + // Initialize block entries + m_block_info.reset(); + m_block_info.set(entry_point / 4); + m_entry_info.reset(); + m_entry_info.set(entry_point / 4); + m_ret_info.reset(); + + // Simple block entry workload list + workload.clear(); + workload.push_back(entry_point); + + std::memset(m_regmod.data(), 0xff, sizeof(m_regmod)); + std::memset(m_use_ra.data(), 0xff, sizeof(m_use_ra)); + std::memset(m_use_rb.data(), 0xff, sizeof(m_use_rb)); + std::memset(m_use_rc.data(), 0xff, sizeof(m_use_rc)); + m_targets.clear(); + m_preds.clear(); + m_preds[entry_point]; + m_bbs.clear(); + m_chunks.clear(); + m_funcs.clear(); + + // Value flags (TODO: only is_const is implemented) + enum class vf : u32 + { + is_const, + is_mask, + is_rel, + + __bitset_enum_max + }; + + // Weak constant propagation context (for guessing branch targets) + std::array, 128> vflags{}; + + // Associated constant values for 32-bit preferred slot + std::array values; + + // SYNC instruction found + bool sync = false; + + u32 hbr_loc = 0; + u32 hbr_tg = -1; + + // Result bounds + u32 lsa = entry_point; + u32 limit = 0x40000; + + if (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + } + + for (u32 wi = 0, wa = workload[0]; wi < workload.size();) + { + const auto next_block = [&] + { + // Reset value information + vflags.fill({}); + sync = false; + hbr_loc = 0; + hbr_tg = -1; + wi++; + + if (wi < workload.size()) + { + wa = workload[wi]; + } + }; + + const u32 pos = wa; + + const auto add_block = [&](u32 target) + { + // Validate new target (TODO) + if (target >= lsa && target < limit) + { + // Check for redundancy + if (!m_block_info[target / 4]) + { + m_block_info[target / 4] = true; + workload.push_back(target); + } + + // Add predecessor + if (m_preds[target].find_first_of(pos) + 1 == 0) + { + m_preds[target].push_back(pos); + } + } + }; + + if (pos < lsa || pos >= limit) + { + // Don't analyse if already beyond the limit + next_block(); + continue; + } + + const u32 data = ls[pos / 4]; + const auto op = spu_opcode_t{data}; + + wa += 4; + + m_targets.erase(pos); + + // Fill register access info + if (auto iflags = g_spu_iflag.decode(data)) + { + if (+iflags & +spu_iflag::use_ra) + m_use_ra[pos / 4] = op.ra; + if (+iflags & +spu_iflag::use_rb) + m_use_rb[pos / 4] = op.rb; + if (+iflags & +spu_iflag::use_rc) + m_use_rc[pos / 4] = op.rc; + } + + // Analyse instruction + switch (const auto type = g_spu_itype.decode(data)) + { + case spu_itype::UNK: + case spu_itype::DFCEQ: + case spu_itype::DFCMEQ: + case spu_itype::DFCGT: + case spu_itype::DFCMGT: + case spu_itype::DFTSV: + { + // Stop before invalid instructions (TODO) + next_block(); + continue; + } + + case spu_itype::SYNC: + case spu_itype::STOP: + case spu_itype::STOPD: + { + if (data == 0) + { + // Stop before null data + next_block(); + continue; + } + + if (g_cfg.core.spu_block_size == spu_block_size_type::safe) + { + // Stop on special instructions (TODO) + m_targets[pos]; + next_block(); + break; + } + + if (type == spu_itype::SYNC) + { + // Remember + sync = true; + } + + break; + } + + case spu_itype::IRET: + { + if (op.d && op.e) + { + spu_log.error("[0x%x] Invalid interrupt flags (DE)", pos); + } + + m_targets[pos]; + next_block(); + break; + } + + case spu_itype::BI: + case spu_itype::BISL: + case spu_itype::BISLED: + case spu_itype::BIZ: + case spu_itype::BINZ: + case spu_itype::BIHZ: + case spu_itype::BIHNZ: + { + if (op.d && op.e) + { + spu_log.error("[0x%x] Invalid interrupt flags (DE)", pos); + } + + const auto af = vflags[op.ra]; + const auto av = values[op.ra]; + const bool sl = type == spu_itype::BISL || type == spu_itype::BISLED; + + if (sl) + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = +vf::is_const; + values[op.rt] = pos + 4; + } + + if (af & vf::is_const) + { + const u32 target = spu_branch_target(av); + + spu_log.warning("[0x%x] At 0x%x: indirect branch to 0x%x%s", entry_point, pos, target, op.d ? " (D)" : op.e ? " (E)" : ""); + + if (type == spu_itype::BI && target == pos + 4 && op.d) + { + // Disable interrupts idiom + break; + } + + m_targets[pos].push_back(target); + + if (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + if (sync) + { + spu_log.notice("[0x%x] At 0x%x: ignoring %scall to 0x%x (SYNC)", entry_point, pos, sl ? "" : "tail ", target); + + if (target > entry_point) + { + limit = std::min(limit, target); + } + } + else + { + m_entry_info[target / 4] = true; + add_block(target); + } + } + else if (target > entry_point) + { + limit = std::min(limit, target); + } + + if (sl && g_cfg.core.spu_block_size != spu_block_size_type::safe) + { + m_ret_info[pos / 4 + 1] = true; + m_entry_info[pos / 4 + 1] = true; + m_targets[pos].push_back(pos + 4); + add_block(pos + 4); + } + } + else if (type == spu_itype::BI && g_cfg.core.spu_block_size != spu_block_size_type::safe && !op.d && !op.e && !sync) + { + // Analyse jump table (TODO) + std::basic_string jt_abs; + std::basic_string jt_rel; + const u32 start = pos + 4; + u64 dabs = 0; + u64 drel = 0; + + for (u32 i = start; i < limit; i += 4) + { + const u32 target = ls[i / 4]; + + if (target == 0 || target % 4) + { + // Address cannot be misaligned: abort + break; + } + + if (target >= lsa && target < 0x40000) + { + // Possible jump table entry (absolute) + jt_abs.push_back(target); + } + + if (target + start >= lsa && target + start < 0x40000) + { + // Possible jump table entry (relative) + jt_rel.push_back(target + start); + } + + if (std::max(jt_abs.size(), jt_rel.size()) * 4 + start <= i) + { + // Neither type of jump table completes + jt_abs.clear(); + jt_rel.clear(); + break; + } + } + + // Choose position after the jt as an anchor and compute the average distance + for (u32 target : jt_abs) + { + dabs += std::abs(static_cast(target - start - jt_abs.size() * 4)); + } + + for (u32 target : jt_rel) + { + drel += std::abs(static_cast(target - start - jt_rel.size() * 4)); + } + + // Add detected jump table blocks + if (jt_abs.size() >= 3 || jt_rel.size() >= 3) + { + if (jt_abs.size() == jt_rel.size()) + { + if (dabs < drel) + { + jt_rel.clear(); + } + + if (dabs > drel) + { + jt_abs.clear(); + } + + ensure(jt_abs.size() != jt_rel.size()); + } + + if (jt_abs.size() >= jt_rel.size()) + { + const u32 new_size = (start - lsa) / 4 + ::size32(jt_abs); + + if (result.data.size() < new_size) + { + result.data.resize(new_size); + } + + for (u32 i = 0; i < jt_abs.size(); i++) + { + add_block(jt_abs[i]); + result.data[(start - lsa) / 4 + i] = std::bit_cast>(jt_abs[i]); + m_targets[start + i * 4]; + } + + m_targets.emplace(pos, std::move(jt_abs)); + } + + if (jt_rel.size() >= jt_abs.size()) + { + const u32 new_size = (start - lsa) / 4 + ::size32(jt_rel); + + if (result.data.size() < new_size) + { + result.data.resize(new_size); + } + + for (u32 i = 0; i < jt_rel.size(); i++) + { + add_block(jt_rel[i]); + result.data[(start - lsa) / 4 + i] = std::bit_cast>(jt_rel[i] - start); + m_targets[start + i * 4]; + } + + m_targets.emplace(pos, std::move(jt_rel)); + } + } + else if (start + 12 * 4 < limit && + ls[start / 4 + 0] == 0x1ce00408u && + ls[start / 4 + 1] == 0x24000389u && + ls[start / 4 + 2] == 0x24004809u && + ls[start / 4 + 3] == 0x24008809u && + ls[start / 4 + 4] == 0x2400c809u && + ls[start / 4 + 5] == 0x24010809u && + ls[start / 4 + 6] == 0x24014809u && + ls[start / 4 + 7] == 0x24018809u && + ls[start / 4 + 8] == 0x1c200807u && + ls[start / 4 + 9] == 0x2401c809u) + { + spu_log.warning("[0x%x] Pattern 1 detected (hbr=0x%x:0x%x)", pos, hbr_loc, hbr_tg); + + // Add 8 targets (TODO) + for (u32 addr = start + 4; addr < start + 36; addr += 4) + { + m_targets[pos].push_back(addr); + add_block(addr); + } + } + else if (hbr_loc > start && hbr_loc < limit && hbr_tg == start) + { + spu_log.warning("[0x%x] No patterns detected (hbr=0x%x:0x%x)", pos, hbr_loc, hbr_tg); + } + } + else if (type == spu_itype::BI && sync) + { + spu_log.notice("[0x%x] At 0x%x: ignoring indirect branch (SYNC)", entry_point, pos); + } + + if (type == spu_itype::BI || sl) + { + if (type == spu_itype::BI || g_cfg.core.spu_block_size == spu_block_size_type::safe) + { + m_targets[pos]; + } + else + { + m_ret_info[pos / 4 + 1] = true; + m_entry_info[pos / 4 + 1] = true; + m_targets[pos].push_back(pos + 4); + add_block(pos + 4); + } + } + else + { + m_targets[pos].push_back(pos + 4); + add_block(pos + 4); + } + + next_block(); + break; + } + + case spu_itype::BRSL: + case spu_itype::BRASL: + { + const u32 target = spu_branch_target(type == spu_itype::BRASL ? 0 : pos, op.i16); + + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = +vf::is_const; + values[op.rt] = pos + 4; + + if (type == spu_itype::BRSL && target == pos + 4) + { + // Get next instruction address idiom + break; + } + + m_targets[pos].push_back(target); + + if (g_cfg.core.spu_block_size != spu_block_size_type::safe) + { + m_ret_info[pos / 4 + 1] = true; + m_entry_info[pos / 4 + 1] = true; + m_targets[pos].push_back(pos + 4); + add_block(pos + 4); + } + + if (g_cfg.core.spu_block_size == spu_block_size_type::giga && !sync) + { + m_entry_info[target / 4] = true; + add_block(target); + } + else + { + if (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + spu_log.notice("[0x%x] At 0x%x: ignoring fixed call to 0x%x (SYNC)", entry_point, pos, target); + } + + if (target > entry_point) + { + limit = std::min(limit, target); + } + } + + next_block(); + break; + } + + case spu_itype::BRA: + { + const u32 target = spu_branch_target(0, op.i16); + + if (g_cfg.core.spu_block_size == spu_block_size_type::giga && !sync) + { + m_entry_info[target / 4] = true; + add_block(target); + } + else + { + if (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + spu_log.notice("[0x%x] At 0x%x: ignoring fixed tail call to 0x%x (SYNC)", entry_point, pos, target); + } + + if (target > entry_point) + { + limit = std::min(limit, target); + } + } + + next_block(); + break; + } + + case spu_itype::BR: + case spu_itype::BRZ: + case spu_itype::BRNZ: + case spu_itype::BRHZ: + case spu_itype::BRHNZ: + { + const u32 target = spu_branch_target(pos, op.i16); + + if (target == pos + 4) + { + // Nop + break; + } + + m_targets[pos].push_back(target); + add_block(target); + + if (type != spu_itype::BR) + { + m_targets[pos].push_back(pos + 4); + add_block(pos + 4); + } + + next_block(); + break; + } + + case spu_itype::DSYNC: + case spu_itype::HEQ: + case spu_itype::HEQI: + case spu_itype::HGT: + case spu_itype::HGTI: + case spu_itype::HLGT: + case spu_itype::HLGTI: + case spu_itype::LNOP: + case spu_itype::NOP: + case spu_itype::MTSPR: + case spu_itype::FSCRWR: + case spu_itype::STQA: + case spu_itype::STQD: + case spu_itype::STQR: + case spu_itype::STQX: + { + // Do nothing + break; + } + + case spu_itype::WRCH: + { + switch (op.ra) + { + case MFC_EAL: + { + m_regmod[pos / 4] = s_reg_mfc_eal; + break; + } + case MFC_LSA: + { + m_regmod[pos / 4] = s_reg_mfc_lsa; + break; + } + case MFC_TagID: + { + m_regmod[pos / 4] = s_reg_mfc_tag; + break; + } + case MFC_Size: + { + m_regmod[pos / 4] = s_reg_mfc_size; + break; + } + case MFC_Cmd: + { + m_use_rb[pos / 4] = s_reg_mfc_eal; + break; + } + default: break; + } + + break; + } + + case spu_itype::LQA: + case spu_itype::LQD: + case spu_itype::LQR: + case spu_itype::LQX: + { + // Unconst + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = {}; + break; + } + + case spu_itype::HBR: + { + hbr_loc = spu_branch_target(pos, op.roh << 7 | op.rt); + hbr_tg = vflags[op.ra] & vf::is_const && !op.c ? values[op.ra] & 0x3fffc : -1; + break; + } + + case spu_itype::HBRA: + { + hbr_loc = spu_branch_target(pos, op.r0h << 7 | op.rt); + hbr_tg = spu_branch_target(0x0, op.i16); + break; + } + + case spu_itype::HBRR: + { + hbr_loc = spu_branch_target(pos, op.r0h << 7 | op.rt); + hbr_tg = spu_branch_target(pos, op.i16); + break; + } + + case spu_itype::IL: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = +vf::is_const; + values[op.rt] = op.si16; + break; + } + case spu_itype::ILA: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = +vf::is_const; + values[op.rt] = op.i18; + break; + } + case spu_itype::ILH: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = +vf::is_const; + values[op.rt] = op.i16 << 16 | op.i16; + break; + } + case spu_itype::ILHU: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = +vf::is_const; + values[op.rt] = op.i16 << 16; + break; + } + case spu_itype::IOHL: + { + m_regmod[pos / 4] = op.rt; + values[op.rt] = values[op.rt] | op.i16; + break; + } + case spu_itype::ORI: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = vflags[op.ra] & vf::is_const; + values[op.rt] = values[op.ra] | op.si10; + break; + } + case spu_itype::OR: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = vflags[op.ra] & vflags[op.rb] & vf::is_const; + values[op.rt] = values[op.ra] | values[op.rb]; + break; + } + case spu_itype::ANDI: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = vflags[op.ra] & vf::is_const; + values[op.rt] = values[op.ra] & op.si10; + break; + } + case spu_itype::AND: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = vflags[op.ra] & vflags[op.rb] & vf::is_const; + values[op.rt] = values[op.ra] & values[op.rb]; + break; + } + case spu_itype::AI: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = vflags[op.ra] & vf::is_const; + values[op.rt] = values[op.ra] + op.si10; + break; + } + case spu_itype::A: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = vflags[op.ra] & vflags[op.rb] & vf::is_const; + values[op.rt] = values[op.ra] + values[op.rb]; + break; + } + case spu_itype::SFI: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = vflags[op.ra] & vf::is_const; + values[op.rt] = op.si10 - values[op.ra]; + break; + } + case spu_itype::SF: + { + m_regmod[pos / 4] = op.rt; + vflags[op.rt] = vflags[op.ra] & vflags[op.rb] & vf::is_const; + values[op.rt] = values[op.rb] - values[op.ra]; + break; + } + case spu_itype::ROTMI: + { + m_regmod[pos / 4] = op.rt; + + if ((0 - op.i7) & 0x20) + { + vflags[op.rt] = +vf::is_const; + values[op.rt] = 0; + break; + } + + vflags[op.rt] = vflags[op.ra] & vf::is_const; + values[op.rt] = values[op.ra] >> ((0 - op.i7) & 0x1f); + break; + } + case spu_itype::SHLI: + { + m_regmod[pos / 4] = op.rt; + + if (op.i7 & 0x20) + { + vflags[op.rt] = +vf::is_const; + values[op.rt] = 0; + break; + } + + vflags[op.rt] = vflags[op.ra] & vf::is_const; + values[op.rt] = values[op.ra] << (op.i7 & 0x1f); + break; + } + + default: + { + // Unconst + const u32 op_rt = type & spu_itype::_quadrop ? +op.rt4 : +op.rt; + m_regmod[pos / 4] = op_rt; + vflags[op_rt] = {}; + break; + } + } + + // Insert raw instruction value + const u32 new_size = (pos - lsa) / 4; + + if (result.data.size() <= new_size) + { + if (result.data.size() < new_size) + { + result.data.resize(new_size); + } + + result.data.emplace_back(std::bit_cast>(data)); + } + else if (u32& raw_val = result.data[new_size]) + { + ensure(raw_val == std::bit_cast>(data)); + } + else + { + raw_val = std::bit_cast>(data); + } + } + + while (lsa > 0 || limit < 0x40000) + { + const u32 initial_size = ::size32(result.data); + + // Check unreachable blocks + limit = std::min(limit, lsa + initial_size * 4); + + for (auto& pair : m_preds) + { + bool reachable = false; + + if (pair.first >= limit) + { + continue; + } + + // All (direct and indirect) predecessors to check + std::basic_string workload; + + // Bit array used to deduplicate workload list + workload.push_back(pair.first); + m_bits[pair.first / 4] = true; + + for (usz i = 0; !reachable && i < workload.size(); i++) + { + for (u32 j = workload[i];; j -= 4) + { + // Go backward from an address until the entry point is reached + if (j == entry_point) + { + reachable = true; + break; + } + + const auto found = m_preds.find(j); + + bool had_fallthrough = false; + + if (found != m_preds.end()) + { + for (u32 new_pred : found->second) + { + // Check whether the predecessor is previous instruction + if (new_pred == j - 4) + { + had_fallthrough = true; + continue; + } + + // Check whether in range and not already added + if (new_pred >= lsa && new_pred < limit && !m_bits[new_pred / 4]) + { + workload.push_back(new_pred); + m_bits[new_pred / 4] = true; + } + } + } + + // Check for possible fallthrough predecessor + if (!had_fallthrough) + { + if (::at32(result.data, (j - lsa) / 4 - 1) == 0 || m_targets.count(j - 4)) + { + break; + } + } + + if (i == 0) + { + // TODO + } + } + } + + for (u32 pred : workload) + { + m_bits[pred / 4] = false; + } + + if (!reachable && pair.first < limit) + { + limit = pair.first; + } + } + + result.data.resize((limit - lsa) / 4); + + // Check holes in safe mode (TODO) + u32 valid_size = 0; + + for (u32 i = 0; i < result.data.size(); i++) + { + if (result.data[i] == 0) + { + const u32 pos = lsa + i * 4; + const u32 data = ls[pos / 4]; + + // Allow only NOP or LNOP instructions in holes + if (data == 0x200000 || (data & 0xffffff80) == 0x40200000) + { + continue; + } + + if (g_cfg.core.spu_block_size != spu_block_size_type::giga) + { + result.data.resize(valid_size); + break; + } + } + else + { + valid_size = i + 1; + } + } + + // Even if NOP or LNOP, should be removed at the end + result.data.resize(valid_size); + + // Repeat if blocks were removed + if (result.data.size() == initial_size) + { + break; + } + } + + limit = std::min(limit, lsa + ::size32(result.data) * 4); + + // Cleanup block info + for (u32 i = 0; i < workload.size(); i++) + { + const u32 addr = workload[i]; + + if (addr < lsa || addr >= limit || !result.data[(addr - lsa) / 4]) + { + m_block_info[addr / 4] = false; + m_entry_info[addr / 4] = false; + m_ret_info[addr / 4] = false; + m_preds.erase(addr); + } + } + + // Complete m_preds and associated m_targets for adjacent blocks + for (auto it = m_preds.begin(); it != m_preds.end();) + { + if (it->first < lsa || it->first >= limit) + { + it = m_preds.erase(it); + continue; + } + + // Erase impossible predecessors + const auto new_end = std::remove_if(it->second.begin(), it->second.end(), [&](u32 addr) + { + return addr < lsa || addr >= limit; + }); + + it->second.erase(new_end, it->second.end()); + + // Don't add fallthrough target if all predecessors are removed + if (it->second.empty() && !m_entry_info[it->first / 4]) + { + // If not an entry point, remove the block completely + m_block_info[it->first / 4] = false; + it = m_preds.erase(it); + continue; + } + + // Previous instruction address + const u32 prev = (it->first - 4) & 0x3fffc; + + // TODO: check the correctness + if (m_targets.count(prev) == 0 && prev >= lsa && prev < limit && result.data[(prev - lsa) / 4]) + { + // Add target and the predecessor + m_targets[prev].push_back(it->first); + it->second.push_back(prev); + } + + it++; + } + + if (out_target_list) + { + out_target_list->insert(m_targets.begin(), m_targets.end()); + } + + // Remove unnecessary target lists + for (auto it = m_targets.begin(); it != m_targets.end();) + { + if (it->first < lsa || it->first >= limit) + { + it = m_targets.erase(it); + continue; + } + + it++; + } + + // Fill holes which contain only NOP and LNOP instructions (TODO: compile) + for (u32 i = 0, nnop = 0, vsize = 0; i <= result.data.size(); i++) + { + if (i >= result.data.size() || result.data[i]) + { + if (nnop && nnop == i - vsize) + { + // Write only complete NOP sequence + for (u32 j = vsize; j < i; j++) + { + result.data[j] = std::bit_cast>(ls[lsa / 4 + j]); + } + } + + nnop = 0; + vsize = i + 1; + } + else + { + const u32 pos = lsa + i * 4; + const u32 data = ls[pos / 4]; + + if (data == 0x200000 || (data & 0xffffff80) == 0x40200000) + { + nnop++; + } + } + } + + // Fill block info + for (auto& pred : m_preds) + { + auto& block = m_bbs[pred.first]; + + // Copy predeccessors (wrong at this point, needs a fixup later) + block.preds = pred.second; + + // Fill register usage info + for (u32 ia = pred.first; ia < limit; ia += 4) + { + block.size++; + + // Decode instruction + const spu_opcode_t op{std::bit_cast>(result.data[(ia - lsa) / 4])}; + + const auto type = g_spu_itype.decode(op.opcode); + + u8 reg_save = 255; + + if (type == spu_itype::STQD && op.ra == s_reg_sp && !block.reg_mod[op.rt] && !block.reg_use[op.rt]) + { + // Register saved onto the stack before use + block.reg_save_dom[op.rt] = true; + + reg_save = op.rt; + } + + for (auto* _use : {&m_use_ra, &m_use_rb, &m_use_rc}) + { + if (u8 reg = (*_use)[ia / 4]; reg < s_reg_max) + { + // Register reg use only if it happens before reg mod + if (!block.reg_mod[reg]) + { + block.reg_use.set(reg); + + if (reg_save != reg && block.reg_save_dom[reg]) + { + // Register is still used after saving; probably not eligible for optimization + block.reg_save_dom[reg] = false; + } + } + } + } + + if (m_use_rb[ia / 4] == s_reg_mfc_eal) + { + // Expand MFC_Cmd reg use + for (u8 reg : {s_reg_mfc_lsa, s_reg_mfc_tag, s_reg_mfc_size}) + { + if (!block.reg_mod[reg]) + block.reg_use.set(reg); + } + } + + // Register reg modification + if (u8 reg = m_regmod[ia / 4]; reg < s_reg_max) + { + block.reg_mod.set(reg); + block.reg_mod_xf.set(reg, type & spu_itype::xfloat); + + if (type == spu_itype::SELB && (block.reg_mod_xf[op.ra] || block.reg_mod_xf[op.rb])) + block.reg_mod_xf.set(reg); + + // Possible post-dominating register load + if (type == spu_itype::LQD && op.ra == s_reg_sp) + block.reg_load_mod[reg] = ia + 1; + else + block.reg_load_mod[reg] = 0; + } + + // Find targets (also means end of the block) + const auto tfound = m_targets.find(ia); + + if (tfound != m_targets.end()) + { + // Copy targets + block.targets = tfound->second; + + // Assume that the call reads and modifies all volatile registers (TODO) + bool is_call = false; + bool is_tail = false; + switch (type) + { + case spu_itype::BRSL: + is_call = spu_branch_target(ia, op.i16) != ia + 4; + break; + case spu_itype::BRASL: + is_call = spu_branch_target(0, op.i16) != ia + 4; + break; + case spu_itype::BRA: + is_call = true; + is_tail = true; + break; + case spu_itype::BISL: + case spu_itype::BISLED: + is_call = true; + break; + default: + break; + } + + if (is_call) + { + for (u32 i = 0; i < s_reg_max; ++i) + { + if (i == s_reg_lr || (i >= 2 && i < s_reg_80) || i > s_reg_127) + { + if (!block.reg_mod[i]) + block.reg_use.set(i); + + if (!is_tail) + { + block.reg_mod.set(i); + block.reg_mod_xf[i] = false; + } + } + } + } + + break; + } + } + } + + // Fixup block predeccessors to point to basic blocks, not last instructions + for (auto& bb : m_bbs) + { + const u32 addr = bb.first; + + for (u32& pred : bb.second.preds) + { + pred = std::prev(m_bbs.upper_bound(pred))->first; + } + + if (m_entry_info[addr / 4] && g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + // Register empty chunk + m_chunks.push_back(addr); + + // Register function if necessary + if (!m_ret_info[addr / 4]) + { + m_funcs[addr]; + } + } + } + + // Ensure there is a function at the lowest address + if (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + if (auto emp = m_funcs.try_emplace(m_bbs.begin()->first); emp.second) + { + const u32 addr = emp.first->first; + spu_log.error("[0x%05x] Fixed first function at 0x%05x", entry_point, addr); + m_entry_info[addr / 4] = true; + m_ret_info[addr / 4] = false; + } + } + + // Split functions + while (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + bool need_repeat = false; + + u32 start = 0; + u32 limit = 0x40000; + + // Walk block list in ascending order + for (auto& block : m_bbs) + { + const u32 addr = block.first; + + if (m_entry_info[addr / 4] && !m_ret_info[addr / 4]) + { + const auto upper = m_funcs.upper_bound(addr); + start = addr; + limit = upper == m_funcs.end() ? 0x40000 : upper->first; + } + + // Find targets that exceed [start; limit) range and make new functions from them + for (u32 target : block.second.targets) + { + const auto tfound = m_bbs.find(target); + + if (tfound == m_bbs.end()) + { + continue; + } + + if (target < start || target >= limit) + { + if (!m_entry_info[target / 4] || m_ret_info[target / 4]) + { + // Create new function entry (likely a tail call) + m_entry_info[target / 4] = true; + + m_ret_info[target / 4] = false; + + m_funcs.try_emplace(target); + + if (target < limit) + { + need_repeat = true; + } + } + } + } + + block.second.func = start; + } + + if (!need_repeat) + { + break; + } + } + + // Fill entry map + while (true) + { + workload.clear(); + workload.push_back(entry_point); + ensure(m_bbs.count(entry_point)); + + std::basic_string new_entries; + + for (u32 wi = 0; wi < workload.size(); wi++) + { + const u32 addr = workload[wi]; + auto& block = ::at32(m_bbs, addr); + const u32 _new = block.chunk; + + if (!m_entry_info[addr / 4]) + { + // Check block predecessors + for (u32 pred : block.preds) + { + const u32 _old = ::at32(m_bbs, pred).chunk; + + if (_old < 0x40000 && _old != _new) + { + // If block has multiple 'entry' points, it becomes an entry point itself + new_entries.push_back(addr); + } + } + } + + // Update chunk address + block.chunk = m_entry_info[addr / 4] ? addr : _new; + + // Process block targets + for (u32 target : block.targets) + { + const auto tfound = m_bbs.find(target); + + if (tfound == m_bbs.end()) + { + continue; + } + + auto& tb = tfound->second; + + const u32 value = m_entry_info[target / 4] ? target : block.chunk; + + if (u32& tval = tb.chunk; tval < 0x40000) + { + // TODO: fix condition + if (tval != value && !m_entry_info[target / 4]) + { + new_entries.push_back(target); + } + } + else + { + tval = value; + workload.emplace_back(target); + } + } + } + + if (new_entries.empty()) + { + break; + } + + for (u32 entry : new_entries) + { + m_entry_info[entry / 4] = true; + + // Acknowledge artificial (reversible) chunk entry point + m_ret_info[entry / 4] = true; + } + + for (auto& bb : m_bbs) + { + // Reset chunk info + bb.second.chunk = 0x40000; + } + } + + workload.clear(); + workload.push_back(entry_point); + + // Fill workload adding targets + for (u32 wi = 0; wi < workload.size(); wi++) + { + const u32 addr = workload[wi]; + auto& block = ::at32(m_bbs, addr); + block.analysed = true; + + for (u32 target : block.targets) + { + const auto tfound = m_bbs.find(target); + + if (tfound == m_bbs.end()) + { + continue; + } + + auto& tb = tfound->second; + + if (!tb.analysed) + { + workload.push_back(target); + tb.analysed = true; + } + + // Limited xfloat hint propagation (possibly TODO) + if (tb.chunk == block.chunk) + { + tb.reg_maybe_xf &= block.reg_mod_xf; + } + else + { + tb.reg_maybe_xf.reset(); + } + } + + block.reg_origin.fill(0x80000000); + block.reg_origin_abs.fill(0x80000000); + } + + // Fill register origin info + while (true) + { + bool must_repeat = false; + + for (u32 wi = 0; wi < workload.size(); wi++) + { + const u32 addr = workload[wi]; + auto& block = ::at32(m_bbs, addr); + + // Initialize entry point with default value: unknown origin (requires load) + if (m_entry_info[addr / 4]) + { + for (u32 i = 0; i < s_reg_max; i++) + { + if (block.reg_origin[i] == 0x80000000) + block.reg_origin[i] = 0x40000; + } + } + + if (g_cfg.core.spu_block_size == spu_block_size_type::giga && m_entry_info[addr / 4] && !m_ret_info[addr / 4]) + { + for (u32 i = 0; i < s_reg_max; i++) + { + if (block.reg_origin_abs[i] == 0x80000000) + block.reg_origin_abs[i] = 0x40000; + else if (block.reg_origin_abs[i] + 1 == 0) + block.reg_origin_abs[i] = -2; + } + } + + for (u32 target : block.targets) + { + const auto tfound = m_bbs.find(target); + + if (tfound == m_bbs.end()) + { + continue; + } + + auto& tb = tfound->second; + + for (u32 i = 0; i < s_reg_max; i++) + { + if (tb.chunk == block.chunk && tb.reg_origin[i] + 1) + { + const u32 expected = block.reg_mod[i] ? addr : block.reg_origin[i]; + + if (tb.reg_origin[i] == 0x80000000) + { + tb.reg_origin[i] = expected; + } + else if (tb.reg_origin[i] != expected) + { + // Set -1 if multiple origins merged (requires PHI node) + tb.reg_origin[i] = -1; + + must_repeat |= !tb.targets.empty(); + } + } + + if (g_cfg.core.spu_block_size == spu_block_size_type::giga && tb.func == block.func && tb.reg_origin_abs[i] + 2) + { + const u32 expected = block.reg_mod[i] ? addr : block.reg_origin_abs[i]; + + if (tb.reg_origin_abs[i] == 0x80000000) + { + tb.reg_origin_abs[i] = expected; + } + else if (tb.reg_origin_abs[i] != expected) + { + if (tb.reg_origin_abs[i] == 0x40000 || expected + 2 == 0 || expected == 0x40000) + { + // Set -2: sticky value indicating possible external reg origin (0x40000) + tb.reg_origin_abs[i] = -2; + + must_repeat |= !tb.targets.empty(); + } + else if (tb.reg_origin_abs[i] + 1) + { + tb.reg_origin_abs[i] = -1; + + must_repeat |= !tb.targets.empty(); + } + } + } + } + } + } + + if (!must_repeat) + { + break; + } + + for (u32 wi = 0; wi < workload.size(); wi++) + { + const u32 addr = workload[wi]; + auto& block = ::at32(m_bbs, addr); + + // Reset values for the next attempt (keep negative values) + for (u32 i = 0; i < s_reg_max; i++) + { + if (block.reg_origin[i] <= 0x40000) + block.reg_origin[i] = 0x80000000; + if (block.reg_origin_abs[i] <= 0x40000) + block.reg_origin_abs[i] = 0x80000000; + } + } + } + + // Fill more block info + for (u32 wi = 0; wi < workload.size(); wi++) + { + if (g_cfg.core.spu_block_size != spu_block_size_type::giga) + { + break; + } + + const u32 addr = workload[wi]; + auto& bb = ::at32(m_bbs, addr); + auto& func = ::at32(m_funcs, bb.func); + + // Update function size + func.size = std::max(func.size, bb.size + (addr - bb.func) / 4); + + // Copy constants according to reg origin info + for (u32 i = 0; i < s_reg_max; i++) + { + const u32 orig = bb.reg_origin_abs[i]; + + if (orig < 0x40000) + { + auto& src = ::at32(m_bbs, orig); + bb.reg_const[i] = src.reg_const[i]; + bb.reg_val32[i] = src.reg_val32[i]; + } + + if (!bb.reg_save_dom[i] && bb.reg_use[i] && (orig == 0x40000 || orig + 2 == 0)) + { + // Destroy offset if external reg value is used + func.reg_save_off[i] = -1; + } + } + + if (u32 orig = bb.reg_origin_abs[s_reg_sp]; orig < 0x40000) + { + auto& prologue = ::at32(m_bbs, orig); + + // Copy stack offset (from the assumed prologue) + bb.stack_sub = prologue.stack_sub; + } + else if (orig > 0x40000) + { + // Unpredictable stack + bb.stack_sub = 0x80000000; + } + + spu_opcode_t op{}; + + auto last_inst = spu_itype::UNK; + + for (u32 ia = addr; ia < addr + bb.size * 4; ia += 4) + { + // Decode instruction again + op.opcode = std::bit_cast>(result.data[(ia - lsa) / 41]); + last_inst = g_spu_itype.decode(op.opcode); + + // Propagate some constants + switch (last_inst) + { + case spu_itype::IL: + { + bb.reg_const[op.rt] = true; + bb.reg_val32[op.rt] = op.si16; + break; + } + case spu_itype::ILA: + { + bb.reg_const[op.rt] = true; + bb.reg_val32[op.rt] = op.i18; + break; + } + case spu_itype::ILHU: + { + bb.reg_const[op.rt] = true; + bb.reg_val32[op.rt] = op.i16 << 16; + break; + } + case spu_itype::ILH: + { + bb.reg_const[op.rt] = true; + bb.reg_val32[op.rt] = op.i16 << 16 | op.i16; + break; + } + case spu_itype::IOHL: + { + bb.reg_val32[op.rt] = bb.reg_val32[op.rt] | op.i16; + break; + } + case spu_itype::ORI: + { + bb.reg_const[op.rt] = bb.reg_const[op.ra]; + bb.reg_val32[op.rt] = bb.reg_val32[op.ra] | op.si10; + break; + } + case spu_itype::OR: + { + bb.reg_const[op.rt] = bb.reg_const[op.ra] && bb.reg_const[op.rb]; + bb.reg_val32[op.rt] = bb.reg_val32[op.ra] | bb.reg_val32[op.rb]; + break; + } + case spu_itype::AI: + { + bb.reg_const[op.rt] = bb.reg_const[op.ra]; + bb.reg_val32[op.rt] = bb.reg_val32[op.ra] + op.si10; + break; + } + case spu_itype::A: + { + bb.reg_const[op.rt] = bb.reg_const[op.ra] && bb.reg_const[op.rb]; + bb.reg_val32[op.rt] = bb.reg_val32[op.ra] + bb.reg_val32[op.rb]; + break; + } + case spu_itype::SFI: + { + bb.reg_const[op.rt] = bb.reg_const[op.ra]; + bb.reg_val32[op.rt] = op.si10 - bb.reg_val32[op.ra]; + break; + } + case spu_itype::SF: + { + bb.reg_const[op.rt] = bb.reg_const[op.ra] && bb.reg_const[op.rb]; + bb.reg_val32[op.rt] = bb.reg_val32[op.rb] - bb.reg_val32[op.ra]; + break; + } + case spu_itype::STQD: + { + if (op.ra == s_reg_sp && bb.stack_sub != 0x80000000 && bb.reg_save_dom[op.rt]) + { + const u32 offset = 0x80000000 + op.si10 * 16 - bb.stack_sub; + + if (func.reg_save_off[op.rt] == 0) + { + // Store reg save offset + func.reg_save_off[op.rt] = offset; + } + else if (func.reg_save_off[op.rt] != offset) + { + // Conflict of different offsets + func.reg_save_off[op.rt] = -1; + } + } + + break; + } + case spu_itype::LQD: + { + if (op.ra == s_reg_sp && bb.stack_sub != 0x80000000 && bb.reg_load_mod[op.rt] == ia + 1) + { + // Adjust reg load offset + bb.reg_load_mod[op.rt] = 0x80000000 + op.si10 * 16 - bb.stack_sub; + } + + // Clear const + bb.reg_const[op.rt] = false; + break; + } + default: + { + // Clear const if reg is modified here + if (u8 reg = m_regmod[ia / 4]; reg < s_reg_max) + bb.reg_const[reg] = false; + break; + } + } + + // $SP is modified + if (m_regmod[ia / 4] == s_reg_sp) + { + if (bb.reg_const[s_reg_sp]) + { + // Making $SP a constant is a funny thing too. + bb.stack_sub = 0x80000000; + } + + if (bb.stack_sub != 0x80000000) + { + switch (last_inst) + { + case spu_itype::AI: + { + if (op.ra == s_reg_sp) + bb.stack_sub -= op.si10; + else + bb.stack_sub = 0x80000000; + break; + } + case spu_itype::A: + { + if (op.ra == s_reg_sp && bb.reg_const[op.rb]) + bb.stack_sub -= bb.reg_val32[op.rb]; + else if (op.rb == s_reg_sp && bb.reg_const[op.ra]) + bb.stack_sub -= bb.reg_val32[op.ra]; + else + bb.stack_sub = 0x80000000; + break; + } + case spu_itype::SF: + { + if (op.rb == s_reg_sp && bb.reg_const[op.ra]) + bb.stack_sub += bb.reg_val32[op.ra]; + else + bb.stack_sub = 0x80000000; + break; + } + default: + { + bb.stack_sub = 0x80000000; + break; + } + } + } + + // Check for funny values. + if (bb.stack_sub >= 0x40000 || bb.stack_sub % 16) + { + bb.stack_sub = 0x80000000; + } + } + } + + // Analyse terminator instruction + const u32 tia = addr + bb.size * 4 - 4; + + switch (last_inst) + { + case spu_itype::BR: + case spu_itype::BRNZ: + case spu_itype::BRZ: + case spu_itype::BRHNZ: + case spu_itype::BRHZ: + case spu_itype::BRSL: + { + const u32 target = spu_branch_target(tia, op.i16); + + if (target == tia + 4) + { + bb.terminator = term_type::fallthrough; + } + else if (last_inst != spu_itype::BRSL) + { + // No-op terminator or simple branch instruction + bb.terminator = term_type::br; + + if (target == bb.func) + { + // Recursive tail call + bb.terminator = term_type::ret; + } + } + else if (op.rt == s_reg_lr) + { + bb.terminator = term_type::call; + } + else + { + bb.terminator = term_type::interrupt_call; + } + + break; + } + case spu_itype::BRA: + case spu_itype::BRASL: + { + bb.terminator = term_type::indirect_call; + break; + } + case spu_itype::BI: + { + if (op.d || op.e || bb.targets.size() == 1) + { + bb.terminator = term_type::interrupt_call; + } + else if (bb.targets.size() > 1) + { + // Jump table + bb.terminator = term_type::br; + } + else if (op.ra == s_reg_lr) + { + // Return (TODO) + bb.terminator = term_type::ret; + } + else + { + // Indirect tail call (TODO) + bb.terminator = term_type::interrupt_call; + } + + break; + } + case spu_itype::BISLED: + case spu_itype::IRET: + { + bb.terminator = term_type::interrupt_call; + break; + } + case spu_itype::BISL: + case spu_itype::BIZ: + case spu_itype::BINZ: + case spu_itype::BIHZ: + case spu_itype::BIHNZ: + { + if (op.d || op.e || bb.targets.size() != 1) + { + bb.terminator = term_type::interrupt_call; + } + else if (last_inst != spu_itype::BISL && bb.targets[0] == tia + 4 && op.ra == s_reg_lr) + { + // Conditional return (TODO) + bb.terminator = term_type::ret; + } + else if (last_inst == spu_itype::BISL) + { + // Indirect call + bb.terminator = term_type::indirect_call; + } + else + { + // TODO + bb.terminator = term_type::interrupt_call; + } + + break; + } + default: + { + // Normal instruction + bb.terminator = term_type::fallthrough; + break; + } + } + } + + // Check function blocks, verify and print some reasons + for (auto& f : m_funcs) + { + if (g_cfg.core.spu_block_size != spu_block_size_type::giga) + { + break; + } + + bool is_ok = true; + + u32 used_stack = 0; + + for (auto it = m_bbs.lower_bound(f.first); it != m_bbs.end() && it->second.func == f.first; ++it) + { + auto& bb = it->second; + auto& func = ::at32(m_funcs, bb.func); + const u32 addr = it->first; + const u32 flim = bb.func + func.size * 4; + + used_stack |= bb.stack_sub; + + if (is_ok && bb.terminator >= term_type::indirect_call) + { + is_ok = false; + } + + if (is_ok && bb.terminator == term_type::ret) + { + // Check $LR (alternative return registers are currently not supported) + if (u32 lr_orig = bb.reg_mod[s_reg_lr] ? addr : bb.reg_origin_abs[s_reg_lr]; lr_orig < 0x40000) + { + auto& src = ::at32(m_bbs, lr_orig); + + if (src.reg_load_mod[s_reg_lr] != func.reg_save_off[s_reg_lr]) + { + spu_log.error("Function 0x%05x: [0x%05x] $LR mismatch (src=0x%x; 0x%x vs 0x%x)", f.first, addr, lr_orig, src.reg_load_mod[0], func.reg_save_off[0]); + is_ok = false; + } + else if (src.reg_load_mod[s_reg_lr] == 0) + { + spu_log.error("Function 0x%05x: [0x%05x] $LR modified (src=0x%x)", f.first, addr, lr_orig); + is_ok = false; + } + } + else if (lr_orig > 0x40000) + { + spu_log.todo("Function 0x%05x: [0x%05x] $LR unpredictable (src=0x%x)", f.first, addr, lr_orig); + is_ok = false; + } + + // Check $80..$127 (should be restored or unmodified) + for (u32 i = s_reg_80; is_ok && i <= s_reg_127; i++) + { + if (u32 orig = bb.reg_mod[i] ? addr : bb.reg_origin_abs[i]; orig < 0x40000) + { + auto& src = ::at32(m_bbs, orig); + + if (src.reg_load_mod[i] != func.reg_save_off[i]) + { + spu_log.error("Function 0x%05x: [0x%05x] $%u mismatch (src=0x%x; 0x%x vs 0x%x)", f.first, addr, i, orig, src.reg_load_mod[i], func.reg_save_off[i]); + is_ok = false; + } + } + else if (orig > 0x40000) + { + spu_log.todo("Function 0x%05x: [0x%05x] $%u unpredictable (src=0x%x)", f.first, addr, i, orig); + is_ok = false; + } + + if (func.reg_save_off[i] + 1 == 0) + { + spu_log.error("Function 0x%05x: [0x%05x] $%u used incorrectly", f.first, addr, i); + is_ok = false; + } + } + + // Check $SP (should be restored or unmodified) + if (bb.stack_sub != 0 && bb.stack_sub != 0x80000000) + { + spu_log.error("Function 0x%05x: [0x%05x] return with stack frame 0x%x", f.first, addr, bb.stack_sub); + is_ok = false; + } + } + + if (is_ok && bb.terminator == term_type::call) + { + // Check call instruction (TODO) + if (bb.stack_sub == 0) + { + // Call without a stack frame + spu_log.error("Function 0x%05x: [0x%05x] frameless call", f.first, addr); + is_ok = false; + } + } + + if (is_ok && bb.terminator == term_type::fallthrough) + { + // Can't just fall out of the function + if (bb.targets.size() != 1 || bb.targets[0] >= flim) + { + spu_log.error("Function 0x%05x: [0x%05x] bad fallthrough to 0x%x", f.first, addr, bb.targets[0]); + is_ok = false; + } + } + + if (is_ok && bb.stack_sub == 0x80000000) + { + spu_log.error("Function 0x%05x: [0x%05x] bad stack frame", f.first, addr); + is_ok = false; + } + + // Fill external function targets (calls, possibly tail calls) + for (u32 target : bb.targets) + { + if (target < bb.func || target >= flim || (bb.terminator == term_type::call && target == bb.func)) + { + if (func.calls.find_first_of(target) + 1 == 0) + { + func.calls.push_back(target); + } + } + } + } + + if (is_ok && used_stack && f.first == entry_point) + { + spu_log.error("Function 0x%05x: considered possible chunk", f.first); + is_ok = false; + } + + // if (is_ok && f.first > 0x1d240 && f.first < 0x1e000) + // { + // spu_log.error("Function 0x%05x: manually disabled", f.first); + // is_ok = false; + // } + + f.second.good = is_ok; + } + + // Check function call graph + while (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + bool need_repeat = false; + + for (auto& f : m_funcs) + { + if (!f.second.good) + { + continue; + } + + for (u32 call : f.second.calls) + { + const auto ffound = std::as_const(m_funcs).find(call); + + if (ffound == m_funcs.cend() || ffound->second.good == false) + { + need_repeat = true; + + if (f.second.good) + { + spu_log.error("Function 0x%05x: calls bad function (0x%05x)", f.first, call); + f.second.good = false; + } + } + } + } + + if (!need_repeat) + { + break; + } + } + + if (result.data.empty()) + { + // Blocks starting from 0x0 or invalid instruction won't be compiled, may need special interpreter fallback + } + + return result; +} + +void spu_recompiler_base::dump(const spu_program& result, std::string& out) +{ + SPUDisAsm dis_asm(cpu_disasm_mode::dump, reinterpret_cast(result.data.data()), result.lower_bound); + + std::string hash; + { + sha1_context ctx; + u8 output[20]; + + sha1_starts(&ctx); + sha1_update(&ctx, reinterpret_cast(result.data.data()), result.data.size() * 4); + sha1_finish(&ctx, output); + fmt::append(hash, "%s", fmt::base57(output)); + } + + fmt::append(out, "========== SPU BLOCK 0x%05x (size %u, %s) ==========\n\n", result.entry_point, result.data.size(), hash); + + for (auto& bb : m_bbs) + { + for (u32 pos = bb.first, end = bb.first + bb.second.size * 4; pos < end; pos += 4) + { + dis_asm.disasm(pos); + + if (!dis_asm.last_opcode.ends_with('\n')) + { + dis_asm.last_opcode += '\n'; + } + + fmt::append(out, ">%s", dis_asm.last_opcode); + } + + out += '\n'; + + if (m_block_info[bb.first / 4]) + { + fmt::append(out, "A: [0x%05x] %s\n", bb.first, m_entry_info[bb.first / 4] ? (m_ret_info[bb.first / 4] ? "Chunk" : "Entry") : "Block"); + + fmt::append(out, "\tF: 0x%05x\n", bb.second.func); + + for (u32 pred : bb.second.preds) + { + fmt::append(out, "\t<- 0x%05x\n", pred); + } + + for (u32 target : bb.second.targets) + { + fmt::append(out, "\t-> 0x%05x%s\n", target, m_bbs.count(target) ? "" : " (null)"); + } + } + else + { + fmt::append(out, "A: [0x%05x] ?\n", bb.first); + } + + out += '\n'; + } + + for (auto& f : m_funcs) + { + fmt::append(out, "F: [0x%05x]%s\n", f.first, f.second.good ? " (good)" : " (bad)"); + + fmt::append(out, "\tN: 0x%05x\n", f.second.size * 4 + f.first); + + for (u32 call : f.second.calls) + { + fmt::append(out, "\t>> 0x%05x%s\n", call, m_funcs.count(call) ? "" : " (null)"); + } + } + + out += '\n'; +} + +#ifdef LLVM_AVAILABLE + +#include "Emu/CPU/CPUTranslator.h" + +#ifdef _MSC_VER +#pragma warning(push, 0) +#else +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#pragma GCC diagnostic ignored "-Wextra" +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#pragma GCC diagnostic ignored "-Weffc++" +#pragma GCC diagnostic ignored "-Wmissing-noreturn" +#endif +#if LLVM_VERSION_MAJOR < 17 +#include "llvm/ADT/Triple.h" +#endif +#include "llvm/TargetParser/Host.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Scalar.h" +#include "llvm/Analysis/PostDominators.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/ADT/PostOrderIterator.h" +#ifdef _MSC_VER +#pragma warning(pop) +#else +#pragma GCC diagnostic pop +#endif + +class spu_llvm_recompiler : public spu_recompiler_base, public cpu_translator +{ + // JIT Instance + jit_compiler m_jit{{}, jit_compiler::cpu(g_cfg.core.llvm_cpu)}; + + // Interpreter table size power + const u8 m_interp_magn; + + // Constant opcode bits + u32 m_op_const_mask = -1; + + // Current function chunk entry point + u32 m_entry; + + // Main entry point offset + u32 m_base; + + // Module name + std::string m_hash; + + // Patchpoint unique id + u32 m_pp_id = 0; + + // Next opcode + u32 m_next_op = 0; + + // Current function (chunk) + llvm::Function* m_function; + + llvm::Value* m_thread; + llvm::Value* m_lsptr; + llvm::Value* m_interp_op; + llvm::Value* m_interp_pc; + llvm::Value* m_interp_table; + llvm::Value* m_interp_7f0; + llvm::Value* m_interp_regs; + + // Helpers + llvm::Value* m_base_pc; + llvm::Value* m_interp_pc_next; + llvm::BasicBlock* m_interp_bblock; + + // i8*, contains constant vm::g_base_addr value + llvm::Value* m_memptr; + + // Pointers to registers in the thread context + std::array m_reg_addr; + + // Global variable (function table) + llvm::GlobalVariable* m_function_table{}; + + // Helpers (interpreter) + llvm::GlobalVariable* m_scale_float_to{}; + llvm::GlobalVariable* m_scale_to_float{}; + + // Function for check_state execution + llvm::Function* m_test_state{}; + + // Chunk for external tail call (dispatch) + llvm::Function* m_dispatch{}; + + llvm::MDNode* m_md_unlikely; + llvm::MDNode* m_md_likely; + + struct block_info + { + // Pointer to the analyser + spu_recompiler_base::block_info* bb{}; + + // Current block's entry block + llvm::BasicBlock* block; + + // Final block (for PHI nodes, set after completion) + llvm::BasicBlock* block_end{}; + + // Additional blocks for sinking instructions after block_end: + std::unordered_map> block_edges; + + // Current register values + std::array reg{}; + + // PHI nodes created for this block (if any) + std::array phi{}; + + // Store instructions + std::array store{}; + + // Store reordering/elimination protection + std::array store_context_last_id = fill_array(0); // Protects against illegal forward ordering + std::array store_context_first_id = fill_array(usz{umax}); // Protects against illegal past store elimination (backwards ordering is not implemented) + std::array store_context_ctr = fill_array(1); // Store barrier cointer + + bool does_gpr_barrier_proceed_last_store(u32 i) const noexcept + { + const usz counter = store_context_ctr[i]; + return counter != 1 && counter > store_context_last_id[i]; + } + + bool does_gpr_barrier_preceed_first_store(u32 i) const noexcept + { + const usz counter = store_context_ctr[i]; + return counter != 1 && counter < store_context_first_id[i]; + } + }; + + struct function_info + { + // Standard callable chunk + llvm::Function* chunk{}; + + // Callable function + llvm::Function* fn{}; + + // Registers possibly loaded in the entry block + std::array load{}; + }; + + // Current block + block_info* m_block; + + // Current function or chunk + function_info* m_finfo; + + // All blocks in the current function chunk + std::unordered_map> m_blocks; + + // Block list for processing + std::vector m_block_queue; + + // All function chunks in current SPU compile unit + std::unordered_map> m_functions; + + // Function chunk list for processing + std::vector m_function_queue; + + // Add or get the function chunk + function_info* add_function(u32 addr) + { + // Enqueue if necessary + const auto empl = m_functions.try_emplace(addr); + + if (!empl.second) + { + return &empl.first->second; + } + + // Chunk function type + // 0. Result (tail call target) + // 1. Thread context + // 2. Local storage pointer + // 3. +#if 0 + const auto chunk_type = get_ftype(); +#else + const auto chunk_type = get_ftype(); +#endif + + // Get function chunk name + const std::string name = fmt::format("__spu-cx%05x-%s", addr, fmt::base57(be_t{m_hash_start})); + llvm::Function* result = llvm::cast(m_module->getOrInsertFunction(name, chunk_type).getCallee()); + + // Set parameters + result->setLinkage(llvm::GlobalValue::InternalLinkage); + result->addParamAttr(0, llvm::Attribute::NoAlias); + result->addParamAttr(1, llvm::Attribute::NoAlias); +#if 1 + result->setCallingConv(llvm::CallingConv::GHC); +#endif + + empl.first->second.chunk = result; + + if (g_cfg.core.spu_block_size == spu_block_size_type::giga) + { + // Find good real function + const auto ffound = m_funcs.find(addr); + + if (ffound != m_funcs.end() && ffound->second.good) + { + // Real function type (not equal to chunk type) + // 4. $SP + // 5. $3 + const auto func_type = get_ftype(); + + const std::string fname = fmt::format("__spu-fx%05x-%s", addr, fmt::base57(be_t{m_hash_start})); + llvm::Function* fn = llvm::cast(m_module->getOrInsertFunction(fname, func_type).getCallee()); + + fn->setLinkage(llvm::GlobalValue::InternalLinkage); + fn->addParamAttr(0, llvm::Attribute::NoAlias); + fn->addParamAttr(1, llvm::Attribute::NoAlias); +#if 1 + fn->setCallingConv(llvm::CallingConv::GHC); +#endif + empl.first->second.fn = fn; + } + } + + // Enqueue + m_function_queue.push_back(addr); + + return &empl.first->second; + } + + // Create tail call to the function chunk (non-tail calls are just out of question) + void tail_chunk(llvm::FunctionCallee callee, llvm::Value* base_pc = nullptr) + { + if (!callee && !g_cfg.core.spu_verification) + { + // Disable patchpoints if verification is disabled + callee = m_dispatch; + } + else if (!callee) + { + // Create branch patchpoint if chunk == nullptr + ensure(m_finfo && (!m_finfo->fn || m_function == m_finfo->chunk)); + + // Register under a unique linkable name + const std::string ppname = fmt::format("%s-pp-%u", m_hash, m_pp_id++); + m_engine->updateGlobalMapping(ppname, reinterpret_cast(m_spurt->make_branch_patchpoint())); + + // Create function with not exactly correct type + const auto ppfunc = llvm::cast(m_module->getOrInsertFunction(ppname, m_finfo->chunk->getFunctionType()).getCallee()); + ppfunc->setCallingConv(m_finfo->chunk->getCallingConv()); + + if (m_finfo->chunk->getReturnType() != get_type()) + { + m_ir->CreateRet(ppfunc); + return; + } + + callee = ppfunc; + base_pc = m_ir->getInt32(0); + } + + ensure(callee); + auto call = m_ir->CreateCall(callee, {m_thread, m_lsptr, base_pc ? base_pc : m_base_pc}); + auto func = m_finfo ? m_finfo->chunk : llvm::dyn_cast(callee.getCallee()); + call->setCallingConv(func->getCallingConv()); + call->setTailCall(); + + if (func->getReturnType() == get_type()) + { + m_ir->CreateRetVoid(); + } + else + { + m_ir->CreateRet(call); + } + } + + // Call the real function + void call_function(llvm::Function* fn, bool tail = false) + { + llvm::Value* lr{}; + llvm::Value* sp{}; + llvm::Value* r3{}; + + if (!m_finfo->fn && !m_block) + { + lr = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::gpr, +s_reg_lr, &v128::_u32, 3)); + sp = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::gpr, +s_reg_sp)); + r3 = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::gpr, 3)); + } + else + { + lr = m_ir->CreateExtractElement(get_reg_fixed(s_reg_lr).value, 3); + sp = get_reg_fixed(s_reg_sp).value; + r3 = get_reg_fixed(3).value; + } + + const auto _call = m_ir->CreateCall(ensure(fn), {m_thread, m_lsptr, m_base_pc, sp, r3}); + + _call->setCallingConv(fn->getCallingConv()); + + // Tail call using loaded LR value (gateway from a chunk) + if (!m_finfo->fn) + { + lr = m_ir->CreateAnd(lr, 0x3fffc); + m_ir->CreateStore(lr, spu_ptr(&spu_thread::pc)); + m_ir->CreateStore(_call, spu_ptr(&spu_thread::gpr, 3)); + m_ir->CreateBr(add_block_indirect({}, value(lr))); + } + else if (tail) + { + _call->setTailCall(); + m_ir->CreateRet(_call); + } + else + { + // TODO: initialize $LR with a constant + for (u32 i = 0; i < s_reg_max; i++) + { + if (i != s_reg_lr && i != s_reg_sp && (i < s_reg_80 || i > s_reg_127)) + { + m_block->reg[i] = m_ir->CreateLoad(get_reg_type(i), init_reg_fixed(i)); + } + } + + // Set result + m_block->reg[3] = _call; + } + } + + // Emit return from the real function + void ret_function() + { + m_ir->CreateRet(get_reg_fixed(3).value); + } + + void set_function(llvm::Function* func) + { + m_function = func; + m_thread = func->getArg(0); + m_lsptr = func->getArg(1); + m_base_pc = func->getArg(2); + + m_reg_addr.fill(nullptr); + m_block = nullptr; + m_finfo = nullptr; + m_blocks.clear(); + m_block_queue.clear(); + m_ir->SetInsertPoint(llvm::BasicBlock::Create(m_context, "", m_function)); + m_memptr = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::memory_base_addr)); + } + + // Add block with current block as a predecessor + llvm::BasicBlock* add_block(u32 target, bool absolute = false) + { + // Check the predecessor + const bool pred_found = m_block_info[target / 4] && m_preds[target].find_first_of(m_pos) + 1; + + if (m_blocks.empty()) + { + // Special case: first block, proceed normally + if (auto fn = std::exchange(m_finfo->fn, nullptr)) + { + // Create a gateway + call_function(fn, true); + + m_finfo->fn = fn; + m_function = fn; + m_thread = fn->getArg(0); + m_lsptr = fn->getArg(1); + m_base_pc = fn->getArg(2); + m_ir->SetInsertPoint(llvm::BasicBlock::Create(m_context, "", fn)); + m_memptr = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::memory_base_addr)); + + // Load registers at the entry chunk + for (u32 i = 0; i < s_reg_max; i++) + { + if (i >= s_reg_80 && i <= s_reg_127) + { + // TODO + //m_finfo->load[i] = llvm::UndefValue::get(get_reg_type(i)); + } + + m_finfo->load[i] = m_ir->CreateLoad(get_reg_type(i), init_reg_fixed(i)); + } + + // Load $SP + m_finfo->load[s_reg_sp] = fn->getArg(3); + + // Load first args + m_finfo->load[3] = fn->getArg(4); + } + } + else if (m_block_info[target / 4] && m_entry_info[target / 4] && !(pred_found && m_entry == target) && (!m_finfo->fn || !m_ret_info[target / 4])) + { + // Generate a tail call to the function chunk + const auto cblock = m_ir->GetInsertBlock(); + const auto result = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->SetInsertPoint(result); + const auto pfinfo = add_function(target); + + if (absolute) + { + ensure(!m_finfo->fn); + + const auto next = llvm::BasicBlock::Create(m_context, "", m_function); + const auto fail = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->CreateCondBr(m_ir->CreateICmpEQ(m_base_pc, m_ir->getInt32(m_base)), next, fail); + m_ir->SetInsertPoint(fail); + m_ir->CreateStore(m_ir->getInt32(target), spu_ptr(&spu_thread::pc)); + tail_chunk(nullptr); + m_ir->SetInsertPoint(next); + } + + if (pfinfo->fn) + { + // Tail call to the real function + call_function(pfinfo->fn, true); + + if (!result->getTerminator()) + ret_function(); + } + else + { + // Just a boring tail call to another chunk + update_pc(target); + tail_chunk(pfinfo->chunk); + } + + m_ir->SetInsertPoint(cblock); + return result; + } + else if (!pred_found || !m_block_info[target / 4]) + { + if (m_block_info[target / 4]) + { + spu_log.error("[%s] [0x%x] Predecessor not found for target 0x%x (chunk=0x%x, entry=0x%x, size=%u)", m_hash, m_pos, target, m_entry, m_function_queue[0], m_size / 4); + } + + const auto cblock = m_ir->GetInsertBlock(); + const auto result = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->SetInsertPoint(result); + + if (absolute) + { + ensure(!m_finfo->fn); + + m_ir->CreateStore(m_ir->getInt32(target), spu_ptr(&spu_thread::pc)); + } + else + { + update_pc(target); + } + + tail_chunk(nullptr); + m_ir->SetInsertPoint(cblock); + return result; + } + + ensure(!absolute); + + auto& result = m_blocks[target].block; + + if (!result) + { + result = llvm::BasicBlock::Create(m_context, fmt::format("b-0x%x", target), m_function); + + // Add the block to the queue + m_block_queue.push_back(target); + } + else if (m_block && m_blocks[target].block_end) + { + // Connect PHI nodes if necessary + for (u32 i = 0; i < s_reg_max; i++) + { + if (const auto phi = m_blocks[target].phi[i]) + { + const auto typ = phi->getType() == get_type() ? get_type() : get_reg_type(i); + phi->addIncoming(get_reg_fixed(i, typ), m_block->block_end); + } + } + } + + return result; + } + + template + llvm::Value* _ptr(llvm::Value* base, u32 offset) + { + return m_ir->CreateGEP(get_type(), base, m_ir->getInt64(offset)); + } + + template + llvm::Value* spu_ptr(Args... offset_args) + { + return _ptr(m_thread, ::offset32(offset_args...)); + } + + template + llvm::Value* spu_ptr(value_t add, Args... offset_args) + { + const auto off = m_ir->CreateGEP(get_type(), m_thread, m_ir->getInt64(::offset32(offset_args...))); + return m_ir->CreateAdd(off, add.value); + } + + // Return default register type + llvm::Type* get_reg_type(u32 index) + { + if (index < 128) + { + return get_type(); + } + + switch (index) + { + case s_reg_mfc_eal: + case s_reg_mfc_lsa: + return get_type(); + case s_reg_mfc_tag: + return get_type(); + case s_reg_mfc_size: + return get_type(); + default: + fmt::throw_exception("get_reg_type(%u): invalid register index", index); + } + } + + u32 get_reg_offset(u32 index) + { + if (index < 128) + { + return ::offset32(&spu_thread::gpr, index); + } + + switch (index) + { + case s_reg_mfc_eal: return ::offset32(&spu_thread::ch_mfc_cmd, &spu_mfc_cmd::eal); + case s_reg_mfc_lsa: return ::offset32(&spu_thread::ch_mfc_cmd, &spu_mfc_cmd::lsa); + case s_reg_mfc_tag: return ::offset32(&spu_thread::ch_mfc_cmd, &spu_mfc_cmd::tag); + case s_reg_mfc_size: return ::offset32(&spu_thread::ch_mfc_cmd, &spu_mfc_cmd::size); + default: + fmt::throw_exception("get_reg_offset(%u): invalid register index", index); + } + } + + llvm::Value* init_reg_fixed(u32 index) + { + if (!m_block) + { + return _ptr(m_thread, get_reg_offset(index)); + } + + auto& ptr = ::at32(m_reg_addr, index); + + if (!ptr) + { + // Save and restore current insert point if necessary + const auto block_cur = m_ir->GetInsertBlock(); + + // Emit register pointer at the beginning of the function chunk + m_ir->SetInsertPoint(m_function->getEntryBlock().getTerminator()); + ptr = _ptr(m_thread, get_reg_offset(index)); + m_ir->SetInsertPoint(block_cur); + } + + return ptr; + } + + // Get pointer to the vector register (interpreter only) + template + llvm::Value* init_vr(const bf_t&) + { + if (!m_interp_magn) + { + m_interp_7f0 = m_ir->getInt32(0x7f0); + m_interp_regs = _ptr(m_thread, get_reg_offset(0)); + } + + // Extract reg index + const auto isl = I >= 4 ? m_interp_op : m_ir->CreateShl(m_interp_op, u64{4 - I}); + const auto isr = I <= 4 ? m_interp_op : m_ir->CreateLShr(m_interp_op, u64{I - 4}); + const auto idx = m_ir->CreateAnd(I > 4 ? isr : isl, m_interp_7f0); + + // Pointer to the register + return m_ir->CreateGEP(get_type(), m_interp_regs, m_ir->CreateZExt(idx, get_type())); + } + + llvm::Value* double_as_uint64(llvm::Value* val) + { + return bitcast(val); + } + + llvm::Value* uint64_as_double(llvm::Value* val) + { + return bitcast(val); + } + + llvm::Value* double_to_xfloat(llvm::Value* val) + { + ensure(val && val->getType() == get_type()); + + const auto d = double_as_uint64(val); + const auto s = m_ir->CreateAnd(m_ir->CreateLShr(d, 32), 0x80000000); + const auto m = m_ir->CreateXor(m_ir->CreateLShr(d, 29), 0x40000000); + const auto r = m_ir->CreateOr(m_ir->CreateAnd(m, 0x7fffffff), s); + return m_ir->CreateTrunc(m_ir->CreateSelect(m_ir->CreateIsNotNull(d), r, splat(0).eval(m_ir)), get_type()); + } + + llvm::Value* xfloat_to_double(llvm::Value* val) + { + ensure(val && val->getType() == get_type()); + + const auto x = m_ir->CreateZExt(val, get_type()); + const auto s = m_ir->CreateShl(m_ir->CreateAnd(x, 0x80000000), 32); + const auto a = m_ir->CreateAnd(x, 0x7fffffff); + const auto m = m_ir->CreateShl(m_ir->CreateAdd(a, splat(0x1c0000000).eval(m_ir)), 29); + const auto r = m_ir->CreateSelect(m_ir->CreateICmpSGT(a, splat(0x7fffff).eval(m_ir)), m, splat(0).eval(m_ir)); + const auto f = m_ir->CreateOr(s, r); + return uint64_as_double(f); + } + + // Clamp double values to ±Smax, flush values smaller than ±Smin to positive zero + llvm::Value* xfloat_in_double(llvm::Value* val) + { + ensure(val && val->getType() == get_type()); + + const auto smax = uint64_as_double(splat(0x47ffffffe0000000).eval(m_ir)); + const auto smin = uint64_as_double(splat(0x3810000000000000).eval(m_ir)); + + const auto d = double_as_uint64(val); + const auto s = m_ir->CreateAnd(d, 0x8000000000000000); + const auto a = uint64_as_double(m_ir->CreateAnd(d, 0x7fffffffe0000000)); + const auto n = m_ir->CreateFCmpOLT(a, smax); + const auto z = m_ir->CreateFCmpOLT(a, smin); + const auto c = double_as_uint64(m_ir->CreateSelect(n, a, smax)); + return m_ir->CreateSelect(z, fsplat(0.).eval(m_ir), uint64_as_double(m_ir->CreateOr(c, s))); + } + + // Expand 32-bit mask for xfloat values to 64-bit, 29 least significant bits are always zero + llvm::Value* conv_xfloat_mask(llvm::Value* val) + { + const auto d = m_ir->CreateZExt(val, get_type()); + const auto s = m_ir->CreateShl(m_ir->CreateAnd(d, 0x80000000), 32); + const auto e = m_ir->CreateLShr(m_ir->CreateAShr(m_ir->CreateShl(d, 33), 4), 1); + return m_ir->CreateOr(s, e); + } + + llvm::Value* get_reg_raw(u32 index) + { + if (!m_block || index >= m_block->reg.size()) + { + return nullptr; + } + + return m_block->reg[index]; + } + + llvm::Value* get_reg_fixed(u32 index, llvm::Type* type) + { + llvm::Value* dummy{}; + + auto& reg = *(m_block ? &::at32(m_block->reg, index) : &dummy); + + if (!reg) + { + // Load register value if necessary + reg = m_finfo && m_finfo->load[index] ? m_finfo->load[index] : m_ir->CreateLoad(get_reg_type(index), init_reg_fixed(index)); + } + + if (reg->getType() == get_type()) + { + if (type == reg->getType()) + { + return reg; + } + + return bitcast(double_to_xfloat(reg), type); + } + + if (type == get_type()) + { + return xfloat_to_double(bitcast(reg)); + } + + return bitcast(reg, type); + } + + template + value_t get_reg_fixed(u32 index) + { + value_t r; + r.value = get_reg_fixed(index, get_type()); + return r; + } + + template + value_t get_vr(const bf_t& index) + { + value_t r; + + if ((m_op_const_mask & index.data_mask()) != index.data_mask()) + { + // Update const mask if necessary + if (I >= (32u - m_interp_magn)) + { + m_op_const_mask |= index.data_mask(); + } + + // Load reg + if (get_type() == get_type()) + { + r.value = xfloat_to_double(m_ir->CreateLoad(get_type(), init_vr(index))); + } + else + { + r.value = m_ir->CreateLoad(get_type(), init_vr(index)); + } + } + else + { + r.value = get_reg_fixed(index, get_type()); + } + + return r; + } + + template + auto get_vr_as(U&&, const bf_t& index) + { + return get_vr::type>(index); + } + + template + std::tuple>...> get_vrs(const Args&... args) + { + return {get_vr(args)...}; + } + + template + llvm_match_t match_vr(const bf_t& index) + { + llvm_match_t r; + + if (m_block) + { + auto v = ::at32(m_block->reg, index); + + if (v && v->getType() == get_type()) + { + r.value = v; + return r; + } + } + + return r; + } + + template + auto match_vr_as(U&&, const bf_t& index) + { + return match_vr::type>(index); + } + + template + bool match_vr(const bf_t& index, F&& pred) + { + return (( match_vr(index) ? pred(match_vr(index), match()) : false ) || ...); + } + + template + std::tuple>...> match_vrs(const Args&... args) + { + return {match_vr(args)...}; + } + + // Extract scalar value from the preferred slot + template + auto get_scalar(value_t value) + { + using e_type = std::remove_extent_t; + + static_assert(sizeof(T) == 16 || std::is_same_v, "Unknown vector type"); + + if (auto [ok, v] = match_expr(value, vsplat(match())); ok) + { + return eval(v); + } + + if constexpr (sizeof(e_type) == 1) + { + return eval(extract(value, 12)); + } + else if constexpr (sizeof(e_type) == 2) + { + return eval(extract(value, 6)); + } + else if constexpr (sizeof(e_type) == 4 || sizeof(T) == 32) + { + return eval(extract(value, 3)); + } + else + { + return eval(extract(value, 1)); + } + } + + // Splat scalar value from the preferred slot + template + auto splat_scalar(T&& arg) + { + using VT = std::remove_extent_t::type>; + + if constexpr (sizeof(VT) == 1) + { + return zshuffle(std::forward(arg), 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12); + } + else if constexpr (sizeof(VT) == 2) + { + return zshuffle(std::forward(arg), 6, 6, 6, 6, 6, 6, 6, 6); + } + else if constexpr (sizeof(VT) == 4) + { + return zshuffle(std::forward(arg), 3, 3, 3, 3); + } + else if constexpr (sizeof(VT) == 8) + { + return zshuffle(std::forward(arg), 1, 1); + } + else + { + static_assert(sizeof(VT) == 16); + return std::forward(arg); + } + } + + void set_reg_fixed(u32 index, llvm::Value* value, bool fixup = true) + { + llvm::StoreInst* dummy{}; + + // Check + ensure(!m_block || m_regmod[m_pos / 4] == index); + + // Test for special case + const bool is_xfloat = value->getType() == get_type(); + + // Clamp value if necessary + const auto saved_value = is_xfloat && fixup ? xfloat_in_double(value) : value; + + // Set register value + if (m_block) + { +#ifndef _WIN32 + if (g_cfg.core.spu_debug) + value->setName(fmt::format("result_0x%05x", m_pos)); +#endif + + ::at32(m_block->reg, index) = saved_value; + } + + // Get register location + const auto addr = init_reg_fixed(index); + + auto& _store = *(m_block ? &m_block->store[index] : &dummy); + + // Erase previous dead store instruction if necessary + if (_store) + { + if (m_block->store_context_last_id[index] == m_block->store_context_ctr[index]) + { + // Erase store of it is not preserved by ensure_gpr_stores() + _store->eraseFromParent(); + } + } + + if (m_block) + { + // Keep the store's location in history of gpr preservaions + m_block->store_context_last_id[index] = m_block->store_context_ctr[index]; + m_block->store_context_first_id[index] = std::min(m_block->store_context_first_id[index], m_block->store_context_ctr[index]); + } + + if (m_finfo && m_finfo->fn) + { + if (index <= 3 || (index >= s_reg_80 && index <= s_reg_127)) + { + // Don't save some registers in true functions + return; + } + } + + // Write register to the context + _store = m_ir->CreateStore(is_xfloat ? double_to_xfloat(saved_value) : m_ir->CreateBitCast(value, get_reg_type(index)), addr); + } + + template + void set_vr(const bf_t& index, T expr, std::function vr_assume = nullptr, bool fixup = true) + { + // Process expression + const auto value = expr.eval(m_ir); + + // Test for special case + const bool is_xfloat = value->getType() == get_type(); + + if ((m_op_const_mask & index.data_mask()) != index.data_mask()) + { + // Update const mask if necessary + if (I >= (32u - m_interp_magn)) + { + m_op_const_mask |= index.data_mask(); + } + + // Clamp value if necessary + const auto saved_value = is_xfloat && fixup ? xfloat_in_double(value) : value; + + // Store value + m_ir->CreateStore(is_xfloat ? double_to_xfloat(saved_value) : m_ir->CreateBitCast(value, get_type()), init_vr(index)); + return; + } + + if (vr_assume) + { + } + + set_reg_fixed(index, value, fixup); + } + + template + value_t get_imm(const bf_t& imm, bool mask = true) + { + if ((m_op_const_mask & imm.data_mask()) != imm.data_mask()) + { + // Update const mask if necessary + if (I >= (32u - m_interp_magn)) + { + m_op_const_mask |= imm.data_mask(); + } + + // Extract unsigned immediate (skip AND if mask == false or truncated anyway) + value_t r; + r.value = m_interp_op; + r.value = I == 0 ? r.value : m_ir->CreateLShr(r.value, u64{I}); + r.value = !mask || N >= r.esize ? r.value : m_ir->CreateAnd(r.value, imm.data_mask() >> I); + + if constexpr (r.esize != 32) + { + r.value = m_ir->CreateZExtOrTrunc(r.value, get_type()->getScalarType()); + } + + if (r.is_vector) + { + r.value = m_ir->CreateVectorSplat(r.is_vector, r.value); + } + + return r; + } + + return eval(splat(imm)); + } + + template + value_t get_imm(const bf_t& imm) + { + if ((m_op_const_mask & imm.data_mask()) != imm.data_mask()) + { + // Update const mask if necessary + if (I >= (32u - m_interp_magn)) + { + m_op_const_mask |= imm.data_mask(); + } + + // Extract signed immediate (skip sign ext if truncated anyway) + value_t r; + r.value = m_interp_op; + r.value = I + N == 32 || N >= r.esize ? r.value : m_ir->CreateShl(r.value, u64{32u - I - N}); + r.value = N == 32 || N >= r.esize ? r.value : m_ir->CreateAShr(r.value, u64{32u - N}); + r.value = I == 0 || N < r.esize ? r.value : m_ir->CreateLShr(r.value, u64{I}); + + if constexpr (r.esize != 32) + { + r.value = m_ir->CreateSExtOrTrunc(r.value, get_type()->getScalarType()); + } + + if (r.is_vector) + { + r.value = m_ir->CreateVectorSplat(r.is_vector, r.value); + } + + return r; + } + + return eval(splat(imm)); + } + + // Get PC for given instruction address + llvm::Value* get_pc(u32 addr) + { + return m_ir->CreateAdd(m_base_pc, m_ir->getInt32(addr - m_base)); + } + + // Update PC for current or explicitly specified instruction address + void update_pc(u32 target = -1) + { + m_ir->CreateStore(m_ir->CreateAnd(get_pc(target + 1 ? target : m_pos), 0x3fffc), spu_ptr(&spu_thread::pc))->setVolatile(true); + } + + // Call cpu_thread::check_state if necessary and return or continue (full check) + void check_state(u32 addr, bool may_be_unsafe_for_savestate = true) + { + const auto pstate = spu_ptr(&spu_thread::state); + const auto _body = llvm::BasicBlock::Create(m_context, "", m_function); + const auto check = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->CreateCondBr(m_ir->CreateICmpEQ(m_ir->CreateLoad(get_type(), pstate, true), m_ir->getInt32(0)), _body, check, m_md_likely); + m_ir->SetInsertPoint(check); + update_pc(addr); + + if (may_be_unsafe_for_savestate && std::none_of(std::begin(m_block->phi), std::end(m_block->phi), FN(!!x))) + { + may_be_unsafe_for_savestate = false; + } + + if (may_be_unsafe_for_savestate) + { + m_ir->CreateStore(m_ir->getInt8(1), spu_ptr(&spu_thread::unsavable))->setVolatile(true); + } + + m_ir->CreateCall(m_test_state, {m_thread}); + + if (may_be_unsafe_for_savestate) + { + m_ir->CreateStore(m_ir->getInt8(0), spu_ptr(&spu_thread::unsavable))->setVolatile(true); + } + + m_ir->CreateBr(_body); + m_ir->SetInsertPoint(_body); + } + +public: + spu_llvm_recompiler(u8 interp_magn = 0) + : spu_recompiler_base() + , cpu_translator(nullptr, false) + , m_interp_magn(interp_magn) + { + } + + virtual void init() override + { + // Initialize if necessary + if (!m_spurt) + { + m_spurt = &g_fxo->get(); + cpu_translator::initialize(m_jit.get_context(), m_jit.get_engine()); + + const auto md_name = llvm::MDString::get(m_context, "branch_weights"); + const auto md_low = llvm::ValueAsMetadata::get(llvm::ConstantInt::get(GetType(), 1)); + const auto md_high = llvm::ValueAsMetadata::get(llvm::ConstantInt::get(GetType(), 999)); + + // Metadata for branch weights + m_md_likely = llvm::MDTuple::get(m_context, {md_name, md_high, md_low}); + m_md_unlikely = llvm::MDTuple::get(m_context, {md_name, md_low, md_high}); + } + } + + virtual spu_function_t compile(spu_program&& _func) override + { + if (_func.data.empty() && m_interp_magn) + { + return compile_interpreter(); + } + + const u32 start0 = _func.entry_point; + + const auto add_loc = m_spurt->add_empty(std::move(_func)); + + if (!add_loc) + { + return nullptr; + } + + const spu_program& func = add_loc->data; + + if (func.entry_point != start0) + { + // Wait for the duplicate + while (!add_loc->compiled) + { + add_loc->compiled.wait(nullptr); + } + + return add_loc->compiled; + } + + std::string log; + + if (auto& cache = g_fxo->get(); cache && g_cfg.core.spu_cache && !add_loc->cached.exchange(1)) + { + cache.add(func); + } + + { + sha1_context ctx; + u8 output[20]; + + sha1_starts(&ctx); + sha1_update(&ctx, reinterpret_cast(func.data.data()), func.data.size() * 4); + sha1_finish(&ctx, output); + + m_hash.clear(); + fmt::append(m_hash, "__spu-0x%05x-%s", func.entry_point, fmt::base57(output)); + + be_t hash_start; + std::memcpy(&hash_start, output, sizeof(hash_start)); + m_hash_start = hash_start; + } + + spu_log.notice("Building function 0x%x... (size %u, %s)", func.entry_point, func.data.size(), m_hash); + + m_pos = func.lower_bound; + m_base = func.entry_point; + m_size = ::size32(func.data) * 4; + const u32 start = m_pos; + const u32 end = start + m_size; + + m_pp_id = 0; + + if (g_cfg.core.spu_debug && !add_loc->logged.exchange(1)) + { + this->dump(func, log); + fs::file(m_spurt->get_cache_path() + "spu.log", fs::write + fs::append).write(log); + } + + using namespace llvm; + + m_engine->clearAllGlobalMappings(); + + // Create LLVM module + std::unique_ptr _module = std::make_unique(m_hash + ".obj", m_context); + _module->setTargetTriple(jit_compiler::triple2()); + _module->setDataLayout(m_jit.get_engine().getTargetMachine()->createDataLayout()); + m_module = _module.get(); + + // Initialize IR Builder + IRBuilder<> irb(m_context); + m_ir = &irb; + + // Add entry function (contains only state/code check) + const auto main_func = llvm::cast(m_module->getOrInsertFunction(m_hash, get_ftype()).getCallee()); + const auto main_arg2 = main_func->getArg(2); + main_func->setCallingConv(CallingConv::GHC); + set_function(main_func); + + // Start compilation + const auto label_test = BasicBlock::Create(m_context, "", m_function); + const auto label_diff = BasicBlock::Create(m_context, "", m_function); + const auto label_body = BasicBlock::Create(m_context, "", m_function); + const auto label_stop = BasicBlock::Create(m_context, "", m_function); + + // Load PC, which will be the actual value of 'm_base' + m_base_pc = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::pc)); + + // Emit state check + const auto pstate = spu_ptr(&spu_thread::state); + m_ir->CreateCondBr(m_ir->CreateICmpNE(m_ir->CreateLoad(get_type(), pstate), m_ir->getInt32(0)), label_stop, label_test, m_md_unlikely); + + // Emit code check + u32 check_iterations = 0; + m_ir->SetInsertPoint(label_test); + + // Set block hash for profiling (if enabled) + if (g_cfg.core.spu_prof && g_cfg.core.spu_verification) + m_ir->CreateStore(m_ir->getInt64((m_hash_start & -65536)), spu_ptr(&spu_thread::block_hash)); + + if (!g_cfg.core.spu_verification) + { + // Disable check (unsafe) + m_ir->CreateBr(label_body); + } + else if (func.data.size() == 1) + { + const auto pu32 = m_ir->CreateGEP(get_type(), m_lsptr, m_base_pc); + const auto cond = m_ir->CreateICmpNE(m_ir->CreateLoad(get_type(), pu32), m_ir->getInt32(func.data[0])); + m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); + } + else if (func.data.size() == 2) + { + const auto pu64 = m_ir->CreateGEP(get_type(), m_lsptr, m_base_pc); + const auto cond = m_ir->CreateICmpNE(m_ir->CreateLoad(get_type(), pu64), m_ir->getInt64(static_cast(func.data[1]) << 32 | func.data[0])); + m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); + } + else + { + u32 starta = start; + + // Skip holes at the beginning (giga only) + for (u32 j = start; j < end; j += 4) + { + if (!func.data[(j - start) / 4]) + { + starta += 4; + } + else + { + break; + } + } + + u32 stride; + u32 elements; + u32 dwords; + + if (m_use_avx512 && g_cfg.core.full_width_avx512) + { + stride = 64; + elements = 16; + dwords = 8; + } + else if (m_use_avx) + { + stride = 32; + elements = 8; + dwords = 4; + } + else + { + stride = 16; + elements = 4; + dwords = 2; + } + + // Get actual pc corresponding to the found beginning of the data + llvm::Value* starta_pc = m_ir->CreateAnd(get_pc(starta), 0x3fffc); + llvm::Value* data_addr = m_ir->CreateGEP(get_type(), m_lsptr, starta_pc); + + llvm::Value* acc = nullptr; + + for (u32 j = starta; j < end; j += stride) + { + int indices[16]; + bool holes = false; + bool data = false; + + for (u32 i = 0; i < elements; i++) + { + const u32 k = j + i * 4; + + if (k < start || k >= end || !func.data[(k - start) / 4]) + { + indices[i] = elements; + holes = true; + } + else + { + indices[i] = i; + data = true; + } + } + + if (!data) + { + // Skip full-sized holes + continue; + } + + llvm::Value* vls = nullptr; + + // Load unaligned code block from LS + if (m_use_avx512 && g_cfg.core.full_width_avx512) + { + vls = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, j - starta), llvm::MaybeAlign{4}); + } + else if (m_use_avx) + { + vls = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, j - starta), llvm::MaybeAlign{4}); + } + else + { + vls = m_ir->CreateAlignedLoad(get_type(), _ptr(data_addr, j - starta), llvm::MaybeAlign{4}); + } + + // Mask if necessary + if (holes) + { + vls = m_ir->CreateShuffleVector(vls, ConstantAggregateZero::get(vls->getType()), llvm::ArrayRef(indices, elements)); + } + + // Perform bitwise comparison and accumulate + u32 words[16]; + + for (u32 i = 0; i < elements; i++) + { + const u32 k = j + i * 4; + words[i] = k >= start && k < end ? func.data[(k - start) / 4] : 0; + } + + vls = m_ir->CreateXor(vls, ConstantDataVector::get(m_context, llvm::ArrayRef(words, elements))); + acc = acc ? m_ir->CreateOr(acc, vls) : vls; + check_iterations++; + } + + // Pattern for PTEST + if (m_use_avx512 && g_cfg.core.full_width_avx512) + { + acc = m_ir->CreateBitCast(acc, get_type()); + } + else if (m_use_avx) + { + acc = m_ir->CreateBitCast(acc, get_type()); + } + else + { + acc = m_ir->CreateBitCast(acc, get_type()); + } + + llvm::Value* elem = m_ir->CreateExtractElement(acc, u64{0}); + + for (u32 i = 1; i < dwords; i++) + { + elem = m_ir->CreateOr(elem, m_ir->CreateExtractElement(acc, i)); + } + + // Compare result with zero + const auto cond = m_ir->CreateICmpNE(elem, m_ir->getInt64(0)); + m_ir->CreateCondBr(cond, label_diff, label_body, m_md_unlikely); + } + + // Increase block counter with statistics + m_ir->SetInsertPoint(label_body); + const auto pbcount = spu_ptr(&spu_thread::block_counter); + m_ir->CreateStore(m_ir->CreateAdd(m_ir->CreateLoad(get_type(), pbcount), m_ir->getInt64(check_iterations)), pbcount); + + // Call the entry function chunk + const auto entry_chunk = add_function(m_pos); + const auto entry_call = m_ir->CreateCall(entry_chunk->chunk, {m_thread, m_lsptr, m_base_pc}); + entry_call->setCallingConv(entry_chunk->chunk->getCallingConv()); + + const auto dispatcher = llvm::cast(m_module->getOrInsertFunction("spu_dispatcher", main_func->getType()).getCallee()); + m_engine->updateGlobalMapping("spu_dispatcher", reinterpret_cast(spu_runtime::tr_all)); + dispatcher->setCallingConv(main_func->getCallingConv()); + + // Proceed to the next code + if (entry_chunk->chunk->getReturnType() != get_type()) + { + const auto next_call = m_ir->CreateCall(main_func->getFunctionType(), entry_call, {m_thread, m_lsptr, m_ir->getInt64(0)}); + next_call->setCallingConv(main_func->getCallingConv()); + next_call->setTailCall(); + } + else + { + entry_call->setTailCall(); + } + + m_ir->CreateRetVoid(); + + m_ir->SetInsertPoint(label_stop); + call("spu_escape", spu_runtime::g_escape, m_thread)->setTailCall(); + m_ir->CreateRetVoid(); + + m_ir->SetInsertPoint(label_diff); + + if (g_cfg.core.spu_verification) + { + const auto pbfail = spu_ptr(&spu_thread::block_failure); + m_ir->CreateStore(m_ir->CreateAdd(m_ir->CreateLoad(get_type(), pbfail), m_ir->getInt64(1)), pbfail); + const auto dispci = call("spu_dispatch", spu_runtime::tr_dispatch, m_thread, m_lsptr, main_arg2); + dispci->setCallingConv(CallingConv::GHC); + dispci->setTailCall(); + m_ir->CreateRetVoid(); + } + else + { + m_ir->CreateUnreachable(); + } + + m_dispatch = cast(_module->getOrInsertFunction("__spu-null", entry_chunk->chunk->getFunctionType()).getCallee()); + m_dispatch->setLinkage(llvm::GlobalValue::InternalLinkage); + m_dispatch->setCallingConv(entry_chunk->chunk->getCallingConv()); + set_function(m_dispatch); + + if (entry_chunk->chunk->getReturnType() == get_type()) + { + const auto next_call = m_ir->CreateCall(main_func->getFunctionType(), dispatcher, {m_thread, m_lsptr, m_ir->getInt64(0)}); + next_call->setCallingConv(main_func->getCallingConv()); + next_call->setTailCall(); + m_ir->CreateRetVoid(); + } + else + { + m_ir->CreateRet(dispatcher); + } + + // Function that executes check_state and escapes if necessary + m_test_state = llvm::cast(m_module->getOrInsertFunction("spu_test_state", get_ftype()).getCallee()); + m_test_state->setLinkage(GlobalValue::InternalLinkage); +#ifdef ARCH_ARM64 + // LLVM doesn't support PreserveAll on arm64. + m_test_state->setCallingConv(CallingConv::PreserveMost); +#else + m_test_state->setCallingConv(CallingConv::PreserveAll); +#endif + m_ir->SetInsertPoint(BasicBlock::Create(m_context, "", m_test_state)); + const auto escape_yes = BasicBlock::Create(m_context, "", m_test_state); + const auto escape_no = BasicBlock::Create(m_context, "", m_test_state); + m_ir->CreateCondBr(call("spu_exec_check_state", &exec_check_state, m_test_state->getArg(0)), escape_yes, escape_no); + m_ir->SetInsertPoint(escape_yes); + call("spu_escape", spu_runtime::g_escape, m_test_state->getArg(0)); + m_ir->CreateRetVoid(); + m_ir->SetInsertPoint(escape_no); + m_ir->CreateRetVoid(); + + // Create function table (uninitialized) + m_function_table = new llvm::GlobalVariable(*m_module, llvm::ArrayType::get(entry_chunk->chunk->getType(), m_size / 4), true, llvm::GlobalValue::InternalLinkage, nullptr); + + // Create function chunks + for (usz fi = 0; fi < m_function_queue.size(); fi++) + { + // Initialize function info + m_entry = m_function_queue[fi]; + set_function(m_functions[m_entry].chunk); + + // Set block hash for profiling (if enabled) + if (g_cfg.core.spu_prof) + m_ir->CreateStore(m_ir->getInt64((m_hash_start & -65536) | (m_entry >> 2)), spu_ptr(&spu_thread::block_hash)); + + m_finfo = &m_functions[m_entry]; + m_ir->CreateBr(add_block(m_entry)); + + // Emit instructions for basic blocks + for (usz bi = 0; bi < m_block_queue.size(); bi++) + { + // Initialize basic block info + const u32 baddr = m_block_queue[bi]; + m_block = &m_blocks[baddr]; + m_ir->SetInsertPoint(m_block->block); + auto& bb = ::at32(m_bbs, baddr); + bool need_check = false; + m_block->bb = &bb; + + if (!bb.preds.empty()) + { + // Initialize registers and build PHI nodes if necessary + for (u32 i = 0; i < s_reg_max; i++) + { + const u32 src = m_finfo->fn ? bb.reg_origin_abs[i] : bb.reg_origin[i]; + + if (src > 0x40000) + { + // Use the xfloat hint to create 256-bit (4x double) PHI + llvm::Type* type = g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate && bb.reg_maybe_xf[i] ? get_type() : get_reg_type(i); + + const auto _phi = m_ir->CreatePHI(type, ::size32(bb.preds), fmt::format("phi0x%05x_r%u", baddr, i)); + m_block->phi[i] = _phi; + m_block->reg[i] = _phi; + + for (u32 pred : bb.preds) + { + const auto bfound = m_blocks.find(pred); + + if (bfound != m_blocks.end() && bfound->second.block_end) + { + auto& value = bfound->second.reg[i]; + + if (!value || value->getType() != _phi->getType()) + { + const auto regptr = init_reg_fixed(i); + const auto cblock = m_ir->GetInsertBlock(); + m_ir->SetInsertPoint(bfound->second.block_end->getTerminator()); + + if (!value) + { + // Value hasn't been loaded yet + value = m_finfo && m_finfo->load[i] ? m_finfo->load[i] : m_ir->CreateLoad(get_reg_type(i), regptr); + } + + if (value->getType() == get_type() && type != get_type()) + { + value = double_to_xfloat(value); + } + else if (value->getType() != get_type() && type == get_type()) + { + value = xfloat_to_double(bitcast(value)); + } + else + { + value = bitcast(value, _phi->getType()); + } + + m_ir->SetInsertPoint(cblock); + + ensure(bfound->second.block_end->getTerminator()); + } + + _phi->addIncoming(value, bfound->second.block_end); + } + } + + if (baddr == m_entry) + { + // Load value at the function chunk's entry block if necessary + const auto regptr = init_reg_fixed(i); + const auto cblock = m_ir->GetInsertBlock(); + m_ir->SetInsertPoint(m_function->getEntryBlock().getTerminator()); + const auto value = m_finfo && m_finfo->load[i] ? m_finfo->load[i] : m_ir->CreateLoad(get_reg_type(i), regptr); + m_ir->SetInsertPoint(cblock); + _phi->addIncoming(value, &m_function->getEntryBlock()); + } + } + else if (src < 0x40000) + { + // Passthrough register value + const auto bfound = m_blocks.find(src); + + if (bfound != m_blocks.end()) + { + m_block->reg[i] = bfound->second.reg[i]; + } + else + { + spu_log.error("[0x%05x] Value not found ($%u from 0x%05x)", baddr, i, src); + } + } + else + { + m_block->reg[i] = m_finfo->load[i]; + } + } + + // Emit state check if necessary (TODO: more conditions) + for (u32 pred : bb.preds) + { + if (pred >= baddr) + { + // If this block is a target of a backward branch (possibly loop), emit a check + need_check = true; + break; + } + } + } + + // State check at the beginning of the chunk + if (need_check || (bi == 0 && g_cfg.core.spu_block_size != spu_block_size_type::safe)) + { + check_state(baddr); + } + + // Emit instructions + for (m_pos = baddr; m_pos >= start && m_pos < end && !m_ir->GetInsertBlock()->getTerminator(); m_pos += 4) + { + if (m_pos != baddr && m_block_info[m_pos / 4]) + { + break; + } + + const u32 op = std::bit_cast>(func.data[(m_pos - start) / 4]); + + if (!op) + { + spu_log.error("[%s] Unexpected fallthrough to 0x%x (chunk=0x%x, entry=0x%x)", m_hash, m_pos, m_entry, m_function_queue[0]); + break; + } + + // Set variable for set_link() + if (m_pos + 4 >= end) + m_next_op = 0; + else + m_next_op = func.data[(m_pos - start) / 4 + 1]; + + // Execute recompiler function (TODO) + (this->*decode(op))({op}); + } + + // Finalize block with fallthrough if necessary + if (!m_ir->GetInsertBlock()->getTerminator()) + { + const u32 target = m_pos == baddr ? baddr : m_pos & 0x3fffc; + + if (m_pos != baddr) + { + m_pos -= 4; + + if (target >= start && target < end) + { + const auto tfound = m_targets.find(m_pos); + + if (tfound == m_targets.end() || tfound->second.find_first_of(target) + 1 == 0) + { + spu_log.error("[%s] Unregistered fallthrough to 0x%x (chunk=0x%x, entry=0x%x)", m_hash, target, m_entry, m_function_queue[0]); + } + } + } + + m_block->block_end = m_ir->GetInsertBlock(); + m_ir->CreateBr(add_block(target)); + } + + ensure(m_block->block_end); + } + + // Work on register stores. + // 1. Remove stores which are overwritten later. + // 2. Sink stores to post-dominating blocks. + llvm::PostDominatorTree pdt(*m_function); + llvm::DominatorTree dt(*m_function); + + // Post-order indices + std::unordered_map pois; + { + usz i = 0; + for (auto* bb : llvm::post_order(m_function)) + pois[bb] = i++; + } + + // Basic block to block_info + std::unordered_map bb_to_info; + + std::vector block_q; + block_q.reserve(m_blocks.size()); + for (auto& [a, b] : m_blocks) + { + block_q.emplace_back(&b); + bb_to_info[b.block] = &b; + } + + for (usz bi = 0; bi < block_q.size();) + { + auto bqbi = block_q[bi++]; + + // TODO: process all registers up to s_reg_max + for (u32 i = 0; i < 128; i++) + { + // Check if the store is beyond the last barrier + if (auto& bs = bqbi->store[i]; bs && !bqbi->does_gpr_barrier_proceed_last_store(i)) + { + for (auto& [a, b] : m_blocks) + { + // Check if the store occurs before any barrier in the block + if (b.store[i] && b.store[i] != bs && b.store_context_first_id[i] == 1) + { + if (pdt.dominates(b.store[i], bs)) + { + bs->eraseFromParent(); + bs = nullptr; + break; + } + } + } + + if (!bs) + continue; + + // Set of store instructions which overwrite bs + std::vector killers; + + for (auto& [a, b] : m_blocks) + { + const auto si = b.store[i]; + + if (si && si != bs) + { + if (pois[bs->getParent()] > pois[si->getParent()]) + { + killers.emplace_back(si->getParent()); + } + else + { + // Reset: store is not the first in the set + killers.clear(); + break; + } + } + } + + if (killers.empty()) + continue; + + // Find nearest common post-dominator + llvm::BasicBlock* common_pdom = killers[0]; + for (auto* bbb : llvm::drop_begin(killers)) + { + if (!common_pdom) + break; + common_pdom = pdt.findNearestCommonDominator(common_pdom, bbb); + } + + // Shortcut + if (!pdt.dominates(common_pdom, bs->getParent())) + common_pdom = nullptr; + + // Look for possibly-dead store in CFG starting from the exit nodes + llvm::SetVector work_list; + std::unordered_map worked_on; + + if (!common_pdom || std::count(killers.begin(), killers.end(), common_pdom) == 0) + { + if (common_pdom) + { + // Shortcut + work_list.insert(common_pdom); + worked_on[common_pdom] = true; + } + else + { + // Check all exits + for (auto* r : pdt.roots()) + { + worked_on[r] = true; + work_list.insert(r); + } + } + } + + // bool flag indicates the presence of a memory barrier before the killer store + std::vector> work2_list; + + for (usz wi = 0; wi < work_list.size(); wi++) + { + auto* cur = work_list[wi]; + if (std::count(killers.begin(), killers.end(), cur)) + { + work2_list.emplace_back(cur, bb_to_info[cur] && bb_to_info[cur]->does_gpr_barrier_preceed_first_store(i)); + continue; + } + + if (cur == bs->getParent()) + { + // Reset: store is not dead + killers.clear(); + break; + } + + for (auto* p : llvm::predecessors(cur)) + { + if (!worked_on[p]) + { + worked_on[p] = true; + work_list.insert(p); + } + } + } + + if (killers.empty()) + continue; + + worked_on.clear(); + + for (usz wi = 0; wi < work2_list.size(); wi++) + { + worked_on[work2_list[wi].first] = true; + } + + // Need to treat tails differently: do not require checking barrier (checked before in a suitable manner) + const usz work_list_tail_blocks_max_index = work2_list.size(); + + for (usz wi = 0; wi < work2_list.size(); wi++) + { + auto [cur, found_user] = work2_list[wi]; + + ensure(cur != bs->getParent()); + + if (!found_user && wi >= work_list_tail_blocks_max_index) + { + if (auto info = bb_to_info[cur]) + { + if (info->store_context_ctr[i] != 1) + { + found_user = true; + } + } + } + + for (auto* p : llvm::predecessors(cur)) + { + if (p == bs->getParent()) + { + if (found_user) + { + // Reset: store is being used and preserved by ensure_gpr_stores() + killers.clear(); + break; + } + + continue; + } + + if (!worked_on[p]) + { + worked_on[p] = true; + work2_list.push_back(std::make_pair(p, found_user)); + } + // Enqueue a second iteration for found_user=true if only found with found_user=false + else if (found_user && !std::find_if(work2_list.rbegin(), work2_list.rend(), [&](auto& it){ return it.first == p; })->second) + { + work2_list.push_back(std::make_pair(p, true)); + } + } + + if (killers.empty()) + { + break; + } + } + + // Finally erase the dead store + if (!killers.empty()) + { + bs->eraseFromParent(); + bs = nullptr; + + // Run the loop from the start + bi = 0; + } + } + } + } + + block_q.clear(); + for (auto& [a, b] : m_blocks) + { + block_q.emplace_back(&b); + } + + for (usz bi = 0; bi < block_q.size(); bi++) + { + for (u32 i = 0; i < 128; i++) + { + // If store isn't erased, try to sink it + if (auto& bs = block_q[bi]->store[i]; bs && block_q[bi]->bb->targets.size() > 1 && !block_q[bi]->does_gpr_barrier_proceed_last_store(i)) + { + std::map> sucs; + + for (u32 tj : block_q[bi]->bb->targets) + { + auto b2it = m_blocks.find(tj); + + if (b2it != m_blocks.end()) + { + sucs.emplace(tj, &b2it->second); + } + } + + for (auto [a2, b2] : sucs) + { + if (b2 != block_q[bi]) + { + auto ins = b2->block->getFirstNonPHI(); + + if (b2->bb->preds.size() == 1) + { + if (!dt.dominates(bs->getOperand(0), ins)) + continue; + if (!pdt.dominates(ins, bs)) + continue; + + m_ir->SetInsertPoint(ins); + auto si = llvm::cast(m_ir->Insert(bs->clone())); + if (b2->store[i] == nullptr) + { + b2->store[i] = si; + b2->store_context_last_id[i] = 0; + + if (!std::count(block_q.begin() + bi, block_q.end(), b2)) + { + // Sunk store can be checked again + block_q.push_back(b2); + } + } + } + else + { + // Initialize additional block between two basic blocks + auto& edge = block_q[bi]->block_edges[a2]; + if (!edge) + { + const auto succ_range = llvm::successors(block_q[bi]->block_end); + + auto succ = b2->block; + + llvm::SmallSetVector succ_q; + succ_q.insert(b2->block); + + for (usz j = 0; j < 32 && j < succ_q.size(); j++) + { + if (!llvm::count(succ_range, (succ = succ_q[j]))) + { + for (auto pred : llvm::predecessors(succ)) + { + succ_q.insert(pred); + } + } + else + { + break; + } + } + + if (!llvm::count(succ_range, succ)) + { + // TODO: figure this out + spu_log.notice("[%s] Failed successor to 0x%05x", fmt::base57(be_t{m_hash_start}), a2); + continue; + } + + edge = llvm::SplitEdge(block_q[bi]->block_end, succ); + pdt.recalculate(*m_function); + dt.recalculate(*m_function); + } + + ins = edge->getTerminator(); + if (!dt.dominates(bs->getOperand(0), ins)) + continue; + if (!pdt.dominates(ins, bs)) + continue; + + m_ir->SetInsertPoint(ins); + m_ir->Insert(bs->clone()); + } + + bs->eraseFromParent(); + bs = nullptr; + + pdt.recalculate(*m_function); + dt.recalculate(*m_function); + break; + } + } + } + } + } + } + + // Create function table if necessary + if (m_function_table->getNumUses()) + { + std::vector chunks; + chunks.reserve(m_size / 4); + + for (u32 i = start; i < end; i += 4) + { + const auto found = m_functions.find(i); + + if (found == m_functions.end()) + { + if (false && g_cfg.core.spu_verification) + { + const std::string ppname = fmt::format("%s-chunkpp-0x%05x", m_hash, i); + m_engine->updateGlobalMapping(ppname, reinterpret_cast(m_spurt->make_branch_patchpoint(i / 4))); + + const auto ppfunc = llvm::cast(m_module->getOrInsertFunction(ppname, m_finfo->chunk->getFunctionType()).getCallee()); + ppfunc->setCallingConv(m_finfo->chunk->getCallingConv()); + + chunks.push_back(ppfunc); + continue; + } + + chunks.push_back(m_dispatch); + continue; + } + + chunks.push_back(found->second.chunk); + } + + m_function_table->setInitializer(llvm::ConstantArray::get(llvm::ArrayType::get(entry_chunk->chunk->getType(), m_size / 4), chunks)); + } + else + { + m_function_table->eraseFromParent(); + } + + // Initialize pass manager + legacy::FunctionPassManager pm(_module.get()); + + // Basic optimizations + pm.add(createEarlyCSEPass()); + pm.add(createCFGSimplificationPass()); + //pm.add(createNewGVNPass()); +#if LLVM_VERSION_MAJOR < 17 + pm.add(createDeadStoreEliminationPass()); +#endif + pm.add(createLICMPass()); +#if LLVM_VERSION_MAJOR < 17 + pm.add(createAggressiveDCEPass()); +#else + pm.add(createDeadCodeEliminationPass()); +#endif + //pm.add(createLintPass()); // Check + + for (auto& f : *m_module) + { + replace_intrinsics(f); + } + + for (const auto& func : m_functions) + { + const auto f = func.second.fn ? func.second.fn : func.second.chunk; + pm.run(*f); + } + + // Clear context (TODO) + m_blocks.clear(); + m_block_queue.clear(); + m_functions.clear(); + m_function_queue.clear(); + m_function_table = nullptr; + + raw_string_ostream out(log); + + if (g_cfg.core.spu_debug) + { + fmt::append(log, "LLVM IR at 0x%x:\n", func.entry_point); + out << *_module; // print IR + out << "\n\n"; + } + + if (verifyModule(*_module, &out)) + { + out.flush(); + spu_log.error("LLVM: Verification failed at 0x%x:\n%s", func.entry_point, log); + + if (g_cfg.core.spu_debug) + { + fs::file(m_spurt->get_cache_path() + "spu-ir.log", fs::write + fs::append).write(log); + } + + fmt::throw_exception("Compilation failed"); + } + +#if defined(__APPLE__) + pthread_jit_write_protect_np(false); +#endif + + if (g_cfg.core.spu_debug) + { + // Testing only + m_jit.add(std::move(_module), m_spurt->get_cache_path() + "llvm/"); + } + else + { + m_jit.add(std::move(_module)); + } + + m_jit.fin(); + + // Register function pointer + const spu_function_t fn = reinterpret_cast(m_jit.get_engine().getPointerToFunction(main_func)); + + // Install unconditionally, possibly replacing existing one from spu_fast + add_loc->compiled = fn; + + // Rebuild trampoline if necessary + if (!m_spurt->rebuild_ubertrampoline(func.data[0])) + { + return nullptr; + } + + add_loc->compiled.notify_all(); + + if (g_cfg.core.spu_debug) + { + out.flush(); + fs::write_file(m_spurt->get_cache_path() + "spu-ir.log", fs::create + fs::write + fs::append, log); + } + +#if defined(__APPLE__) + pthread_jit_write_protect_np(true); +#endif +#if defined(ARCH_ARM64) + // Flush all cache lines after potentially writing executable code + asm("ISB"); + asm("DSB ISH"); +#endif + + if (g_fxo->get().operator bool()) + { + spu_log.success("New block compiled successfully"); + } + + return fn; + } + + static void interp_check(spu_thread* _spu, bool after) + { + static thread_local std::array s_gpr; + + if (!after) + { + // Preserve reg state + s_gpr = _spu->gpr; + + // Execute interpreter instruction + const u32 op = *reinterpret_cast*>(_spu->_ptr(0) + _spu->pc); + if (!g_fxo->get().decode(op)(*_spu, {op})) + spu_log.fatal("Bad instruction"); + + // Swap state + for (u32 i = 0; i < s_gpr.size(); ++i) + std::swap(_spu->gpr[i], s_gpr[i]); + } + else + { + // Check saved state + for (u32 i = 0; i < s_gpr.size(); ++i) + { + if (_spu->gpr[i] != s_gpr[i]) + { + spu_log.fatal("Register mismatch: $%u\n%s\n%s", i, _spu->gpr[i], s_gpr[i]); + _spu->state += cpu_flag::dbg_pause; + } + } + } + } + + spu_function_t compile_interpreter() + { + using namespace llvm; + + m_engine->clearAllGlobalMappings(); + + // Create LLVM module + std::unique_ptr _module = std::make_unique("spu_interpreter.obj", m_context); + _module->setTargetTriple(jit_compiler::triple2()); + _module->setDataLayout(m_jit.get_engine().getTargetMachine()->createDataLayout()); + m_module = _module.get(); + + // Initialize IR Builder + IRBuilder<> irb(m_context); + m_ir = &irb; + + // Create interpreter table + const auto if_type = get_ftype(); + m_function_table = new GlobalVariable(*m_module, ArrayType::get(if_type->getPointerTo(), 1ull << m_interp_magn), true, GlobalValue::InternalLinkage, nullptr); + + // Add return function + const auto ret_func = cast(_module->getOrInsertFunction("spu_ret", if_type).getCallee()); + ret_func->setCallingConv(CallingConv::GHC); + ret_func->setLinkage(GlobalValue::InternalLinkage); + m_ir->SetInsertPoint(BasicBlock::Create(m_context, "", ret_func)); + m_thread = ret_func->getArg(1); + m_interp_pc = ret_func->getArg(2); + m_ir->CreateRetVoid(); + + // Add entry function, serves as a trampoline + const auto main_func = llvm::cast(m_module->getOrInsertFunction("spu_interpreter", get_ftype()).getCallee()); +#ifdef _WIN32 + main_func->setCallingConv(CallingConv::Win64); +#endif + set_function(main_func); + + // Load pc and opcode + m_interp_pc = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::pc)); + m_interp_op = m_ir->CreateLoad(get_type(), m_ir->CreateGEP(get_type(), m_lsptr, m_ir->CreateZExt(m_interp_pc, get_type()))); + m_interp_op = m_ir->CreateCall(get_intrinsic(Intrinsic::bswap), {m_interp_op}); + + // Pinned constant, address of interpreter table + m_interp_table = m_ir->CreateGEP(m_function_table->getValueType(), m_function_table, {m_ir->getInt64(0), m_ir->getInt64(0)}); + + // Pinned constant, mask for shifted register index + m_interp_7f0 = m_ir->getInt32(0x7f0); + + // Pinned constant, address of first register + m_interp_regs = _ptr(m_thread, get_reg_offset(0)); + + // Save host thread's stack pointer + const auto native_sp = spu_ptr(&spu_thread::saved_native_sp); +#if defined(ARCH_X64) + const auto rsp_name = MetadataAsValue::get(m_context, MDNode::get(m_context, {MDString::get(m_context, "rsp")})); +#elif defined(ARCH_ARM64) + const auto rsp_name = MetadataAsValue::get(m_context, MDNode::get(m_context, {MDString::get(m_context, "sp")})); +#endif + m_ir->CreateStore(m_ir->CreateCall(get_intrinsic(Intrinsic::read_register), {rsp_name}), native_sp); + + // Decode (shift) and load function pointer + const auto first = m_ir->CreateLoad(if_type->getPointerTo(), m_ir->CreateGEP(if_type->getPointerTo(), m_interp_table, m_ir->CreateLShr(m_interp_op, 32u - m_interp_magn))); + const auto call0 = m_ir->CreateCall(if_type, first, {m_lsptr, m_thread, m_interp_pc, m_interp_op, m_interp_table, m_interp_7f0, m_interp_regs}); + call0->setCallingConv(CallingConv::GHC); + m_ir->CreateRetVoid(); + + // Create helper globals + { + std::vector float_to; + std::vector to_float; + float_to.reserve(256); + to_float.reserve(256); + + for (int i = 0; i < 256; ++i) + { + float_to.push_back(ConstantFP::get(get_type(), std::exp2(173 - i))); + to_float.push_back(ConstantFP::get(get_type(), std::exp2(i - 155))); + } + + const auto atype = ArrayType::get(get_type(), 256); + m_scale_float_to = new GlobalVariable(*m_module, atype, true, GlobalValue::InternalLinkage, ConstantArray::get(atype, float_to)); + m_scale_to_float = new GlobalVariable(*m_module, atype, true, GlobalValue::InternalLinkage, ConstantArray::get(atype, to_float)); + } + + // Fill interpreter table + std::array ifuncs{}; + std::vector iptrs; + iptrs.reserve(1ull << m_interp_magn); + + m_block = nullptr; + + auto last_itype = spu_itype::type{255}; + + for (u32 i = 0; i < 1u << m_interp_magn;) + { + // Fake opcode + const u32 op = i << (32u - m_interp_magn); + + // Instruction type + const auto itype = g_spu_itype.decode(op); + + // Function name + std::string fname = fmt::format("spu_%s", g_spu_iname.decode(op)); + + if (last_itype != itype) + { + // Trigger automatic information collection (probing) + m_op_const_mask = 0; + } + else + { + // Inject const mask into function name + fmt::append(fname, "_%X", (i & (m_op_const_mask >> (32u - m_interp_magn))) | (1u << m_interp_magn)); + } + + // Decode instruction name, access function + const auto f = cast(_module->getOrInsertFunction(fname, if_type).getCallee()); + + // Build if necessary + if (f->empty()) + { + if (last_itype != itype) + { + ifuncs[static_cast(itype)] = f; + } + + f->setCallingConv(CallingConv::GHC); + + m_function = f; + m_lsptr = f->getArg(0); + m_thread = f->getArg(1); + m_interp_pc = f->getArg(2); + m_interp_op = f->getArg(3); + m_interp_table = f->getArg(4); + m_interp_7f0 = f->getArg(5); + m_interp_regs = f->getArg(6); + + m_ir->SetInsertPoint(BasicBlock::Create(m_context, "", f)); + m_memptr = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::memory_base_addr)); + + switch (itype) + { + case spu_itype::UNK: + case spu_itype::DFCEQ: + case spu_itype::DFCMEQ: + case spu_itype::DFCGT: + case spu_itype::DFCMGT: + case spu_itype::DFTSV: + case spu_itype::STOP: + case spu_itype::STOPD: + case spu_itype::RDCH: + case spu_itype::WRCH: + { + // Invalid or abortable instruction. Save current address. + m_ir->CreateStore(m_interp_pc, spu_ptr(&spu_thread::pc)); + [[fallthrough]]; + } + default: + { + break; + } + } + + { + m_interp_bblock = nullptr; + + // Next instruction (no wraparound at the end of LS) + m_interp_pc_next = m_ir->CreateAdd(m_interp_pc, m_ir->getInt32(4)); + + bool check = false; + + if (itype == spu_itype::WRCH || + itype == spu_itype::RDCH || + itype == spu_itype::RCHCNT || + itype == spu_itype::STOP || + itype == spu_itype::STOPD || + itype & spu_itype::floating || + itype & spu_itype::branch) + { + check = false; + } + + if (itype & spu_itype::branch) + { + // Instruction changes pc - change order. + (this->*decode(op))({op}); + + if (m_interp_bblock) + { + m_ir->SetInsertPoint(m_interp_bblock); + m_interp_bblock = nullptr; + } + } + + if (!m_ir->GetInsertBlock()->getTerminator()) + { + if (check) + { + m_ir->CreateStore(m_interp_pc, spu_ptr(&spu_thread::pc)); + } + + // Decode next instruction. + const auto next_pc = itype & spu_itype::branch ? m_interp_pc : m_interp_pc_next; + const auto be32_op = m_ir->CreateLoad(get_type(), m_ir->CreateGEP(get_type(), m_lsptr, m_ir->CreateZExt(next_pc, get_type()))); + const auto next_op = m_ir->CreateCall(get_intrinsic(Intrinsic::bswap), {be32_op}); + const auto next_if = m_ir->CreateLoad(if_type->getPointerTo(), m_ir->CreateGEP(if_type->getPointerTo(), m_interp_table, m_ir->CreateLShr(next_op, 32u - m_interp_magn))); + llvm::cast(next_if)->setVolatile(true); + + if (!(itype & spu_itype::branch)) + { + if (check) + { + call("spu_interp_check", &interp_check, m_thread, m_ir->getFalse()); + } + + // Normal instruction. + (this->*decode(op))({op}); + + if (check && !m_ir->GetInsertBlock()->getTerminator()) + { + call("spu_interp_check", &interp_check, m_thread, m_ir->getTrue()); + } + + m_interp_pc = m_interp_pc_next; + } + + if (last_itype != itype) + { + // Reset to discard dead code + llvm::cast(next_if)->setVolatile(false); + + if (itype & spu_itype::branch) + { + const auto _stop = BasicBlock::Create(m_context, "", f); + const auto _next = BasicBlock::Create(m_context, "", f); + m_ir->CreateCondBr(m_ir->CreateIsNotNull(m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::state))), _stop, _next, m_md_unlikely); + m_ir->SetInsertPoint(_stop); + m_ir->CreateStore(m_interp_pc, spu_ptr(&spu_thread::pc)); + + const auto escape_yes = BasicBlock::Create(m_context, "", f); + const auto escape_no = BasicBlock::Create(m_context, "", f); + m_ir->CreateCondBr(call("spu_exec_check_state", &exec_check_state, m_thread), escape_yes, escape_no); + m_ir->SetInsertPoint(escape_yes); + call("spu_escape", spu_runtime::g_escape, m_thread); + m_ir->CreateBr(_next); + m_ir->SetInsertPoint(escape_no); + m_ir->CreateBr(_next); + m_ir->SetInsertPoint(_next); + } + + llvm::Value* fret = m_interp_table; + + if (itype == spu_itype::WRCH || + itype == spu_itype::RDCH || + itype == spu_itype::RCHCNT || + itype == spu_itype::STOP || + itype == spu_itype::STOPD || + itype == spu_itype::UNK || + itype == spu_itype::DFCMEQ || + itype == spu_itype::DFCMGT || + itype == spu_itype::DFCGT || + itype == spu_itype::DFCEQ || + itype == spu_itype::DFTSV) + { + m_interp_7f0 = m_ir->getInt32(0x7f0); + m_interp_regs = _ptr(m_thread, get_reg_offset(0)); + fret = ret_func; + } + else if (!(itype & spu_itype::branch)) + { + // Hack: inline ret instruction before final jmp; this is not reliable. +#ifdef ARCH_X64 + m_ir->CreateCall(InlineAsm::get(get_ftype(), "ret", "", true, false, InlineAsm::AD_Intel)); +#else + m_ir->CreateCall(InlineAsm::get(get_ftype(), "ret", "", true, false)); +#endif + fret = ret_func; + } + + const auto arg3 = UndefValue::get(get_type()); + const auto _ret = m_ir->CreateCall(if_type, fret, {m_lsptr, m_thread, m_interp_pc, arg3, m_interp_table, m_interp_7f0, m_interp_regs}); + _ret->setCallingConv(CallingConv::GHC); + _ret->setTailCall(); + m_ir->CreateRetVoid(); + } + + if (!m_ir->GetInsertBlock()->getTerminator()) + { + // Call next instruction. + const auto _stop = BasicBlock::Create(m_context, "", f); + const auto _next = BasicBlock::Create(m_context, "", f); + m_ir->CreateCondBr(m_ir->CreateIsNotNull(m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::state))), _stop, _next, m_md_unlikely); + m_ir->SetInsertPoint(_next); + + if (itype == spu_itype::WRCH || + itype == spu_itype::RDCH || + itype == spu_itype::RCHCNT || + itype == spu_itype::STOP || + itype == spu_itype::STOPD) + { + m_interp_7f0 = m_ir->getInt32(0x7f0); + m_interp_regs = _ptr(m_thread, get_reg_offset(0)); + } + + const auto ncall = m_ir->CreateCall(if_type, next_if, {m_lsptr, m_thread, m_interp_pc, next_op, m_interp_table, m_interp_7f0, m_interp_regs}); + ncall->setCallingConv(CallingConv::GHC); + ncall->setTailCall(); + m_ir->CreateRetVoid(); + m_ir->SetInsertPoint(_stop); + m_ir->CreateStore(m_interp_pc, spu_ptr(&spu_thread::pc)); + call("spu_escape", spu_runtime::g_escape, m_thread)->setTailCall(); + m_ir->CreateRetVoid(); + } + } + } + } + + if (last_itype != itype && g_cfg.core.spu_decoder != spu_decoder_type::llvm) + { + // Repeat after probing + last_itype = itype; + } + else + { + // Add to the table + iptrs.push_back(f); + i++; + } + } + + m_function_table->setInitializer(ConstantArray::get(ArrayType::get(if_type->getPointerTo(), 1ull << m_interp_magn), iptrs)); + m_function_table = nullptr; + + // Initialize pass manager + legacy::FunctionPassManager pm(_module.get()); + + // Basic optimizations + pm.add(createEarlyCSEPass()); + pm.add(createCFGSimplificationPass()); +#if LLVM_VERSION_MAJOR < 17 + pm.add(createDeadStoreEliminationPass()); + pm.add(createAggressiveDCEPass()); +#else + pm.add(createDeadCodeEliminationPass()); +#endif + //pm.add(createLintPass()); + + for (auto& f : *_module) + { + replace_intrinsics(f); + //pm.run(f); + } + + std::string log; + + raw_string_ostream out(log); + + if (g_cfg.core.spu_debug) + { + fmt::append(log, "LLVM IR (interpreter):\n"); + out << *_module; // print IR + out << "\n\n"; + } + + if (verifyModule(*_module, &out)) + { + out.flush(); + spu_log.error("LLVM: Verification failed:\n%s", log); + + if (g_cfg.core.spu_debug) + { + fs::write_file(m_spurt->get_cache_path() + "spu-ir.log", fs::create + fs::write + fs::append, log); + } + + fmt::throw_exception("Compilation failed"); + } + + if (g_cfg.core.spu_debug) + { + // Testing only + m_jit.add(std::move(_module), m_spurt->get_cache_path() + "llvm/"); + } + else + { + m_jit.add(std::move(_module)); + } + + m_jit.fin(); + + // Register interpreter entry point + spu_runtime::g_interpreter = reinterpret_cast(m_jit.get_engine().getPointerToFunction(main_func)); + + for (u32 i = 0; i < spu_runtime::g_interpreter_table.size(); i++) + { + // Fill exported interpreter table + spu_runtime::g_interpreter_table[i] = ifuncs[i] ? reinterpret_cast(m_jit.get_engine().getPointerToFunction(ifuncs[i])) : 0; + } + + if (!spu_runtime::g_interpreter) + { + return nullptr; + } + + if (g_cfg.core.spu_debug) + { + out.flush(); + fs::write_file(m_spurt->get_cache_path() + "spu-ir.log", fs::create + fs::write + fs::append, log); + } + + return spu_runtime::g_interpreter; + } + + static bool exec_check_state(spu_thread* _spu) + { + return _spu->check_state(); + } + + template + static void exec_fall(spu_thread* _spu, spu_opcode_t op) + { + if (F(*_spu, op)) + { + _spu->pc += 4; + } + } + + template + void fall(spu_opcode_t op) + { + std::string name = fmt::format("spu_%s", g_spu_iname.decode(op.opcode)); + + if (m_interp_magn) + { + call(name, F, m_thread, m_interp_op); + return; + } + + update_pc(); + call(name, &exec_fall, m_thread, m_ir->getInt32(op.opcode)); + } + + [[noreturn]] static void exec_unk(spu_thread*, u32 op) + { + fmt::throw_exception("Unknown/Illegal instruction (0x%08x)", op); + } + + void UNK(spu_opcode_t op_unk) + { + if (m_interp_magn) + { + m_ir->CreateStore(m_interp_pc, spu_ptr(&spu_thread::pc)); + call("spu_unknown", &exec_unk, m_thread, m_ir->getInt32(op_unk.opcode)); + return; + } + + m_block->block_end = m_ir->GetInsertBlock(); + update_pc(); + call("spu_unknown", &exec_unk, m_thread, m_ir->getInt32(op_unk.opcode)); + } + + static void exec_stop(spu_thread* _spu, u32 code) + { + if (!_spu->stop_and_signal(code) || _spu->state & cpu_flag::again) + { + spu_runtime::g_escape(_spu); + } + + if (_spu->test_stopped()) + { + _spu->pc += 4; + spu_runtime::g_escape(_spu); + } + } + + void STOP(spu_opcode_t op) // + { + if (m_interp_magn) + { + call("spu_syscall", &exec_stop, m_thread, m_ir->CreateAnd(m_interp_op, m_ir->getInt32(0x3fff))); + return; + } + + update_pc(); + call("spu_syscall", &exec_stop, m_thread, m_ir->getInt32(op.opcode & 0x3fff)); + + if (g_cfg.core.spu_block_size == spu_block_size_type::safe) + { + m_block->block_end = m_ir->GetInsertBlock(); + update_pc(m_pos + 4); + ensure_gpr_stores(); + tail_chunk(m_dispatch); + return; + } + } + + void STOPD(spu_opcode_t) // + { + if (m_interp_magn) + { + call("spu_syscall", &exec_stop, m_thread, m_ir->getInt32(0x3fff)); + return; + } + + STOP(spu_opcode_t{0x3fff}); + } + + static u32 exec_rdch(spu_thread* _spu, u32 ch) + { + const s64 result = _spu->get_ch_value(ch); + + if (result < 0 || _spu->state & cpu_flag::again) + { + spu_runtime::g_escape(_spu); + } + + static_cast(_spu->test_stopped()); + return static_cast(result & 0xffffffff); + } + + static u32 exec_read_in_mbox(spu_thread* _spu) + { + // TODO + return exec_rdch(_spu, SPU_RdInMbox); + } + + static u32 exec_read_dec(spu_thread* _spu) + { + const u32 res = _spu->read_dec().first; + + if (res > 1500 && g_cfg.core.spu_loop_detection) + { + _spu->state += cpu_flag::wait; + std::this_thread::yield(); + static_cast(_spu->test_stopped()); + } + + return res; + } + + static u32 exec_read_events(spu_thread* _spu) + { + // TODO + return exec_rdch(_spu, SPU_RdEventStat); + } + + void ensure_gpr_stores() + { + if (m_block) + { + // Make previous stores not able to be reordered beyond this point or be deleted + std::for_each(m_block->store_context_ctr.begin(), m_block->store_context_ctr.end(), FN(x++)); + } + } + + llvm::Value* get_rdch(spu_opcode_t op, u32 off, bool atomic) + { + const auto ptr = _ptr(m_thread, off); + llvm::Value* val0; + + if (atomic) + { + const auto val = m_ir->CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, ptr, m_ir->getInt64(0), llvm::MaybeAlign{8}, llvm::AtomicOrdering::Acquire); + val0 = val; + } + else + { + const auto val = m_ir->CreateLoad(get_type(), ptr); + val->setAtomic(llvm::AtomicOrdering::Acquire); + m_ir->CreateStore(m_ir->getInt64(0), ptr)->setAtomic(llvm::AtomicOrdering::Release); + val0 = val; + } + + const auto _cur = m_ir->GetInsertBlock(); + const auto done = llvm::BasicBlock::Create(m_context, "", m_function); + const auto wait = llvm::BasicBlock::Create(m_context, "", m_function); + const auto cond = m_ir->CreateICmpSLT(val0, m_ir->getInt64(0)); + val0 = m_ir->CreateTrunc(val0, get_type()); + m_ir->CreateCondBr(cond, done, wait); + m_ir->SetInsertPoint(wait); + update_pc(); + const auto val1 = call("spu_read_channel", &exec_rdch, m_thread, m_ir->getInt32(op.ra)); + m_ir->CreateBr(done); + m_ir->SetInsertPoint(done); + const auto rval = m_ir->CreatePHI(get_type(), 2); + rval->addIncoming(val0, _cur); + rval->addIncoming(val1, wait); + return rval; + } + + void RDCH(spu_opcode_t op) // + { + value_t res; + + if (m_interp_magn) + { + res.value = call("spu_read_channel", &exec_rdch, m_thread, get_imm(op.ra).value); + set_vr(op.rt, insert(splat(0), 3, res)); + return; + } + + switch (op.ra) + { + case SPU_RdSRR0: + { + res.value = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::srr0)); + break; + } + case SPU_RdInMbox: + { + update_pc(); + ensure_gpr_stores(); + res.value = call("spu_read_in_mbox", &exec_read_in_mbox, m_thread); + break; + } + case MFC_RdTagStat: + { + res.value = get_rdch(op, ::offset32(&spu_thread::ch_tag_stat), false); + break; + } + case MFC_RdTagMask: + { + res.value = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_tag_mask)); + break; + } + case SPU_RdSigNotify1: + { + update_pc(); + ensure_gpr_stores(); + res.value = get_rdch(op, ::offset32(&spu_thread::ch_snr1), true); + break; + } + case SPU_RdSigNotify2: + { + update_pc(); + ensure_gpr_stores(); + res.value = get_rdch(op, ::offset32(&spu_thread::ch_snr2), true); + break; + } + case MFC_RdAtomicStat: + { + res.value = get_rdch(op, ::offset32(&spu_thread::ch_atomic_stat), false); + break; + } + case MFC_RdListStallStat: + { + res.value = get_rdch(op, ::offset32(&spu_thread::ch_stall_stat), false); + break; + } + case SPU_RdDec: + { + if (utils::get_tsc_freq() && !(g_cfg.core.spu_loop_detection) && (g_cfg.core.clocks_scale == 100)) + { + const auto timestamp = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_dec_start_timestamp)); + const auto dec_value = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_dec_value)); + const auto tsc = m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::x86_rdtsc)); + const auto tscx = m_ir->CreateMul(m_ir->CreateUDiv(tsc, m_ir->getInt64(utils::get_tsc_freq())), m_ir->getInt64(80000000)); + const auto tscm = m_ir->CreateUDiv(m_ir->CreateMul(m_ir->CreateURem(tsc, m_ir->getInt64(utils::get_tsc_freq())), m_ir->getInt64(80000000)), m_ir->getInt64(utils::get_tsc_freq())); + const auto tsctb = m_ir->CreateAdd(tscx, tscm); + + const auto frz = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::is_dec_frozen)); + const auto frzev = m_ir->CreateICmpEQ(frz, m_ir->getInt8(0)); + + const auto delta = m_ir->CreateTrunc(m_ir->CreateSub(tsctb, timestamp), get_type()); + const auto deltax = m_ir->CreateSelect(frzev, delta, m_ir->getInt32(0)); + res.value = m_ir->CreateSub(dec_value, deltax); + break; + } + + res.value = call("spu_read_decrementer", &exec_read_dec, m_thread); + break; + } + case SPU_RdEventMask: + { + const auto value = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_events)); + value->setAtomic(llvm::AtomicOrdering::Acquire); + res.value = m_ir->CreateTrunc(m_ir->CreateLShr(value, 32), get_type()); + break; + } + case SPU_RdEventStat: + { + update_pc(); + + if (g_cfg.savestate.compatible_mode) + { + ensure_gpr_stores(); + } + else + { + m_ir->CreateStore(m_ir->getInt8(1), spu_ptr(&spu_thread::unsavable)); + } + + res.value = call("spu_read_events", &exec_read_events, m_thread); + + if (!g_cfg.savestate.compatible_mode) + { + m_ir->CreateStore(m_ir->getInt8(0), spu_ptr(&spu_thread::unsavable)); + } + + break; + } + case SPU_RdMachStat: + { + res.value = m_ir->CreateZExt(m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::interrupts_enabled)), get_type()); + res.value = m_ir->CreateOr(res.value, m_ir->CreateAnd(m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::thread_type)), m_ir->getInt32(2))); + break; + } + + default: + { + update_pc(); + ensure_gpr_stores(); + res.value = call("spu_read_channel", &exec_rdch, m_thread, m_ir->getInt32(op.ra)); + break; + } + } + + set_vr(op.rt, insert(splat(0), 3, res)); + } + + static u32 exec_rchcnt(spu_thread* _spu, u32 ch) + { + return _spu->get_ch_count(ch); + } + + static u32 exec_get_events(spu_thread* _spu, u32 mask) + { + return _spu->get_events(mask).count; + } + + llvm::Value* get_rchcnt(u32 off, u64 inv = 0) + { + const auto val = m_ir->CreateLoad(get_type(), _ptr(m_thread, off)); + val->setAtomic(llvm::AtomicOrdering::Acquire); + const auto shv = m_ir->CreateLShr(val, spu_channel::off_count); + return m_ir->CreateTrunc(m_ir->CreateXor(shv, u64{inv}), get_type()); + } + + void RCHCNT(spu_opcode_t op) // + { + value_t res; + + if (m_interp_magn) + { + res.value = call("spu_read_channel_count", &exec_rchcnt, m_thread, get_imm(op.ra).value); + set_vr(op.rt, insert(splat(0), 3, res)); + return; + } + + switch (op.ra) + { + case SPU_WrOutMbox: + { + res.value = get_rchcnt(::offset32(&spu_thread::ch_out_mbox), true); + break; + } + case SPU_WrOutIntrMbox: + { + res.value = get_rchcnt(::offset32(&spu_thread::ch_out_intr_mbox), true); + break; + } + case MFC_RdTagStat: + { + res.value = get_rchcnt(::offset32(&spu_thread::ch_tag_stat)); + break; + } + case MFC_RdListStallStat: + { + res.value = get_rchcnt(::offset32(&spu_thread::ch_stall_stat)); + break; + } + case SPU_RdSigNotify1: + { + res.value = get_rchcnt(::offset32(&spu_thread::ch_snr1)); + break; + } + case SPU_RdSigNotify2: + { + res.value = get_rchcnt(::offset32(&spu_thread::ch_snr2)); + break; + } + case MFC_RdAtomicStat: + { + res.value = get_rchcnt(::offset32(&spu_thread::ch_atomic_stat)); + break; + } + case MFC_WrTagUpdate: + { + res.value = m_ir->getInt32(1); + break; + } + case MFC_Cmd: + { + res.value = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::mfc_size)); + res.value = m_ir->CreateSub(m_ir->getInt32(16), res.value); + break; + } + case SPU_RdInMbox: + { + const auto value = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_in_mbox)); + value->setAtomic(llvm::AtomicOrdering::Acquire); + res.value = value; + res.value = m_ir->CreateLShr(res.value, 8); + res.value = m_ir->CreateAnd(res.value, 7); + break; + } + case SPU_RdEventStat: + { + const auto mask = m_ir->CreateTrunc(m_ir->CreateLShr(m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_events)), 32), get_type()); + res.value = call("spu_get_events", &exec_get_events, m_thread, mask); + break; + } + + // Channels with a constant count of 1: + case SPU_WrEventMask: + case SPU_WrEventAck: + case SPU_WrDec: + case SPU_RdDec: + case SPU_RdEventMask: + case SPU_RdMachStat: + case SPU_WrSRR0: + case SPU_RdSRR0: + case SPU_Set_Bkmk_Tag: + case SPU_PM_Start_Ev: + case SPU_PM_Stop_Ev: + case MFC_RdTagMask: + case MFC_LSA: + case MFC_EAH: + case MFC_EAL: + case MFC_Size: + case MFC_TagID: + case MFC_WrTagMask: + case MFC_WrListStallAck: + { + res.value = m_ir->getInt32(1); + break; + } + + default: + { + res.value = call("spu_read_channel_count", &exec_rchcnt, m_thread, m_ir->getInt32(op.ra)); + break; + } + } + + set_vr(op.rt, insert(splat(0), 3, res)); + } + + static void exec_wrch(spu_thread* _spu, u32 ch, u32 value) + { + if (!_spu->set_ch_value(ch, value) || _spu->state & cpu_flag::again) + { + spu_runtime::g_escape(_spu); + } + + static_cast(_spu->test_stopped()); + } + + static void exec_list_unstall(spu_thread* _spu, u32 tag) + { + for (u32 i = 0; i < _spu->mfc_size; i++) + { + if (_spu->mfc_queue[i].tag == (tag | 0x80)) + { + _spu->mfc_queue[i].tag &= 0x7f; + } + } + + _spu->do_mfc(); + } + + static void exec_mfc_cmd(spu_thread* _spu) + { + if (!_spu->process_mfc_cmd() || _spu->state & cpu_flag::again) + { + spu_runtime::g_escape(_spu); + } + + static_cast(_spu->test_stopped()); + } + + void WRCH(spu_opcode_t op) // + { + const auto val = eval(extract(get_vr(op.rt), 3)); + + if (m_interp_magn) + { + call("spu_write_channel", &exec_wrch, m_thread, get_imm(op.ra).value, val.value); + return; + } + + switch (op.ra) + { + case SPU_WrSRR0: + { + m_ir->CreateStore(eval(val & 0x3fffc).value, spu_ptr(&spu_thread::srr0)); + return; + } + case SPU_WrOutIntrMbox: + { + // TODO + break; + } + case SPU_WrOutMbox: + { + // TODO + break; + } + case MFC_WrTagMask: + { + // TODO + m_ir->CreateStore(val.value, spu_ptr(&spu_thread::ch_tag_mask)); + const auto next = llvm::BasicBlock::Create(m_context, "", m_function); + const auto _mfc = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->CreateCondBr(m_ir->CreateICmpNE(m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_tag_upd)), m_ir->getInt32(MFC_TAG_UPDATE_IMMEDIATE)), _mfc, next); + m_ir->SetInsertPoint(_mfc); + update_pc(); + call("spu_write_channel", &exec_wrch, m_thread, m_ir->getInt32(op.ra), val.value); + m_ir->CreateBr(next); + m_ir->SetInsertPoint(next); + return; + } + case MFC_WrTagUpdate: + { + if (true) + { + const auto tag_mask = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_tag_mask)); + const auto mfc_fence = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::mfc_fence)); + const auto completed = m_ir->CreateAnd(tag_mask, m_ir->CreateNot(mfc_fence)); + const auto upd_ptr = spu_ptr(&spu_thread::ch_tag_upd); + const auto stat_ptr = spu_ptr(&spu_thread::ch_tag_stat); + const auto stat_val = m_ir->CreateOr(m_ir->CreateZExt(completed, get_type()), s64{smin}); + + const auto next = llvm::BasicBlock::Create(m_context, "", m_function); + const auto next0 = llvm::BasicBlock::Create(m_context, "", m_function); + const auto imm = llvm::BasicBlock::Create(m_context, "", m_function); + const auto any = llvm::BasicBlock::Create(m_context, "", m_function); + const auto fail = llvm::BasicBlock::Create(m_context, "", m_function); + const auto update = llvm::BasicBlock::Create(m_context, "", m_function); + + m_ir->CreateCondBr(m_ir->CreateICmpEQ(val.value, m_ir->getInt32(MFC_TAG_UPDATE_IMMEDIATE)), imm, next0); + m_ir->SetInsertPoint(imm); + m_ir->CreateStore(val.value, upd_ptr); + m_ir->CreateStore(stat_val, stat_ptr); + m_ir->CreateBr(next); + m_ir->SetInsertPoint(next0); + m_ir->CreateCondBr(m_ir->CreateICmpULE(val.value, m_ir->getInt32(MFC_TAG_UPDATE_ALL)), any, fail, m_md_likely); + + // Illegal update, access violate with special address + m_ir->SetInsertPoint(fail); + const auto ptr = _ptr(m_memptr, 0xffdead04); + m_ir->CreateStore(m_ir->getInt32("TAG\0"_u32), ptr); + m_ir->CreateBr(next); + + m_ir->SetInsertPoint(any); + const auto cond = m_ir->CreateSelect(m_ir->CreateICmpEQ(val.value, m_ir->getInt32(MFC_TAG_UPDATE_ANY)) + , m_ir->CreateICmpNE(completed, m_ir->getInt32(0)), m_ir->CreateICmpEQ(completed, tag_mask)); + + m_ir->CreateStore(m_ir->CreateSelect(cond, m_ir->getInt32(MFC_TAG_UPDATE_IMMEDIATE), val.value), upd_ptr); + m_ir->CreateCondBr(cond, update, next, m_md_likely); + m_ir->SetInsertPoint(update); + m_ir->CreateStore(stat_val, stat_ptr); + m_ir->CreateBr(next); + m_ir->SetInsertPoint(next); + return; + } + } + case MFC_LSA: + { + set_reg_fixed(s_reg_mfc_lsa, val.value); + return; + } + case MFC_EAH: + { + if (auto ci = llvm::dyn_cast(val.value)) + { + if (ci->getZExtValue() == 0) + { + return; + } + } + + spu_log.warning("[0x%x] MFC_EAH: $%u is not a zero constant", m_pos, +op.rt); + //m_ir->CreateStore(val.value, spu_ptr(&spu_thread::ch_mfc_cmd, &spu_mfc_cmd::eah)); + return; + } + case MFC_EAL: + { + set_reg_fixed(s_reg_mfc_eal, val.value); + return; + } + case MFC_Size: + { + set_reg_fixed(s_reg_mfc_size, trunc(val).eval(m_ir)); + return; + } + case MFC_TagID: + { + set_reg_fixed(s_reg_mfc_tag, trunc(val & 0x1f).eval(m_ir)); + return; + } + case MFC_Cmd: + { + // Prevent store elimination (TODO) + m_block->store_context_ctr[s_reg_mfc_eal]++; + m_block->store_context_ctr[s_reg_mfc_lsa]++; + m_block->store_context_ctr[s_reg_mfc_tag]++; + m_block->store_context_ctr[s_reg_mfc_size]++; + + if (auto ci = llvm::dyn_cast(trunc(val).eval(m_ir))) + { + if (g_cfg.core.mfc_debug) + { + break; + } + + bool must_use_cpp_functions = !!g_cfg.core.spu_accurate_dma; + + if (u64 cmdh = ci->getZExtValue() & ~(MFC_BARRIER_MASK | MFC_FENCE_MASK | MFC_RESULT_MASK); g_cfg.core.rsx_fifo_accuracy || g_cfg.video.strict_rendering_mode || !g_use_rtm) + { + // TODO: don't require TSX (current implementation is TSX-only) + if (cmdh == MFC_PUT_CMD || cmdh == MFC_SNDSIG_CMD) + { + must_use_cpp_functions = true; + } + } + + const auto eal = get_reg_fixed(s_reg_mfc_eal); + const auto lsa = get_reg_fixed(s_reg_mfc_lsa); + const auto tag = get_reg_fixed(s_reg_mfc_tag); + + const auto size = get_reg_fixed(s_reg_mfc_size); + const auto mask = m_ir->CreateShl(m_ir->getInt32(1), zext(tag).eval(m_ir)); + const auto exec = llvm::BasicBlock::Create(m_context, "", m_function); + const auto fail = llvm::BasicBlock::Create(m_context, "", m_function); + const auto next = llvm::BasicBlock::Create(m_context, "", m_function); + + const auto pf = spu_ptr(&spu_thread::mfc_fence); + const auto pb = spu_ptr(&spu_thread::mfc_barrier); + + switch (u64 cmd = ci->getZExtValue()) + { + case MFC_SDCRT_CMD: + case MFC_SDCRTST_CMD: + { + return; + } + case MFC_PUTL_CMD: + case MFC_PUTLB_CMD: + case MFC_PUTLF_CMD: + case MFC_PUTRL_CMD: + case MFC_PUTRLB_CMD: + case MFC_PUTRLF_CMD: + case MFC_GETL_CMD: + case MFC_GETLB_CMD: + case MFC_GETLF_CMD: + { + ensure_gpr_stores(); + [[fallthrough]]; + } + case MFC_SDCRZ_CMD: + case MFC_GETLLAR_CMD: + case MFC_PUTLLC_CMD: + case MFC_PUTLLUC_CMD: + case MFC_PUTQLLUC_CMD: + { + // TODO + m_ir->CreateBr(next); + m_ir->SetInsertPoint(exec); + m_ir->CreateUnreachable(); + m_ir->SetInsertPoint(fail); + m_ir->CreateUnreachable(); + m_ir->SetInsertPoint(next); + m_ir->CreateStore(ci, spu_ptr(&spu_thread::ch_mfc_cmd, &spu_mfc_cmd::cmd)); + update_pc(); + call("spu_exec_mfc_cmd", &exec_mfc_cmd, m_thread); + return; + } + case MFC_SNDSIG_CMD: + case MFC_SNDSIGB_CMD: + case MFC_SNDSIGF_CMD: + case MFC_PUT_CMD: + case MFC_PUTB_CMD: + case MFC_PUTF_CMD: + case MFC_PUTR_CMD: + case MFC_PUTRB_CMD: + case MFC_PUTRF_CMD: + case MFC_GET_CMD: + case MFC_GETB_CMD: + case MFC_GETF_CMD: + { + // Try to obtain constant size + u64 csize = -1; + + if (auto ci = llvm::dyn_cast(size.value)) + { + csize = ci->getZExtValue(); + } + + if (cmd >= MFC_SNDSIG_CMD && csize != 4) + { + csize = -1; + } + + llvm::Value* src = m_ir->CreateGEP(get_type(), m_lsptr, zext(lsa).eval(m_ir)); + llvm::Value* dst = m_ir->CreateGEP(get_type(), m_memptr, zext(eal).eval(m_ir)); + + if (cmd & MFC_GET_CMD) + { + std::swap(src, dst); + } + + llvm::Value* barrier = m_ir->CreateLoad(get_type(), pb); + + if (cmd & (MFC_BARRIER_MASK | MFC_FENCE_MASK)) + { + barrier = m_ir->CreateOr(barrier, m_ir->CreateLoad(get_type(), pf)); + } + + const auto cond = m_ir->CreateIsNull(m_ir->CreateAnd(mask, barrier)); + m_ir->CreateCondBr(cond, exec, fail, m_md_likely); + m_ir->SetInsertPoint(exec); + + const auto copy = llvm::BasicBlock::Create(m_context, "", m_function); + + // Always use interpreter function for MFC debug option + if (!must_use_cpp_functions) + { + const auto mmio = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->CreateCondBr(m_ir->CreateICmpUGE(eal.value, m_ir->getInt32(0xe0000000)), mmio, copy, m_md_unlikely); + m_ir->SetInsertPoint(mmio); + } + + m_ir->CreateStore(ci, spu_ptr(&spu_thread::ch_mfc_cmd, &spu_mfc_cmd::cmd)); + call("spu_exec_mfc_cmd", &exec_mfc_cmd, m_thread); + m_ir->CreateBr(next); + m_ir->SetInsertPoint(copy); + + llvm::Type* vtype = get_type(); + + switch (csize) + { + case 0: + case umax: + { + break; + } + case 1: + { + vtype = get_type(); + break; + } + case 2: + { + vtype = get_type(); + break; + } + case 4: + { + vtype = get_type(); + break; + } + case 8: + { + vtype = get_type(); + break; + } + default: + { + if (csize % 16 || csize > 0x4000) + { + spu_log.error("[0x%x] MFC_Cmd: invalid size %u", m_pos, csize); + } + } + } + + // Check if the LS address is constant and 256 bit aligned + u64 clsa = umax; + + if (auto ci = llvm::dyn_cast(lsa.value)) + { + clsa = ci->getZExtValue(); + } + + u32 stride = 16; + + if (m_use_avx && csize >= 32 && !(clsa % 32)) + { + vtype = get_type(); + stride = 32; + } + + if (csize > 0 && csize <= 16) + { + // Generate single copy operation + m_ir->CreateStore(m_ir->CreateLoad(vtype, src), dst); + } + else if (csize <= stride * 16 && !(csize % 32)) + { + // Generate fixed sequence of copy operations + for (u32 i = 0; i < csize; i += stride) + { + const auto _src = m_ir->CreateGEP(get_type(), src, m_ir->getInt32(i)); + const auto _dst = m_ir->CreateGEP(get_type(), dst, m_ir->getInt32(i)); + if (csize - i < stride) + { + m_ir->CreateStore(m_ir->CreateLoad(get_type(), _src), _dst); + } + else + { + m_ir->CreateAlignedStore(m_ir->CreateAlignedLoad(vtype, _src, llvm::MaybeAlign{16}), _dst, llvm::MaybeAlign{16}); + } + } + } + else if (csize) + { + // TODO + auto spu_memcpy = [](u8* dst, const u8* src, u32 size) + { + std::memcpy(dst, src, size); + }; + call("spu_memcpy", +spu_memcpy, dst, src, zext(size).eval(m_ir)); + } + + // Disable certain thing + m_ir->CreateStore(m_ir->getInt32(0), spu_ptr(&spu_thread::last_faddr)); + m_ir->CreateBr(next); + break; + } + case MFC_BARRIER_CMD: + case MFC_EIEIO_CMD: + case MFC_SYNC_CMD: + { + const auto cond = m_ir->CreateIsNull(m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::mfc_size))); + m_ir->CreateCondBr(cond, exec, fail, m_md_likely); + m_ir->SetInsertPoint(exec); + m_ir->CreateFence(llvm::AtomicOrdering::SequentiallyConsistent); + m_ir->CreateBr(next); + break; + } + default: + { + // TODO + spu_log.error("[0x%x] MFC_Cmd: unknown command (0x%x)", m_pos, cmd); + m_ir->CreateBr(next); + m_ir->SetInsertPoint(exec); + m_ir->CreateUnreachable(); + break; + } + } + + // Fallback: enqueue the command + m_ir->SetInsertPoint(fail); + + // Get MFC slot, redirect to invalid memory address + const auto slot = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::mfc_size)); + const auto off0 = m_ir->CreateAdd(m_ir->CreateMul(slot, m_ir->getInt32(sizeof(spu_mfc_cmd))), m_ir->getInt32(::offset32(&spu_thread::mfc_queue))); + const auto ptr0 = m_ir->CreateGEP(get_type(), m_thread, m_ir->CreateZExt(off0, get_type())); + const auto ptr1 = m_ir->CreateGEP(get_type(), m_memptr, m_ir->getInt64(0xffdeadf0)); + const auto pmfc = m_ir->CreateSelect(m_ir->CreateICmpULT(slot, m_ir->getInt32(16)), ptr0, ptr1); + m_ir->CreateStore(ci, _ptr(pmfc, ::offset32(&spu_mfc_cmd::cmd))); + + switch (u64 cmd = ci->getZExtValue()) + { + case MFC_GETLLAR_CMD: + case MFC_PUTLLC_CMD: + case MFC_PUTLLUC_CMD: + case MFC_PUTQLLUC_CMD: + { + break; + } + case MFC_PUTL_CMD: + case MFC_PUTLB_CMD: + case MFC_PUTLF_CMD: + case MFC_PUTRL_CMD: + case MFC_PUTRLB_CMD: + case MFC_PUTRLF_CMD: + case MFC_GETL_CMD: + case MFC_GETLB_CMD: + case MFC_GETLF_CMD: + { + break; + } + case MFC_SDCRZ_CMD: + { + break; + } + case MFC_SNDSIG_CMD: + case MFC_SNDSIGB_CMD: + case MFC_SNDSIGF_CMD: + case MFC_PUT_CMD: + case MFC_PUTB_CMD: + case MFC_PUTF_CMD: + case MFC_PUTR_CMD: + case MFC_PUTRB_CMD: + case MFC_PUTRF_CMD: + case MFC_GET_CMD: + case MFC_GETB_CMD: + case MFC_GETF_CMD: + { + m_ir->CreateStore(tag.value, _ptr(pmfc, ::offset32(&spu_mfc_cmd::tag))); + m_ir->CreateStore(size.value, _ptr(pmfc, ::offset32(&spu_mfc_cmd::size))); + m_ir->CreateStore(lsa.value, _ptr(pmfc, ::offset32(&spu_mfc_cmd::lsa))); + m_ir->CreateStore(eal.value, _ptr(pmfc, ::offset32(&spu_mfc_cmd::eal))); + m_ir->CreateStore(m_ir->CreateOr(m_ir->CreateLoad(get_type(), pf), mask), pf); + if (cmd & MFC_BARRIER_MASK) + m_ir->CreateStore(m_ir->CreateOr(m_ir->CreateLoad(get_type(), pb), mask), pb); + break; + } + case MFC_BARRIER_CMD: + case MFC_EIEIO_CMD: + case MFC_SYNC_CMD: + { + m_ir->CreateStore(m_ir->getInt32(-1), pb); + m_ir->CreateStore(m_ir->CreateOr(m_ir->CreateLoad(get_type(), pf), mask), pf); + break; + } + default: + { + m_ir->CreateUnreachable(); + break; + } + } + + m_ir->CreateStore(m_ir->CreateAdd(slot, m_ir->getInt32(1)), spu_ptr(&spu_thread::mfc_size)); + m_ir->CreateBr(next); + m_ir->SetInsertPoint(next); + return; + } + + // Fallback to unoptimized WRCH implementation (TODO) + spu_log.warning("[0x%x] MFC_Cmd: $%u is not a constant", m_pos, +op.rt); + break; + } + case MFC_WrListStallAck: + { + const auto mask = eval(splat(1) << (val & 0x1f)); + const auto _ptr = spu_ptr(&spu_thread::ch_stall_mask); + const auto _old = m_ir->CreateLoad(get_type(), _ptr); + const auto _new = m_ir->CreateAnd(_old, m_ir->CreateNot(mask.value)); + m_ir->CreateStore(_new, _ptr); + const auto next = llvm::BasicBlock::Create(m_context, "", m_function); + const auto _mfc = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->CreateCondBr(m_ir->CreateICmpNE(_old, _new), _mfc, next); + m_ir->SetInsertPoint(_mfc); + ensure_gpr_stores(); + update_pc(); + call("spu_list_unstall", &exec_list_unstall, m_thread, eval(val & 0x1f).value); + m_ir->CreateBr(next); + m_ir->SetInsertPoint(next); + return; + } + case SPU_WrDec: + { + call("spu_get_events", &exec_get_events, m_thread, m_ir->getInt32(SPU_EVENT_TM)); + + if (utils::get_tsc_freq() && !(g_cfg.core.spu_loop_detection) && (g_cfg.core.clocks_scale == 100)) + { + const auto tsc = m_ir->CreateCall(get_intrinsic(llvm::Intrinsic::x86_rdtsc)); + const auto tscx = m_ir->CreateMul(m_ir->CreateUDiv(tsc, m_ir->getInt64(utils::get_tsc_freq())), m_ir->getInt64(80000000)); + const auto tscm = m_ir->CreateUDiv(m_ir->CreateMul(m_ir->CreateURem(tsc, m_ir->getInt64(utils::get_tsc_freq())), m_ir->getInt64(80000000)), m_ir->getInt64(utils::get_tsc_freq())); + const auto tsctb = m_ir->CreateAdd(tscx, tscm); + m_ir->CreateStore(tsctb, spu_ptr(&spu_thread::ch_dec_start_timestamp)); + } + else + { + m_ir->CreateStore(call("get_timebased_time", &get_timebased_time), spu_ptr(&spu_thread::ch_dec_start_timestamp)); + } + + m_ir->CreateStore(val.value, spu_ptr(&spu_thread::ch_dec_value)); + m_ir->CreateStore(m_ir->getInt8(0), spu_ptr(&spu_thread::is_dec_frozen)); + return; + } + case SPU_Set_Bkmk_Tag: + case SPU_PM_Start_Ev: + case SPU_PM_Stop_Ev: + { + return; + } + default: break; + } + + update_pc(); + ensure_gpr_stores(); + call("spu_write_channel", &exec_wrch, m_thread, m_ir->getInt32(op.ra), val.value); + } + + void LNOP(spu_opcode_t) // + { + } + + void NOP(spu_opcode_t) // + { + } + + void SYNC(spu_opcode_t) // + { + // This instruction must be used following a store instruction that modifies the instruction stream. + m_ir->CreateFence(llvm::AtomicOrdering::SequentiallyConsistent); + + if (g_cfg.core.spu_block_size == spu_block_size_type::safe && !m_interp_magn) + { + m_block->block_end = m_ir->GetInsertBlock(); + update_pc(m_pos + 4); + tail_chunk(m_dispatch); + } + } + + void DSYNC(spu_opcode_t) // + { + // This instruction forces all earlier load, store, and channel instructions to complete before proceeding. + m_ir->CreateFence(llvm::AtomicOrdering::SequentiallyConsistent); + } + + void MFSPR(spu_opcode_t op) // + { + // Check SPUInterpreter for notes. + set_vr(op.rt, splat(0)); + } + + void MTSPR(spu_opcode_t) // + { + // Check SPUInterpreter for notes. + } + + template + auto mpyh(TA&& a, TB&& b) + { + return bitcast(bitcast((std::forward(a) >> 16)) * bitcast(std::forward(b))) << 16; + } + + template + auto mpyu(TA&& a, TB&& b) + { + return (std::forward(a) << 16 >> 16) * (std::forward(b) << 16 >> 16); + } + + void SF(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.rb) - get_vr(op.ra)); + } + + void OR(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) | get_vr(op.rb)); + } + + void BG(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + set_vr(op.rt, zext(a <= b)); + } + + void SFH(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.rb) - get_vr(op.ra)); + } + + void NOR(spu_opcode_t op) + { + set_vr(op.rt, ~(get_vr(op.ra) | get_vr(op.rb))); + } + + void ABSDB(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + set_vr(op.rt, absd(a, b)); + } + + void ROT(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + set_vr(op.rt, rol(a, b)); + } + + void ROTM(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + + auto minusb = eval(-b); + if (auto [ok, x] = match_expr(b, -match()); ok) + { + minusb = eval(x); + } + + if (auto k = get_known_bits(minusb); !!(k.Zero & 32)) + { + set_vr(op.rt, a >> (minusb & 31)); + return; + } + + set_vr(op.rt, inf_lshr(a, minusb & 63)); + } + + void ROTMA(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + + auto minusb = eval(-b); + if (auto [ok, x] = match_expr(b, -match()); ok) + { + minusb = eval(x); + } + + if (auto k = get_known_bits(minusb); !!(k.Zero & 32)) + { + set_vr(op.rt, a >> (minusb & 31)); + return; + } + + set_vr(op.rt, inf_ashr(a, minusb & 63)); + } + + void SHL(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + + if (auto k = get_known_bits(b); !!(k.Zero & 32)) + { + set_vr(op.rt, a << (b & 31)); + return; + } + + set_vr(op.rt, inf_shl(a, b & 63)); + } + + void ROTH(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + set_vr(op.rt, rol(a, b)); + } + + void ROTHM(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + + auto minusb = eval(-b); + if (auto [ok, x] = match_expr(b, -match()); ok) + { + minusb = eval(x); + } + + if (auto k = get_known_bits(minusb); !!(k.Zero & 16)) + { + set_vr(op.rt, a >> (minusb & 15)); + return; + } + + set_vr(op.rt, inf_lshr(a, minusb & 31)); + } + + void ROTMAH(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + + auto minusb = eval(-b); + if (auto [ok, x] = match_expr(b, -match()); ok) + { + minusb = eval(x); + } + + if (auto k = get_known_bits(minusb); !!(k.Zero & 16)) + { + set_vr(op.rt, a >> (minusb & 15)); + return; + } + + set_vr(op.rt, inf_ashr(a, minusb & 31)); + } + + void SHLH(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + + if (auto k = get_known_bits(b); !!(k.Zero & 16)) + { + set_vr(op.rt, a << (b & 15)); + return; + } + + set_vr(op.rt, inf_shl(a, b & 31)); + } + + void ROTI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto i = get_imm(op.i7, false); + set_vr(op.rt, rol(a, i)); + } + + void ROTMI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto i = get_imm(op.i7, false); + set_vr(op.rt, inf_lshr(a, -i & 63)); + } + + void ROTMAI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto i = get_imm(op.i7, false); + set_vr(op.rt, inf_ashr(a, -i & 63)); + } + + void SHLI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto i = get_imm(op.i7, false); + set_vr(op.rt, inf_shl(a, i & 63)); + } + + void ROTHI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto i = get_imm(op.i7, false); + set_vr(op.rt, rol(a, i)); + } + + void ROTHMI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto i = get_imm(op.i7, false); + set_vr(op.rt, inf_lshr(a, -i & 31)); + } + + void ROTMAHI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto i = get_imm(op.i7, false); + set_vr(op.rt, inf_ashr(a, -i & 31)); + } + + void SHLHI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto i = get_imm(op.i7, false); + set_vr(op.rt, inf_shl(a, i & 31)); + } + + void A(spu_opcode_t op) + { + if (auto [a, b] = match_vrs(op.ra, op.rb); a && b) + { + static const auto MP = match(); + + if (auto [ok, a0, b0, b1, a1] = match_expr(a, mpyh(MP, MP) + mpyh(MP, MP)); ok) + { + if (auto [ok, a2, b2] = match_expr(b, mpyu(MP, MP)); ok && a2.eq(a0, a1) && b2.eq(b0, b1)) + { + // 32-bit multiplication + spu_log.notice("mpy32 in %s at 0x%05x", m_hash, m_pos); + set_vr(op.rt, a0 * b0); + return; + } + } + } + + set_vr(op.rt, get_vr(op.ra) + get_vr(op.rb)); + } + + void AND(spu_opcode_t op) + { + if (match_vr(op.ra, [&](auto a, auto /*MP1*/) + { + if (auto b = match_vr_as(a, op.rb)) + { + set_vr(op.rt, a & b); + return true; + } + + return match_vr(op.rb, [&](auto /*b*/, auto /*MP2*/) + { + set_vr(op.rt, a & get_vr_as(a, op.rb)); + return true; + }); + })) + { + return; + } + + set_vr(op.rt, get_vr(op.ra) & get_vr(op.rb)); + } + + void CG(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + set_vr(op.rt, zext(a + b < a)); + } + + void AH(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) + get_vr(op.rb)); + } + + void NAND(spu_opcode_t op) + { + set_vr(op.rt, ~(get_vr(op.ra) & get_vr(op.rb))); + } + + void AVGB(spu_opcode_t op) + { + set_vr(op.rt, avg(get_vr(op.ra), get_vr(op.rb))); + } + + void GB(spu_opcode_t op) + { + // GFNI trick to extract selected bit from bytes + // By treating the first input as constant, and the second input as variable, + // with only 1 bit set in our constant, gf2p8affineqb will extract that selected bit + // from each byte of the second operand + if (m_use_gfni) + { + const auto a = get_vr(op.ra); + const auto as = zshuffle(a, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 12, 8, 4, 0); + set_vr(op.rt, gf2p8affineqb(build(0x0, 0x0, 0x0, 0x0, 0x01, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x01, 0x0, 0x0, 0x0), as, 0x0)); + return; + } + + const auto a = get_vr(op.ra); + const auto m = zext(bitcast(trunc(a))); + set_vr(op.rt, insert(splat(0), 3, eval(m))); + } + + void GBH(spu_opcode_t op) + { + if (m_use_gfni) + { + const auto a = get_vr(op.ra); + const auto as = zshuffle(a, 16, 16, 16, 16, 16, 16, 16, 16, 14, 12, 10, 8, 6, 4, 2, 0); + set_vr(op.rt, gf2p8affineqb(build(0x0, 0x0, 0x0, 0x0, 0x01, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x01, 0x0, 0x0, 0x0), as, 0x0)); + return; + } + + const auto a = get_vr(op.ra); + const auto m = zext(bitcast(trunc(a))); + set_vr(op.rt, insert(splat(0), 3, eval(m))); + } + + void GBB(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + + if (m_use_gfni) + { + const auto as = zshuffle(a, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + const auto m = gf2p8affineqb(build(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x01, 0x01, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0), as, 0x0); + set_vr(op.rt, zshuffle(m, 16, 16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); + return; + } + + const auto m = zext(bitcast(trunc(a))); + set_vr(op.rt, insert(splat(0), 3, eval(m))); + } + + void FSM(spu_opcode_t op) + { + // FSM following a comparison instruction + if (match_vr(op.ra, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + set_vr(op.rt, (splat_scalar(c))); + return true; + } + + return false; + })) + { + return; + } + + const auto v = extract(get_vr(op.ra), 3); + const auto m = bitcast(trunc(v)); + set_vr(op.rt, sext(m)); + } + + void FSMH(spu_opcode_t op) + { + const auto v = extract(get_vr(op.ra), 3); + const auto m = bitcast(trunc(v)); + set_vr(op.rt, sext(m)); + } + + void FSMB(spu_opcode_t op) + { + const auto v = extract(get_vr(op.ra), 3); + const auto m = bitcast(trunc(v)); + set_vr(op.rt, sext(m)); + } + + template + static auto byteswap(TA&& a) + { + return zshuffle(std::forward(a), 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + } + + void ROTQBYBI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + + // Data with swapped endian from a load instruction + if (auto [ok, as] = match_expr(a, byteswap(match())); ok) + { + const auto sc = build(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const auto sh = sc + (splat_scalar(get_vr(op.rb)) >> 3); + + if (m_use_avx512_icl) + { + set_vr(op.rt, vpermb(as, sh)); + return; + } + + set_vr(op.rt, pshufb(as, (sh & 0xf))); + return; + } + const auto sc = build(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const auto sh = sc - (splat_scalar(get_vr(op.rb)) >> 3); + + if (m_use_avx512_icl) + { + set_vr(op.rt, vpermb(a, sh)); + return; + } + + set_vr(op.rt, pshufb(a, (sh & 0xf))); + } + + void ROTQMBYBI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + + auto minusb = eval(-(b >> 3)); + if (auto [ok, v0, v1] = match_expr(b, match() - match()); ok) + { + if (auto [ok1, data] = get_const_vector(v0.value, m_pos); ok1) + { + if (data == v128::from32p(7)) + { + minusb = eval(v1 >> 3); + } + } + } + + const auto minusbx = eval(bitcast(minusb) & 0x1f); + + // Data with swapped endian from a load instruction + if (auto [ok, as] = match_expr(a, byteswap(match())); ok) + { + const auto sc = build(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const auto sh = sc - splat_scalar(minusbx); + set_vr(op.rt, pshufb(as, sh)); + return; + } + + const auto sc = build(112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127); + const auto sh = sc + splat_scalar(minusbx); + set_vr(op.rt, pshufb(a, sh)); + } + + void SHLQBYBI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + + // Data with swapped endian from a load instruction + if (auto [ok, as] = match_expr(a, byteswap(match())); ok) + { + const auto sc = build(127, 126, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112); + const auto sh = sc + (splat_scalar(b) >> 3); + set_vr(op.rt, pshufb(as, sh)); + return; + } + + const auto sc = build(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const auto sh = sc - (splat_scalar(b) >> 3); + set_vr(op.rt, pshufb(a, sh)); + } + + template + auto spu_get_insertion_shuffle_mask(T&& index) + { + const auto c = bitcast(build(0x1f, 0x1e, 0x1d, 0x1c, 0x1b, 0x1a, 0x19, 0x18, 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10)); + using e_type = std::remove_extent_t; + const auto v = splat(static_cast(sizeof(e_type) == 8 ? 0x01020304050607ull : 0x010203ull)); + return insert(c, std::forward(index), v); + } + + void CBX(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn && op.ra == s_reg_sp) + { + // Optimization with aligned stack assumption. Strange because SPU code could use CBD instead, but encountered in wild. + set_vr(op.rt, spu_get_insertion_shuffle_mask(~get_scalar(get_vr(op.rb)) & 0xf)); + return; + } + + const auto s = get_scalar(get_vr(op.ra)) + get_scalar(get_vr(op.rb)); + set_vr(op.rt, spu_get_insertion_shuffle_mask(~s & 0xf)); + } + + void CHX(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn && op.ra == s_reg_sp) + { + // See CBX. + set_vr(op.rt, spu_get_insertion_shuffle_mask(~get_scalar(get_vr(op.rb)) >> 1 & 0x7)); + return; + } + + const auto s = get_scalar(get_vr(op.ra)) + get_scalar(get_vr(op.rb)); + set_vr(op.rt, spu_get_insertion_shuffle_mask(~s >> 1 & 0x7)); + } + + void CWX(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn && op.ra == s_reg_sp) + { + // See CBX. + set_vr(op.rt, spu_get_insertion_shuffle_mask(~get_scalar(get_vr(op.rb)) >> 2 & 0x3)); + return; + } + + const auto s = get_scalar(get_vr(op.ra)) + get_scalar(get_vr(op.rb)); + set_vr(op.rt, spu_get_insertion_shuffle_mask(~s >> 2 & 0x3)); + } + + void CDX(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn && op.ra == s_reg_sp) + { + // See CBX. + set_vr(op.rt, spu_get_insertion_shuffle_mask(~get_scalar(get_vr(op.rb)) >> 3 & 0x1)); + return; + } + + const auto s = get_scalar(get_vr(op.ra)) + get_scalar(get_vr(op.rb)); + set_vr(op.rt, spu_get_insertion_shuffle_mask(~s >> 3 & 0x1)); + } + + void ROTQBI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = splat_scalar(get_vr(op.rb) & 0x7); + set_vr(op.rt, fshl(a, zshuffle(a, 3, 0, 1, 2), b)); + } + + void ROTQMBI(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + + auto minusb = eval(-b); + if (auto [ok, x] = match_expr(b, -match()); ok) + { + minusb = eval(x); + } + + const auto bx = splat_scalar(minusb) & 0x7; + set_vr(op.rt, fshr(zshuffle(a, 1, 2, 3, 4), a, bx)); + } + + void SHLQBI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = splat_scalar(get_vr(op.rb) & 0x7); + set_vr(op.rt, fshl(a, zshuffle(a, 4, 0, 1, 2), b)); + } + +#if defined(ARCH_X64) + static __m128i exec_rotqby(__m128i a, u8 b) + { + alignas(32) const __m128i buf[2]{a, a}; + return _mm_loadu_si128(reinterpret_cast(reinterpret_cast(buf) + (16 - (b & 0xf)))); + } +#elif defined(ARCH_ARM64) +#else +#error "Unimplemented" +#endif + + void ROTQBY(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + +#if defined(ARCH_X64) + if (!m_use_ssse3) + { + value_t r; + r.value = call("spu_rotqby", &exec_rotqby, a.value, eval(extract(b, 12)).value); + set_vr(op.rt, r); + return; + } +#endif + + // Data with swapped endian from a load instruction + if (auto [ok, as] = match_expr(a, byteswap(match())); ok) + { + const auto sc = build(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const auto sh = eval(sc + splat_scalar(b)); + + if (m_use_avx512_icl) + { + set_vr(op.rt, vpermb(as, sh)); + return; + } + + set_vr(op.rt, pshufb(as, (sh & 0xf))); + return; + } + + const auto sc = build(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const auto sh = eval(sc - splat_scalar(b)); + + if (m_use_avx512_icl) + { + set_vr(op.rt, vpermb(a, sh)); + return; + } + + set_vr(op.rt, pshufb(a, (sh & 0xf))); + } + + void ROTQMBY(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + + auto minusb = eval(-b); + if (auto [ok, x] = match_expr(b, -match()); ok) + { + minusb = eval(x); + } + + const auto minusbx = bitcast(minusb); + + // Data with swapped endian from a load instruction + if (auto [ok, as] = match_expr(a, byteswap(match())); ok) + { + const auto sc = build(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const auto sh = sc - (splat_scalar(minusbx) & 0x1f); + set_vr(op.rt, pshufb(as, sh)); + return; + } + + const auto sc = build(112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127); + const auto sh = sc + (splat_scalar(minusbx) & 0x1f); + set_vr(op.rt, pshufb(a, sh)); + } + + void SHLQBY(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + + // Data with swapped endian from a load instruction + if (auto [ok, as] = match_expr(a, byteswap(match())); ok) + { + const auto sc = build(127, 126, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112); + const auto sh = sc + (splat_scalar(b) & 0x1f); + set_vr(op.rt, pshufb(as, sh)); + return; + } + + const auto sc = build(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const auto sh = sc - (splat_scalar(b) & 0x1f); + set_vr(op.rt, pshufb(a, sh)); + } + + template + static llvm_calli orx(T&& a) + { + return {"spu_orx", {std::forward(a)}}; + } + + void ORX(spu_opcode_t op) + { + register_intrinsic("spu_orx", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto x = zshuffle(a, 2, 3, 0, 1) | a; + const auto y = zshuffle(x, 1, 0, 3, 2) | x; + return zshuffle(y, 4, 4, 4, 3); + }); + + set_vr(op.rt, orx(get_vr(op.ra))); + } + + void CBD(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn && op.ra == s_reg_sp) + { + // Known constant with aligned stack assumption (optimization). + set_vr(op.rt, spu_get_insertion_shuffle_mask(~get_imm(op.i7) & 0xf)); + return; + } + + const auto a = get_scalar(get_vr(op.ra)) + get_imm(op.i7); + set_vr(op.rt, spu_get_insertion_shuffle_mask(~a & 0xf)); + } + + void CHD(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn && op.ra == s_reg_sp) + { + // See CBD. + set_vr(op.rt, spu_get_insertion_shuffle_mask(~get_imm(op.i7) >> 1 & 0x7)); + return; + } + + const auto a = get_scalar(get_vr(op.ra)) + get_imm(op.i7); + set_vr(op.rt, spu_get_insertion_shuffle_mask(~a >> 1 & 0x7)); + } + + void CWD(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn && op.ra == s_reg_sp) + { + // See CBD. + set_vr(op.rt, spu_get_insertion_shuffle_mask(~get_imm(op.i7) >> 2 & 0x3)); + return; + } + + const auto a = get_scalar(get_vr(op.ra)) + get_imm(op.i7); + set_vr(op.rt, spu_get_insertion_shuffle_mask(~a >> 2 & 0x3)); + } + + void CDD(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn && op.ra == s_reg_sp) + { + // See CBD. + set_vr(op.rt, spu_get_insertion_shuffle_mask(~get_imm(op.i7) >> 3 & 0x1)); + return; + } + + const auto a = get_scalar(get_vr(op.ra)) + get_imm(op.i7); + set_vr(op.rt, spu_get_insertion_shuffle_mask(~a >> 3 & 0x1)); + } + + void ROTQBII(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = eval(get_imm(op.i7, false) & 0x7); + set_vr(op.rt, fshl(a, zshuffle(a, 3, 0, 1, 2), b)); + } + + void ROTQMBII(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = eval(-get_imm(op.i7, false) & 0x7); + set_vr(op.rt, fshr(zshuffle(a, 1, 2, 3, 4), a, b)); + } + + void SHLQBII(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = eval(get_imm(op.i7, false) & 0x7); + set_vr(op.rt, fshl(a, zshuffle(a, 4, 0, 1, 2), b)); + } + + void ROTQBYI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto sc = build(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const auto sh = (sc - get_imm(op.i7, false)) & 0xf; + set_vr(op.rt, pshufb(a, sh)); + } + + void ROTQMBYI(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto sc = build(112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127); + const auto sh = sc + (-get_imm(op.i7, false) & 0x1f); + set_vr(op.rt, pshufb(a, sh)); + } + + void SHLQBYI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.i7) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); // For expressions matching + const auto a = get_vr(op.ra); + const auto sc = build(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const auto sh = sc - (get_imm(op.i7, false) & 0x1f); + set_vr(op.rt, pshufb(a, sh)); + } + + void CGT(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_vr(op.rb))); + } + + void XOR(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) ^ get_vr(op.rb)); + } + + void CGTH(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_vr(op.rb))); + } + + void EQV(spu_opcode_t op) + { + set_vr(op.rt, ~(get_vr(op.ra) ^ get_vr(op.rb))); + } + + void CGTB(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_vr(op.rb))); + } + + void SUMB(spu_opcode_t op) + { + if (m_use_avx512) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + const auto zeroes = splat(0); + + if (op.ra == op.rb && !m_interp_magn) + { + set_vr(op.rt, vdbpsadbw(a, zeroes, 0)); + return; + } + + const auto ax = vdbpsadbw(a, zeroes, 0); + const auto bx = vdbpsadbw(b, zeroes, 0); + set_vr(op.rt, shuffle2(ax, bx, 0, 9, 2, 11, 4, 13, 6, 15)); + return; + } + + if (m_use_vnni) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + const auto zeroes = splat(0); + const auto ones = splat(0x01010101); + const auto ax = bitcast(vpdpbusd(zeroes, a, ones)); + const auto bx = bitcast(vpdpbusd(zeroes, b, ones)); + set_vr(op.rt, shuffle2(ax, bx, 0, 8, 2, 10, 4, 12, 6, 14)); + return; + } + + const auto [a, b] = get_vrs(op.ra, op.rb); + const auto ahs = eval((a >> 8) + (a & 0xff)); + const auto bhs = eval((b >> 8) + (b & 0xff)); + const auto lsh = shuffle2(ahs, bhs, 0, 9, 2, 11, 4, 13, 6, 15); + const auto hsh = shuffle2(ahs, bhs, 1, 8, 3, 10, 5, 12, 7, 14); + set_vr(op.rt, lsh + hsh); + } + + void CLZ(spu_opcode_t op) + { + set_vr(op.rt, ctlz(get_vr(op.ra))); + } + + void XSWD(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) << 32 >> 32); + } + + void XSHW(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) << 16 >> 16); + } + + void CNTB(spu_opcode_t op) + { + set_vr(op.rt, ctpop(get_vr(op.ra))); + } + + void XSBH(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) << 8 >> 8); + } + + void CLGT(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_vr(op.rb))); + } + + void ANDC(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) & ~get_vr(op.rb)); + } + + void CLGTH(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_vr(op.rb))); + } + + void ORC(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) | ~get_vr(op.rb)); + } + + void CLGTB(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_vr(op.rb))); + } + + void CEQ(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) == get_vr(op.rb))); + } + + void MPYHHU(spu_opcode_t op) + { + set_vr(op.rt, (get_vr(op.ra) >> 16) * (get_vr(op.rb) >> 16)); + } + + void ADDX(spu_opcode_t op) + { + set_vr(op.rt, llvm_sum{get_vr(op.ra), get_vr(op.rb), get_vr(op.rt) & 1}); + } + + void SFX(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.rb) - get_vr(op.ra) - (~get_vr(op.rt) & 1)); + } + + void CGX(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + const auto x = (get_vr(op.rt) << 31) >> 31; + const auto s = eval(a + b); + set_vr(op.rt, noncast(sext(s < a) | (sext(s == noncast(x)) & x)) >> 31); + } + + void BGX(spu_opcode_t op) + { + const auto [a, b] = get_vrs(op.ra, op.rb); + const auto c = get_vr(op.rt) << 31; + set_vr(op.rt, noncast(sext(b > a) | (sext(a == b) & c)) >> 31); + } + + void MPYHHA(spu_opcode_t op) + { + set_vr(op.rt, (get_vr(op.ra) >> 16) * (get_vr(op.rb) >> 16) + get_vr(op.rt)); + } + + void MPYHHAU(spu_opcode_t op) + { + set_vr(op.rt, (get_vr(op.ra) >> 16) * (get_vr(op.rb) >> 16) + get_vr(op.rt)); + } + + void MPY(spu_opcode_t op) + { + set_vr(op.rt, (get_vr(op.ra) << 16 >> 16) * (get_vr(op.rb) << 16 >> 16)); + } + + void MPYH(spu_opcode_t op) + { + set_vr(op.rt, mpyh(get_vr(op.ra), get_vr(op.rb))); + } + + void MPYHH(spu_opcode_t op) + { + set_vr(op.rt, (get_vr(op.ra) >> 16) * (get_vr(op.rb) >> 16)); + } + + void MPYS(spu_opcode_t op) + { + set_vr(op.rt, (get_vr(op.ra) << 16 >> 16) * (get_vr(op.rb) << 16 >> 16) >> 16); + } + + void CEQH(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) == get_vr(op.rb))); + } + + void MPYU(spu_opcode_t op) + { + set_vr(op.rt, mpyu(get_vr(op.ra), get_vr(op.rb))); + } + + void CEQB(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) == get_vr(op.rb))); + } + + void FSMBI(spu_opcode_t op) + { + const auto m = bitcast(get_imm(op.i16)); + set_vr(op.rt, sext(m)); + } + + void IL(spu_opcode_t op) + { + set_vr(op.rt, get_imm(op.si16)); + } + + void ILHU(spu_opcode_t op) + { + set_vr(op.rt, get_imm(op.i16) << 16); + } + + void ILH(spu_opcode_t op) + { + set_vr(op.rt, get_imm(op.i16)); + } + + void IOHL(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.rt) | get_imm(op.i16)); + } + + void ORI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.si10) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); // For expressions matching + set_vr(op.rt, get_vr(op.ra) | get_imm(op.si10)); + } + + void ORHI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.si10) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) | get_imm(op.si10)); + } + + void ORBI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.si10) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) | get_imm(op.si10)); + } + + void SFI(spu_opcode_t op) + { + set_vr(op.rt, get_imm(op.si10) - get_vr(op.ra)); + } + + void SFHI(spu_opcode_t op) + { + set_vr(op.rt, get_imm(op.si10) - get_vr(op.ra)); + } + + void ANDI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && op.si10 == -1) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) & get_imm(op.si10)); + } + + void ANDHI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && op.si10 == -1) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) & get_imm(op.si10)); + } + + void ANDBI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && static_cast(op.si10) == -1) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) & get_imm(op.si10)); + } + + void AI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.si10) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) + get_imm(op.si10)); + } + + void AHI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.si10) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) + get_imm(op.si10)); + } + + void XORI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.si10) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) ^ get_imm(op.si10)); + } + + void XORHI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.si10) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) ^ get_imm(op.si10)); + } + + void XORBI(spu_opcode_t op) + { + if (get_reg_raw(op.ra) && !op.si10) return set_reg_fixed(op.rt, get_reg_raw(op.ra), false); + set_vr(op.rt, get_vr(op.ra) ^ get_imm(op.si10)); + } + + void CGTI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_imm(op.si10))); + } + + void CGTHI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_imm(op.si10))); + } + + void CGTBI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_imm(op.si10))); + } + + void CLGTI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_imm(op.si10))); + } + + void CLGTHI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_imm(op.si10))); + } + + void CLGTBI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) > get_imm(op.si10))); + } + + void MPYI(spu_opcode_t op) + { + set_vr(op.rt, (get_vr(op.ra) << 16 >> 16) * get_imm(op.si10)); + } + + void MPYUI(spu_opcode_t op) + { + set_vr(op.rt, (get_vr(op.ra) << 16 >> 16) * (get_imm(op.si10) & 0xffff)); + } + + void CEQI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) == get_imm(op.si10))); + } + + void CEQHI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) == get_imm(op.si10))); + } + + void CEQBI(spu_opcode_t op) + { + set_vr(op.rt, sext(get_vr(op.ra) == get_imm(op.si10))); + } + + void ILA(spu_opcode_t op) + { + set_vr(op.rt, get_imm(op.i18)); + } + + void SELB(spu_opcode_t op) + { + if (match_vr(op.rc, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + // If the control mask comes from a comparison instruction, replace SELB with select + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + if constexpr (std::extent_v == 2) // u64[2] + { + // Try to select floats as floats if a OR b is typed as f64[2] + if (auto [a, b] = match_vrs(op.ra, op.rb); a || b) + { + set_vr(op.rt4, select(x, get_vr(op.rb), get_vr(op.ra))); + return true; + } + } + + if constexpr (std::extent_v == 4) // u32[4] + { + // Match division (adjusted) (TODO) + if (auto a = match_vr(op.ra)) + { + static const auto MT = match(); + + if (auto [div_ok, diva, divb] = match_expr(a, MT / MT); div_ok) + { + if (auto b = match_vr(op.rb)) + { + if (auto [add1_ok] = match_expr(b, bitcast(a) + splat(1)); add1_ok) + { + if (auto [fm_ok, a1, b1] = match_expr(x, bitcast(fm(MT, MT)) > splat(-1)); fm_ok) + { + if (auto [fnma_ok] = match_expr(a1, fnms(divb, bitcast(b), diva)); fnma_ok) + { + if (fabs(b1).eval(m_ir) == fsplat(1.0).eval(m_ir)) + { + set_vr(op.rt4, diva / divb); + return true; + } + + if (auto [sel_ok] = match_expr(b1, bitcast((bitcast(diva) & 0x80000000) | 0x3f800000)); sel_ok) + { + set_vr(op.rt4, diva / divb); + return true; + } + } + } + } + } + } + } + + if (auto [a, b] = match_vrs(op.ra, op.rb); a || b) + { + set_vr(op.rt4, select(x, get_vr(op.rb), get_vr(op.ra))); + return true; + } + + if (auto [a, b] = match_vrs(op.ra, op.rb); a || b) + { + set_vr(op.rt4, select(x, get_vr(op.rb), get_vr(op.ra))); + return true; + } + } + + if (auto [ok, y] = match_expr(x, bitcast]>(match>>())); ok) + { + // Don't ruin FSMB/FSM/FSMH instructions + return false; + } + + set_vr(op.rt4, select(x, get_vr(op.rb), get_vr(op.ra))); + return true; + } + + return false; + })) + { + return; + } + + const auto c = get_vr(op.rc); + + // Check if the constant mask doesn't require bit granularity + if (auto [ok, mask] = get_const_vector(c.value, m_pos); ok) + { + bool sel_32 = true; + for (u32 i = 0; i < 4; i++) + { + if (mask._u32[i] && mask._u32[i] != 0xFFFFFFFF) + { + sel_32 = false; + break; + } + } + + if (sel_32) + { + if (auto [a, b] = match_vrs(op.ra, op.rb); a || b) + { + set_vr(op.rt4, select(noncast(c) != 0, get_vr(op.rb), get_vr(op.ra))); + return; + } + else if (auto [a, b] = match_vrs(op.ra, op.rb); a || b) + { + set_vr(op.rt4, select(noncast(c) != 0, get_vr(op.rb), get_vr(op.ra))); + return; + } + + set_vr(op.rt4, select(noncast(c) != 0, get_vr(op.rb), get_vr(op.ra))); + return; + } + + bool sel_16 = true; + for (u32 i = 0; i < 8; i++) + { + if (mask._u16[i] && mask._u16[i] != 0xFFFF) + { + sel_16 = false; + break; + } + } + + if (sel_16) + { + set_vr(op.rt4, select(bitcast(c) != 0, get_vr(op.rb), get_vr(op.ra))); + return; + } + + bool sel_8 = true; + for (u32 i = 0; i < 16; i++) + { + if (mask._u8[i] && mask._u8[i] != 0xFF) + { + sel_8 = false; + break; + } + } + + if (sel_8) + { + set_vr(op.rt4, select(bitcast(c) != 0,get_vr(op.rb), get_vr(op.ra))); + return; + } + } + + const auto op1 = get_reg_raw(op.rb); + const auto op2 = get_reg_raw(op.ra); + + if ((op1 && op1->getType() == get_type()) || (op2 && op2->getType() == get_type())) + { + // Optimization: keep xfloat values in doubles even if the mask is unpredictable (hard way) + const auto c = get_vr(op.rc); + const auto b = get_vr(op.rb); + const auto a = get_vr(op.ra); + const auto m = conv_xfloat_mask(c.value); + const auto x = m_ir->CreateAnd(double_as_uint64(b.value), m); + const auto y = m_ir->CreateAnd(double_as_uint64(a.value), m_ir->CreateNot(m)); + set_reg_fixed(op.rt4, uint64_as_double(m_ir->CreateOr(x, y))); + return; + } + + set_vr(op.rt4, (get_vr(op.rb) & c) | (get_vr(op.ra) & ~c)); + } + + void SHUFB(spu_opcode_t op) // + { + if (match_vr(op.rc, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + // If the mask comes from a constant generation instruction, replace SHUFB with insert + if (auto [ok, i] = match_expr(c, spu_get_insertion_shuffle_mask(match())); ok) + { + set_vr(op.rt4, insert(get_vr(op.rb), i, get_scalar(get_vr(op.ra)))); + return true; + } + + return false; + })) + { + return; + } + + const auto c = get_vr(op.rc); + + if (auto [ok, mask] = get_const_vector(c.value, m_pos); ok) + { + // Optimization: SHUFB with constant mask + if (((mask._u64[0] | mask._u64[1]) & 0xe0e0e0e0e0e0e0e0) == 0) + { + // Trivial insert or constant shuffle (TODO) + static constexpr struct mask_info + { + u64 i1; + u64 i0; + decltype(&cpu_translator::get_type) type; + u64 extract_from; + u64 insert_to; + } s_masks[30] + { + { 0x0311121314151617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 12, 15 }, + { 0x1003121314151617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 12, 14 }, + { 0x1011031314151617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 12, 13 }, + { 0x1011120314151617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 12, 12 }, + { 0x1011121303151617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 12, 11 }, + { 0x1011121314031617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 12, 10 }, + { 0x1011121314150317, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 12, 9 }, + { 0x1011121314151603, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 12, 8 }, + { 0x1011121314151617, 0x03191a1b1c1d1e1f, &cpu_translator::get_type, 12, 7 }, + { 0x1011121314151617, 0x18031a1b1c1d1e1f, &cpu_translator::get_type, 12, 6 }, + { 0x1011121314151617, 0x1819031b1c1d1e1f, &cpu_translator::get_type, 12, 5 }, + { 0x1011121314151617, 0x18191a031c1d1e1f, &cpu_translator::get_type, 12, 4 }, + { 0x1011121314151617, 0x18191a1b031d1e1f, &cpu_translator::get_type, 12, 3 }, + { 0x1011121314151617, 0x18191a1b1c031e1f, &cpu_translator::get_type, 12, 2 }, + { 0x1011121314151617, 0x18191a1b1c1d031f, &cpu_translator::get_type, 12, 1 }, + { 0x1011121314151617, 0x18191a1b1c1d1e03, &cpu_translator::get_type, 12, 0 }, + { 0x0203121314151617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 6, 7 }, + { 0x1011020314151617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 6, 6 }, + { 0x1011121302031617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 6, 5 }, + { 0x1011121314150203, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 6, 4 }, + { 0x1011121314151617, 0x02031a1b1c1d1e1f, &cpu_translator::get_type, 6, 3 }, + { 0x1011121314151617, 0x181902031c1d1e1f, &cpu_translator::get_type, 6, 2 }, + { 0x1011121314151617, 0x18191a1b02031e1f, &cpu_translator::get_type, 6, 1 }, + { 0x1011121314151617, 0x18191a1b1c1d0203, &cpu_translator::get_type, 6, 0 }, + { 0x0001020314151617, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 3, 3 }, + { 0x1011121300010203, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 3, 2 }, + { 0x1011121314151617, 0x000102031c1d1e1f, &cpu_translator::get_type, 3, 1 }, + { 0x1011121314151617, 0x18191a1b00010203, &cpu_translator::get_type, 3, 0 }, + { 0x0001020304050607, 0x18191a1b1c1d1e1f, &cpu_translator::get_type, 1, 1 }, + { 0x1011121303151617, 0x0001020304050607, &cpu_translator::get_type, 1, 0 }, + }; + + // Check important constants from CWD-like constant generation instructions + for (const auto& cm : s_masks) + { + if (mask._u64[0] == cm.i0 && mask._u64[1] == cm.i1) + { + const auto t = (this->*cm.type)(); + const auto a = get_reg_fixed(op.ra, t); + const auto b = get_reg_fixed(op.rb, t); + const auto e = m_ir->CreateExtractElement(a, cm.extract_from); + set_reg_fixed(op.rt4, m_ir->CreateInsertElement(b, e, cm.insert_to)); + return; + } + } + } + + // Adjusted shuffle mask + v128 smask = ~mask & v128::from8p(op.ra == op.rb ? 0xf : 0x1f); + + // Blend mask for encoded constants + v128 bmask{}; + + for (u32 i = 0; i < 16; i++) + { + if (mask._bytes[i] >= 0xe0) + bmask._bytes[i] = 0x80; + else if (mask._bytes[i] >= 0xc0) + bmask._bytes[i] = 0xff; + } + + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + const auto c = make_const_vector(smask, get_type()); + const auto d = make_const_vector(bmask, get_type()); + + llvm::Value* r = d; + + if ((~mask._u64[0] | ~mask._u64[1]) & 0x8080808080808080) [[likely]] + { + r = m_ir->CreateShuffleVector(b.value, op.ra == op.rb ? b.value : a.value, m_ir->CreateZExt(c, get_type())); + + if ((mask._u64[0] | mask._u64[1]) & 0x8080808080808080) + { + r = m_ir->CreateSelect(m_ir->CreateICmpSLT(make_const_vector(mask, get_type()), llvm::ConstantInt::get(get_type(), 0)), d, r); + } + } + + set_reg_fixed(op.rt4, r); + return; + } + + // Check whether shuffle mask doesn't contain fixed value selectors + bool perm_only = false; + + if (auto k = get_known_bits(c); !!(k.Zero & 0x80)) + { + perm_only = true; + } + + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + + // Data with swapped endian from a load instruction + if (auto [ok, as] = match_expr(a, byteswap(match())); ok) + { + if (auto [ok, bs] = match_expr(b, byteswap(match())); ok) + { + // Undo endian swapping, and rely on pshufb/vperm2b to re-reverse endianness + if (m_use_avx512_icl && (op.ra != op.rb)) + { + if (perm_only) + { + set_vr(op.rt4, vperm2b(as, bs, c)); + return; + } + + const auto m = gf2p8affineqb(c, build(0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20), 0x7f); + const auto mm = select(noncast(m) >= 0, splat(0), m); + const auto ab = vperm2b(as, bs, c); + set_vr(op.rt4, select(noncast(c) >= 0, ab, mm)); + return; + } + + const auto x = pshufb(build(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x80, 0x80), (c >> 4)); + const auto ax = pshufb(as, c); + const auto bx = pshufb(bs, c); + + if (perm_only) + set_vr(op.rt4, select_by_bit4(c, ax, bx)); + else + set_vr(op.rt4, select_by_bit4(c, ax, bx) | x); + return; + } + + if (auto [ok, data] = get_const_vector(b.value, m_pos); ok) + { + if (data == v128::from8p(data._u8[0])) + { + if (m_use_avx512_icl) + { + if (perm_only) + { + set_vr(op.rt4, vperm2b256to128(as, b, c)); + return; + } + + const auto m = gf2p8affineqb(c, build(0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20), 0x7f); + const auto mm = select(noncast(m) >= 0, splat(0), m); + const auto ab = vperm2b256to128(as, b, c); + set_vr(op.rt4, select(noncast(c) >= 0, ab, mm)); + return; + } + // See above + const auto x = pshufb(build(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x80, 0x80), (c >> 4)); + const auto ax = pshufb(as, c); + + if (perm_only) + set_vr(op.rt4, select_by_bit4(c, ax, b)); + else + set_vr(op.rt4, select_by_bit4(c, ax, b) | x); + return; + } + } + } + + if (auto [ok, bs] = match_expr(b, byteswap(match())); ok) + { + if (auto [ok, data] = get_const_vector(a.value, m_pos); ok) + { + if (data == v128::from8p(data._u8[0])) + { + // See above + const auto x = pshufb(build(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x80, 0x80), (c >> 4)); + const auto bx = pshufb(bs, c); + + if (perm_only) + set_vr(op.rt4, select_by_bit4(c, a, bx)); + else + set_vr(op.rt4, select_by_bit4(c, a, bx) | x); + return; + } + } + } + + if (m_use_avx512_icl && (op.ra != op.rb || m_interp_magn)) + { + if (auto [ok, data] = get_const_vector(b.value, m_pos); ok) + { + if (data == v128::from8p(data._u8[0])) + { + if (perm_only) + { + set_vr(op.rt4, vperm2b256to128(a, b, eval(c ^ 0xf))); + return; + } + + const auto m = gf2p8affineqb(c, build(0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20), 0x7f); + const auto mm = select(noncast(m) >= 0, splat(0), m); + const auto ab = vperm2b256to128(a, b, eval(c ^ 0xf)); + set_vr(op.rt4, select(noncast(c) >= 0, ab, mm)); + return; + } + } + + if (auto [ok, data] = get_const_vector(a.value, m_pos); ok) + { + if (data == v128::from8p(data._u8[0])) + { + if (perm_only) + { + set_vr(op.rt4, vperm2b256to128(b, a, eval(c ^ 0x1f))); + return; + } + + const auto m = gf2p8affineqb(c, build(0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20), 0x7f); + const auto mm = select(noncast(m) >= 0, splat(0), m); + const auto ab = vperm2b256to128(b, a, eval(c ^ 0x1f)); + set_vr(op.rt4, select(noncast(c) >= 0, ab, mm)); + return; + } + } + + if (perm_only) + { + set_vr(op.rt4, vperm2b(a, b, eval(c ^ 0xf))); + return; + } + + const auto m = gf2p8affineqb(c, build(0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x40, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20), 0x7f); + const auto mm = select(noncast(m) >= 0, splat(0), m); + const auto cr = eval(c ^ 0xf); + const auto ab = vperm2b(a, b, cr); + set_vr(op.rt4, select(noncast(c) >= 0, ab, mm)); + return; + } + + const auto x = pshufb(build(0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x80, 0x80), (c >> 4)); + const auto cr = eval(c ^ 0xf); + const auto ax = pshufb(a, cr); + const auto bx = pshufb(b, cr); + + if (perm_only) + set_vr(op.rt4, select_by_bit4(cr, ax, bx)); + else + set_vr(op.rt4, select_by_bit4(cr, ax, bx) | x); + } + + void MPYA(spu_opcode_t op) + { + set_vr(op.rt4, (get_vr(op.ra) << 16 >> 16) * (get_vr(op.rb) << 16 >> 16) + get_vr(op.rc)); + } + + void FSCRRD(spu_opcode_t op) // + { + // Hack + set_vr(op.rt, splat(0)); + } + + void FSCRWR(spu_opcode_t /*op*/) // + { + // Hack + } + + void DFCGT(spu_opcode_t op) // + { + return UNK(op); + } + + void DFCEQ(spu_opcode_t op) // + { + return UNK(op); + } + + void DFCMGT(spu_opcode_t op) // + { + return UNK(op); + } + + void DFCMEQ(spu_opcode_t op) // + { + return UNK(op); + } + + void DFTSV(spu_opcode_t op) // + { + return UNK(op); + } + + void DFA(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) + get_vr(op.rb)); + } + + void DFS(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) - get_vr(op.rb)); + } + + void DFM(spu_opcode_t op) + { + set_vr(op.rt, get_vr(op.ra) * get_vr(op.rb)); + } + + void DFMA(spu_opcode_t op) + { + const auto [a, b, c] = get_vrs(op.ra, op.rb, op.rt); + + if (g_cfg.core.use_accurate_dfma) + set_vr(op.rt, fmuladd(a, b, c, true)); + else + set_vr(op.rt, a * b + c); + } + + void DFMS(spu_opcode_t op) + { + const auto [a, b, c] = get_vrs(op.ra, op.rb, op.rt); + + if (g_cfg.core.use_accurate_dfma) + set_vr(op.rt, fmuladd(a, b, -c, true)); + else + set_vr(op.rt, a * b - c); + } + + void DFNMS(spu_opcode_t op) + { + const auto [a, b, c] = get_vrs(op.ra, op.rb, op.rt); + + if (g_cfg.core.use_accurate_dfma) + set_vr(op.rt, fmuladd(-a, b, c, true)); + else + set_vr(op.rt, c - (a * b)); + } + + void DFNMA(spu_opcode_t op) + { + const auto [a, b, c] = get_vrs(op.ra, op.rb, op.rt); + + if (g_cfg.core.use_accurate_dfma) + set_vr(op.rt, -fmuladd(a, b, c, true)); + else + set_vr(op.rt, -(a * b + c)); + } + + bool is_input_positive(value_t a) + { + if (auto [ok, v0, v1] = match_expr(a, match() * match()); ok && v0.eq(v1)) + { + return true; + } + + return false; + } + + // clamping helpers + value_t clamp_positive_smax(value_t v) + { + return eval(bitcast(min(bitcast(v),splat(0x7f7fffff)))); + } + + value_t clamp_negative_smax(value_t v) + { + if (is_input_positive(v)) + { + return v; + } + + return eval(bitcast(min(bitcast(v),splat(0xff7fffff)))); + } + + value_t clamp_smax(value_t v) + { + if (m_use_avx512) + { + if (is_input_positive(v)) + { + return eval(clamp_positive_smax(v)); + } + + if (auto [ok, data] = get_const_vector(v.value, m_pos); ok) + { + // Avoid pessimation when input is constant + return eval(clamp_positive_smax(clamp_negative_smax(v))); + } + + return eval(vrangeps(v, fsplat(std::bit_cast(0x7f7fffff)), 0x2, 0xff)); + } + + return eval(clamp_positive_smax(clamp_negative_smax(v))); + } + + // FMA favouring zeros + value_t xmuladd(value_t a, value_t b, value_t c) + { + const auto ma = eval(sext(fcmp_uno(a != fsplat(0.)))); + const auto mb = eval(sext(fcmp_uno(b != fsplat(0.)))); + const auto ca = eval(bitcast(bitcast(a) & mb)); + const auto cb = eval(bitcast(bitcast(b) & ma)); + return eval(fmuladd(ca, cb, c)); + } + + // Checks for postive and negative zero, or Denormal (treated as zero) + // If sign is +-1 check equality againts all sign bits + bool is_spu_float_zero(v128 a, int sign = 0) + { + for (u32 i = 0; i < 4; i++) + { + const u32 exponent = a._u32[i] & 0x7f800000u; + + if (exponent || (sign && (sign >= 0) != (a._s32[i] >= 0))) + { + // Normalized number + return false; + } + } + return true; + } + + template + static llvm_calli frest(T&& a) + { + return {"spu_frest", {std::forward(a)}}; + } + + void FREST(spu_opcode_t op) + { + // TODO + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + const auto a = get_vr(op.ra); + const auto mask_ov = sext(bitcast(fabs(a)) > splat(0x7e7fffff)); + const auto mask_de = eval(noncast(sext(fcmp_ord(a == fsplat(0.)))) >> 1); + set_vr(op.rt, (bitcast(fsplat(1.0) / a) & ~mask_ov) | noncast(mask_de)); + return; + } + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + register_intrinsic("spu_frest", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + // Gives accuracy penalty, frest result is within one newton-raphson iteration for accuracy + const auto approx_result = fsplat(0.999875069f) / a; + // Zeroes the last 11 bytes of the mantissa so FI calculations end up correct if needed + return bitcast(bitcast(approx_result) & splat(0xFFFFF800)); + }); + } + else + { + register_intrinsic("spu_frest", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + // Fast but this makes the result vary per cpu + return fre(a); + }); + } + + set_vr(op.rt, frest(get_vr(op.ra))); + } + + template + static llvm_calli frsqest(T&& a) + { + return {"spu_frsqest", {std::forward(a)}}; + } + + void FRSQEST(spu_opcode_t op) + { + // TODO + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, fsplat(1.0) / fsqrt(fabs(get_vr(op.ra)))); + return; + } + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + register_intrinsic("spu_frsqest", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + // Gives accuracy penalty, frsqest result is within one newton-raphson iteration for accuracy + const auto approx_result = fsplat(0.999763668f) / fsqrt(fabs(a)); + // Zeroes the last 11 bytes of the mantissa so FI calculations end up correct if needed + return bitcast(bitcast(approx_result) & splat(0xFFFFF800)); + }); + } + else + { + register_intrinsic("spu_frsqest", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + // Fast but this makes the result vary per cpu + return frsqe(fabs(a)); + }); + } + + set_vr(op.rt, frsqest(get_vr(op.ra))); + } + + template + static llvm_calli fcgt(T&& a, U&& b) + { + return {"spu_fcgt", {std::forward(a), std::forward(b)}}; + } + + void FCGT(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, sext(fcmp_ord(get_vr(op.ra) > get_vr(op.rb)))); + return; + } + + register_intrinsic("spu_fcgt", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + + const value_t ab[2]{a, b}; + + std::bitset<2> safe_int_compare(0); + std::bitset<2> safe_nonzero_compare(0); + + for (u32 i = 0; i < 2; i++) + { + if (auto [ok, data] = get_const_vector(ab[i].value, m_pos, __LINE__ + i); ok) + { + safe_int_compare.set(i); + safe_nonzero_compare.set(i); + + for (u32 j = 0; j < 4; j++) + { + const u32 value = data._u32[j]; + const u8 exponent = static_cast(value >> 23); + + if (value >= 0x7f7fffffu || !exponent) + { + // Postive or negative zero, Denormal (treated as zero), Negative constant, or Normalized number with exponent +127 + // Cannot used signed integer compare safely + // Note: Technically this optimization is accurate for any positive value, but due to the fact that + // we don't produce "extended range" values the same way as real hardware, it's not safe to apply + // this optimization for values outside of the range of x86 floating point hardware. + safe_int_compare.reset(i); + if (!exponent) safe_nonzero_compare.reset(i); + } + } + } + } + + if (safe_int_compare.any()) + { + return eval(sext(bitcast(a) > bitcast(b))); + } + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate || g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::relaxed) + { + const auto ai = eval(bitcast(a)); + const auto bi = eval(bitcast(b)); + + if (!safe_nonzero_compare.any()) + { + return eval(sext(fcmp_uno(a != b) & select((ai & bi) >= 0, ai > bi, ai < bi))); + } + else + { + return eval(sext(select((ai & bi) >= 0, ai > bi, ai < bi))); + } + } + else + { + return eval(sext(fcmp_ord(a > b))); + } + }); + + set_vr(op.rt, fcgt(get_vr(op.ra), get_vr(op.rb))); + } + + template + static llvm_calli fcmgt(T&& a, U&& b) + { + return {"spu_fcmgt", {std::forward(a), std::forward(b)}}; + } + + void FCMGT(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, sext(fcmp_ord(fabs(get_vr(op.ra)) > fabs(get_vr(op.rb))))); + return; + } + + register_intrinsic("spu_fcmgt", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + + const value_t ab[2]{a, b}; + + std::bitset<2> safe_int_compare(0); + + for (u32 i = 0; i < 2; i++) + { + if (auto [ok, data] = get_const_vector(ab[i].value, m_pos, __LINE__ + i); ok) + { + safe_int_compare.set(i); + + for (u32 j = 0; j < 4; j++) + { + const u32 value = data._u32[j]; + const u8 exponent = static_cast(value >> 23); + + if ((value & 0x7fffffffu) >= 0x7f7fffffu || !exponent) + { + // See above + safe_int_compare.reset(i); + } + } + } + } + + const auto ma = eval(fabs(a)); + const auto mb = eval(fabs(b)); + + const auto mai = eval(bitcast(ma)); + const auto mbi = eval(bitcast(mb)); + + if (safe_int_compare.any()) + { + return eval(sext(mai > mbi)); + } + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + return eval(sext(fcmp_uno(ma > mb) & (mai > mbi))); + } + else + { + return eval(sext(fcmp_ord(ma > mb))); + } + }); + + set_vr(op.rt, fcmgt(get_vr(op.ra), get_vr(op.rb))); + } + + template + static llvm_calli fa(T&& a, U&& b) + { + return {"spu_fa", {std::forward(a), std::forward(b)}}; + } + + void FA(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, get_vr(op.ra) + get_vr(op.rb)); + return; + } + + register_intrinsic("spu_fa", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + + return a + b; + }); + + set_vr(op.rt, fa(get_vr(op.ra), get_vr(op.rb))); + } + + template + static llvm_calli fs(T&& a, U&& b) + { + return {"spu_fs", {std::forward(a), std::forward(b)}}; + } + + void FS(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, get_vr(op.ra) - get_vr(op.rb)); + return; + } + + register_intrinsic("spu_fs", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + const auto bc = clamp_smax(b); // for #4478 + return eval(a - bc); + } + else + { + return eval(a - b); + } + }); + + set_vr(op.rt, fs(get_vr(op.ra), get_vr(op.rb))); + } + + template + static llvm_calli fm(T&& a, U&& b) + { + return {"spu_fm", {std::forward(a), std::forward(b)}}; + } + + void FM(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, get_vr(op.ra) * get_vr(op.rb)); + return; + } + + register_intrinsic("spu_fm", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + if (a.value == b.value) + { + return eval(a * b); + } + + const auto ma = sext(fcmp_uno(a != fsplat(0.))); + const auto mb = sext(fcmp_uno(b != fsplat(0.))); + return eval(bitcast(bitcast(a * b) & ma & mb)); + } + else + { + return eval(a * b); + } + }); + + const auto [a, b] = get_vrs(op.ra, op.rb); + + if (op.ra == op.rb && !m_interp_magn) + { + set_vr(op.rt, fm(a, a)); + return; + } + + set_vr(op.rt, fm(a, b)); + } + + template + static llvm_calli fesd(T&& a) + { + return {"spu_fesd", {std::forward(a)}}; + } + + void FESD(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + const auto r = zshuffle(get_vr(op.ra), 1, 3); + const auto d = bitcast(r); + const auto a = eval(d & 0x7fffffffffffffff); + const auto s = eval(d & 0x8000000000000000); + const auto i = select(a == 0x47f0000000000000, eval(s | 0x7ff0000000000000), d); + const auto n = select(a > 0x47f0000000000000, splat(0x7ff8000000000000), i); + set_vr(op.rt, bitcast(n)); + return; + } + + register_intrinsic("spu_fesd", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + + return fpcast(zshuffle(a, 1, 3)); + }); + + set_vr(op.rt, fesd(get_vr(op.ra))); + } + + template + static llvm_calli frds(T&& a) + { + return {"spu_frds", {std::forward(a)}}; + } + + void FRDS(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + const auto r = get_vr(op.ra); + const auto d = bitcast(r); + const auto a = eval(d & 0x7fffffffffffffff); + const auto s = eval(d & 0x8000000000000000); + const auto i = select(a > 0x47f0000000000000, eval(s | 0x47f0000000000000), d); + const auto n = select(a > 0x7ff0000000000000, splat(0x47f8000000000000), i); + const auto z = select(a < 0x3810000000000000, s, n); + set_vr(op.rt, zshuffle(bitcast(z), 2, 0, 3, 1), nullptr, false); + return; + } + + register_intrinsic("spu_frds", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + + return zshuffle(fpcast(a), 2, 0, 3, 1); + }); + + set_vr(op.rt, frds(get_vr(op.ra))); + } + + template + static llvm_calli fceq(T&& a, U&& b) + { + return {"spu_fceq", {std::forward(a), std::forward(b)}}; + } + + void FCEQ(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, sext(fcmp_ord(get_vr(op.ra) == get_vr(op.rb)))); + return; + } + + register_intrinsic("spu_fceq", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + + const value_t ab[2]{a, b}; + + std::bitset<2> safe_float_compare(0); + std::bitset<2> safe_int_compare(0); + + for (u32 i = 0; i < 2; i++) + { + if (auto [ok, data] = get_const_vector(ab[i].value, m_pos, __LINE__ + i); ok) + { + safe_float_compare.set(i); + safe_int_compare.set(i); + + for (u32 j = 0; j < 4; j++) + { + const u32 value = data._u32[j]; + const u8 exponent = static_cast(value >> 23); + + // unsafe if nan + if (exponent == 255) + { + safe_float_compare.reset(i); + } + + // unsafe if denormal or 0 + if (!exponent) + { + safe_int_compare.reset(i); + } + } + } + } + + if (safe_float_compare.any()) + { + return eval(sext(fcmp_ord(a == b))); + } + + if (safe_int_compare.any()) + { + return eval(sext(bitcast(a) == bitcast(b))); + } + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + return eval(sext(fcmp_ord(a == b)) | sext(bitcast(a) == bitcast(b))); + } + else + { + return eval(sext(fcmp_ord(a == b))); + } + }); + + set_vr(op.rt, fceq(get_vr(op.ra), get_vr(op.rb))); + } + + template + static llvm_calli fcmeq(T&& a, U&& b) + { + return {"spu_fcmeq", {std::forward(a), std::forward(b)}}; + } + + void FCMEQ(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, sext(fcmp_ord(fabs(get_vr(op.ra)) == fabs(get_vr(op.rb))))); + return; + } + + register_intrinsic("spu_fcmeq", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + + const value_t ab[2]{a, b}; + + std::bitset<2> safe_float_compare(0); + std::bitset<2> safe_int_compare(0); + + for (u32 i = 0; i < 2; i++) + { + if (auto [ok, data] = get_const_vector(ab[i].value, m_pos, __LINE__ + i); ok) + { + safe_float_compare.set(i); + safe_int_compare.set(i); + + for (u32 j = 0; j < 4; j++) + { + const u32 value = data._u32[j]; + const u8 exponent = static_cast(value >> 23); + + // unsafe if nan + if (exponent == 255) + { + safe_float_compare.reset(i); + } + + // unsafe if denormal or 0 + if (!exponent) + { + safe_int_compare.reset(i); + } + } + } + } + + const auto fa = eval(fabs(a)); + const auto fb = eval(fabs(b)); + + if (safe_float_compare.any()) + { + return eval(sext(fcmp_ord(fa == fb))); + } + + if (safe_int_compare.any()) + { + return eval(sext(bitcast(fa) == bitcast(fb))); + } + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + return eval(sext(fcmp_ord(fa == fb)) | sext(bitcast(fa) == bitcast(fb))); + } + else + { + return eval(sext(fcmp_ord(fa == fb))); + } + }); + + set_vr(op.rt, fcmeq(get_vr(op.ra), get_vr(op.rb))); + } + + value_t fma32x4(value_t a, value_t b, value_t c) + { + // Optimization: Emit only a floating multiply if the addend is zero + // This is odd since SPU code could just use the FM instruction, but it seems common enough + if (auto [ok, data] = get_const_vector(c.value, m_pos); ok) + { + if (is_spu_float_zero(data, -1)) + { + return eval(a * b); + } + + if (!m_use_fma && is_spu_float_zero(data, +1)) + { + return eval(a * b + fsplat(0.f)); + } + } + + if ([&]() + { + if (auto [ok, data] = get_const_vector(a.value, m_pos); ok) + { + if (!is_spu_float_zero(data, +1)) + { + return false; + } + + if (auto [ok0, data0] = get_const_vector(b.value, m_pos); ok0) + { + if (is_spu_float_zero(data0, +1)) + { + return true; + } + } + } + + if (auto [ok, data] = get_const_vector(a.value, m_pos); ok) + { + if (!is_spu_float_zero(data, -1)) + { + return false; + } + + if (auto [ok0, data0] = get_const_vector(b.value, m_pos); ok0) + { + if (is_spu_float_zero(data0, -1)) + { + return true; + } + } + } + + return false; + }()) + { + // Just return the added value if both a and b is +0 or -0 (+0 and -0 arent't allowed alone) + return c; + } + + if (m_use_fma) + { + return eval(fmuladd(a, b, c, true)); + } + + // Convert to doubles + const auto xa = fpcast(a); + const auto xb = fpcast(b); + const auto xc = fpcast(c); + const auto xr = fmuladd(xa, xb, xc, false); + return eval(fpcast(xr)); + } + + template + static llvm_calli fnms(T&& a, U&& b, V&& c) + { + return {"spu_fnms", {std::forward(a), std::forward(b), std::forward(c)}}; + } + + void FNMS(spu_opcode_t op) + { + // See FMA. + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + const auto [a, b, c] = get_vrs(op.ra, op.rb, op.rc); + set_vr(op.rt4, fmuladd(-a, b, c)); + return; + } + + register_intrinsic("spu_fnms", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + const auto c = value(ci->getOperand(2)); + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate || g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::relaxed) + { + return fma32x4(eval(-clamp_smax(a)), clamp_smax(b), c); + } + else + { + return fma32x4(eval(-a), b, c); + } + }); + + set_vr(op.rt4, fnms(get_vr(op.ra), get_vr(op.rb), get_vr(op.rc))); + } + + template + static llvm_calli fma(T&& a, U&& b, V&& c) + { + return {"spu_fma", {std::forward(a), std::forward(b), std::forward(c)}}; + } + + void FMA(spu_opcode_t op) + { + // Hardware FMA produces the same result as multiple + add on the limited double range (xfloat). + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + const auto [a, b, c] = get_vrs(op.ra, op.rb, op.rc); + set_vr(op.rt4, fmuladd(a, b, c)); + return; + } + + register_intrinsic("spu_fma", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + const auto c = value(ci->getOperand(2)); + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + const auto ma = sext(fcmp_uno(a != fsplat(0.))); + const auto mb = sext(fcmp_uno(b != fsplat(0.))); + const auto ca = bitcast(bitcast(a) & mb); + const auto cb = bitcast(bitcast(b) & ma); + return fma32x4(eval(ca), eval(cb), c); + } + else + { + return fma32x4(a, b, c); + } + }); + + const auto [a, b, c] = get_vrs(op.ra, op.rb, op.rc); + + static const auto MT = match(); + + // Match sqrt + if (auto [ok_fnma, a1, b1] = match_expr(a, fnms(MT, MT, fsplat(1.00000011920928955078125))); ok_fnma) + { + if (auto [ok_fm2, a2] = match_expr(b, fm(MT, fsplat(0.5))); ok_fm2 && a2.eq(b1)) + { + if (auto [ok_fm1, a3, b3] = match_expr(c, fm(MT, MT)); ok_fm1 && a3.eq(a1)) + { + if (auto [ok_sqrte, src] = match_expr(a3, spu_rsqrte(MT)); ok_sqrte && src.eq(b3)) + { + erase_stores(a, b, c, a3); + set_vr(op.rt4, fsqrt(fabs(src))); + return; + } + } + } + } + + // Match division (fast) + if (auto [ok_fnma, divb, diva] = match_expr(a, fnms(c, MT, MT)); ok_fnma) + { + if (auto [ok_fm] = match_expr(c, fm(diva, b)); ok_fm) + { + if (auto [ok_re] = match_expr(b, spu_re(divb)); ok_re) + { + erase_stores(b, c); + set_vr(op.rt4, diva / divb); + return; + } + } + } + + set_vr(op.rt4, fma(a, b, c)); + } + + template + static llvm_calli fms(T&& a, U&& b, V&& c) + { + return {"spu_fms", {std::forward(a), std::forward(b), std::forward(c)}}; + } + + void FMS(spu_opcode_t op) + { + // See FMA. + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + const auto [a, b, c] = get_vrs(op.ra, op.rb, op.rc); + set_vr(op.rt4, fmuladd(a, b, -c)); + return; + } + + register_intrinsic("spu_fms", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + const auto b = value(ci->getOperand(1)); + const auto c = value(ci->getOperand(2)); + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + return fma32x4(clamp_smax(a), clamp_smax(b), eval(-c)); + } + else + { + return fma32x4(a, b, eval(-c)); + } + }); + + set_vr(op.rt4, fms(get_vr(op.ra), get_vr(op.rb), get_vr(op.rc))); + } + + template + static llvm_calli fi(T&& a, U&& b) + { + return {"spu_fi", {std::forward(a), std::forward(b)}}; + } + + template + static llvm_calli spu_re(T&& a) + { + return {"spu_re", {std::forward(a)}}; + } + + template + static llvm_calli spu_rsqrte(T&& a) + { + return {"spu_rsqrte", {std::forward(a)}}; + } + + void FI(spu_opcode_t op) + { + // TODO + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + set_vr(op.rt, get_vr(op.rb)); + // const auto [a, b] = get_vrs(op.ra, op.rb); + + // const auto mask_se = splat(0xfff0000000000000ull); + // const auto mask_bf = splat(0x000fff8000000000ull); + // const auto mask_sf = splat(0x0000007fe0000000ull); + // const auto mask_yf = splat(0x0000ffffe0000000ull); + + // const auto base = bitcast((bitcast(b) & mask_bf) | 0x3ff0000000000000ull); + // const auto step = fpcast(bitcast(b) & mask_sf) * fsplat(std::exp2(-13.f)); + // const auto yval = fpcast(bitcast(a) & mask_yf) * fsplat(std::exp2(-19.f)); + // set_vr(op.rt, bitcast((bitcast(b) & mask_se) | (bitcast(base - step * yval) & ~mask_se))); + return; + } + + register_intrinsic("spu_fi", [&](llvm::CallInst* ci) + { + const auto a = bitcast(value(ci->getOperand(0))); + const auto b = bitcast(value(ci->getOperand(1))); + + const auto base = (b & 0x007ffc00u) << 9; // Base fraction + const auto ymul = (b & 0x3ff) * (a & 0x7ffff); // Step fraction * Y fraction (fixed point at 2^-32) + const auto bnew = bitcast((base - ymul) >> 9) + (sext(ymul <= base) & (1 << 23)); // Subtract and correct invisible fraction bit + return bitcast((b & 0xff800000u) | (bitcast(fpcast(bnew)) & ~0xff800000u)); // Inject old sign and exponent + }); + + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::approximate) + { + register_intrinsic("spu_re", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + // Gives accuracy penalty, frest result is within one newton-raphson iteration for accuracy + const auto approx_result = fsplat(0.999875069f) / a; + return approx_result; + }); + + register_intrinsic("spu_rsqrte", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + // Gives accuracy penalty, frsqest result is within one newton-raphson iteration for accuracy + const auto approx_result = fsplat(0.999763668f) / fsqrt(fabs(a)); + return approx_result; + }); + } + else + { + // For relaxed use intrinsics, those make the results vary per cpu + register_intrinsic("spu_re", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + return fre(a); + }); + + register_intrinsic("spu_rsqrte", [&](llvm::CallInst* ci) + { + const auto a = value(ci->getOperand(0)); + return frsqe(a); + }); + } + + const auto [a, b] = get_vrs(op.ra, op.rb); + + if (const auto [ok, mb] = match_expr(b, frest(match())); ok && mb.eq(a)) + { + erase_stores(b); + set_vr(op.rt, spu_re(a)); + return; + } + + if (const auto [ok, mb] = match_expr(b, frsqest(match())); ok && mb.eq(a)) + { + erase_stores(b); + set_vr(op.rt, spu_rsqrte(a)); + return; + } + + const auto r = eval(fi(a, b)); + if (!m_interp_magn) + spu_log.todo("[%s:0x%05x] Unmatched spu_fi found", m_hash, m_pos); + + set_vr(op.rt, r); + } + + void CFLTS(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + value_t a = get_vr(op.ra); + value_t s; + if (m_interp_magn) + s = eval(vsplat(bitcast(((1023 + 173) - get_imm(op.i8)) << 52))); + else + s = eval(fsplat(std::exp2(static_cast(173 - op.i8)))); + if (op.i8 != 173 || m_interp_magn) + a = eval(a * s); + + value_t r; + + if (auto ca = llvm::dyn_cast(a.value)) + { + const f64 data[4] + { + ca->getElementAsDouble(0), + ca->getElementAsDouble(1), + ca->getElementAsDouble(2), + ca->getElementAsDouble(3) + }; + + v128 result; + + for (u32 i = 0; i < 4; i++) + { + if (data[i] >= std::exp2(31.f)) + { + result._s32[i] = smax; + } + else if (data[i] < std::exp2(-31.f)) + { + result._s32[i] = smin; + } + else + { + result._s32[i] = static_cast(data[i]); + } + } + + r.value = make_const_vector(result, get_type()); + set_vr(op.rt, r); + return; + } + + if (llvm::isa(a.value)) + { + set_vr(op.rt, splat(0)); + return; + } + + r.value = m_ir->CreateFPToSI(a.value, get_type()); + set_vr(op.rt, r ^ sext(fcmp_ord(a >= fsplat(std::exp2(31.f))))); + } + else + { + value_t a = get_vr(op.ra); + value_t s; + if (m_interp_magn) + s = eval(vsplat(load_const(m_scale_float_to, get_imm(op.i8)))); + else + s = eval(fsplat(std::exp2(static_cast(static_cast(173 - op.i8))))); + if (op.i8 != 173 || m_interp_magn) + a = eval(a * s); + + value_t r; + r.value = m_ir->CreateFPToSI(a.value, get_type()); + set_vr(op.rt, r ^ sext(bitcast(a) > splat(((31 + 127) << 23) - 1))); + } + } + + void CFLTU(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + value_t a = get_vr(op.ra); + value_t s; + if (m_interp_magn) + s = eval(vsplat(bitcast(((1023 + 173) - get_imm(op.i8)) << 52))); + else + s = eval(fsplat(std::exp2(static_cast(173 - op.i8)))); + if (op.i8 != 173 || m_interp_magn) + a = eval(a * s); + + value_t r; + + if (auto ca = llvm::dyn_cast(a.value)) + { + const f64 data[4] + { + ca->getElementAsDouble(0), + ca->getElementAsDouble(1), + ca->getElementAsDouble(2), + ca->getElementAsDouble(3) + }; + + v128 result; + + for (u32 i = 0; i < 4; i++) + { + if (data[i] >= std::exp2(32.f)) + { + result._u32[i] = umax; + } + else if (data[i] < 0.) + { + result._u32[i] = 0; + } + else + { + result._u32[i] = static_cast(data[i]); + } + } + + r.value = make_const_vector(result, get_type()); + set_vr(op.rt, r); + return; + } + + if (llvm::isa(a.value)) + { + set_vr(op.rt, splat(0)); + return; + } + + r.value = m_ir->CreateFPToUI(a.value, get_type()); + set_vr(op.rt, select(fcmp_ord(a >= fsplat(std::exp2(32.f))), splat(-1), r & sext(fcmp_ord(a >= fsplat(0.))))); + } + else + { + value_t a = get_vr(op.ra); + value_t s; + if (m_interp_magn) + s = eval(vsplat(load_const(m_scale_float_to, get_imm(op.i8)))); + else + s = eval(fsplat(std::exp2(static_cast(static_cast(173 - op.i8))))); + if (op.i8 != 173 || m_interp_magn) + a = eval(a * s); + + value_t r; + + if (m_use_avx512) + { + const auto sc = eval(bitcast(max(bitcast(a),splat(0x0)))); + r.value = m_ir->CreateFPToUI(sc.value, get_type()); + set_vr(op.rt, r); + return; + } + + r.value = m_ir->CreateFPToUI(a.value, get_type()); + set_vr(op.rt, select(bitcast(a) > splat(((32 + 127) << 23) - 1), splat(-1), r & ~(bitcast(a) >> 31))); + } + } + + void CSFLT(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + value_t a = get_vr(op.ra); + value_t r; + + if (auto [ok, data] = get_const_vector(a.value, m_pos); ok) + { + r.value = build(data._s32[0], data._s32[1], data._s32[2], data._s32[3]).eval(m_ir); + } + else + { + r.value = m_ir->CreateSIToFP(a.value, get_type()); + } + + value_t s; + if (m_interp_magn) + s = eval(vsplat(bitcast((get_imm(op.i8) + (1023 - 155)) << 52))); + else + s = eval(fsplat(std::exp2(static_cast(op.i8 - 155)))); + if (op.i8 != 155 || m_interp_magn) + r = eval(r * s); + set_vr(op.rt, r); + } + else + { + value_t r; + r.value = m_ir->CreateSIToFP(get_vr(op.ra).value, get_type()); + value_t s; + if (m_interp_magn) + s = eval(vsplat(load_const(m_scale_to_float, get_imm(op.i8)))); + else + s = eval(fsplat(std::exp2(static_cast(static_cast(op.i8 - 155))))); + if (op.i8 != 155 || m_interp_magn) + r = eval(r * s); + set_vr(op.rt, r); + } + } + + void CUFLT(spu_opcode_t op) + { + if (g_cfg.core.spu_xfloat_accuracy == xfloat_accuracy::accurate) + { + value_t a = get_vr(op.ra); + value_t r; + + if (auto [ok, data] = get_const_vector(a.value, m_pos); ok) + { + r.value = build(data._u32[0], data._u32[1], data._u32[2], data._u32[3]).eval(m_ir); + } + else + { + r.value = m_ir->CreateUIToFP(a.value, get_type()); + } + + value_t s; + if (m_interp_magn) + s = eval(vsplat(bitcast((get_imm(op.i8) + (1023 - 155)) << 52))); + else + s = eval(fsplat(std::exp2(static_cast(op.i8 - 155)))); + if (op.i8 != 155 || m_interp_magn) + r = eval(r * s); + set_vr(op.rt, r); + } + else + { + value_t r; + r.value = m_ir->CreateUIToFP(get_vr(op.ra).value, get_type()); + value_t s; + if (m_interp_magn) + s = eval(vsplat(load_const(m_scale_to_float, get_imm(op.i8)))); + else + s = eval(fsplat(std::exp2(static_cast(static_cast(op.i8 - 155))))); + if (op.i8 != 155 || m_interp_magn) + r = eval(r * s); + set_vr(op.rt, r); + } + } + + void make_store_ls(value_t addr, value_t data) + { + const auto bswapped = byteswap(data); + m_ir->CreateStore(bswapped.eval(m_ir), m_ir->CreateGEP(get_type(), m_lsptr, addr.value)); + } + + auto make_load_ls(value_t addr) + { + value_t data; + data.value = m_ir->CreateLoad(get_type(), m_ir->CreateGEP(get_type(), m_lsptr, addr.value)); + return byteswap(data); + } + + void STQX(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + + for (auto pair : std::initializer_list, value_t>>{{a, b}, {b, a}}) + { + if (auto [ok, data] = get_const_vector(pair.first.value, m_pos); ok) + { + data._u32[3] %= SPU_LS_SIZE; + + if (data._u32[3] % 0x10 == 0) + { + value_t addr = eval(splat(data._u32[3]) + zext(extract(pair.second, 3) & 0x3fff0)); + make_store_ls(addr, get_vr(op.rt)); + return; + } + } + } + + value_t addr = eval(zext((extract(a, 3) + extract(b, 3)) & 0x3fff0)); + make_store_ls(addr, get_vr(op.rt)); + } + + void LQX(spu_opcode_t op) + { + const auto a = get_vr(op.ra); + const auto b = get_vr(op.rb); + + for (auto pair : std::initializer_list, value_t>>{{a, b}, {b, a}}) + { + if (auto [ok, data] = get_const_vector(pair.first.value, m_pos); ok) + { + data._u32[3] %= SPU_LS_SIZE; + + if (data._u32[3] % 0x10 == 0) + { + value_t addr = eval(splat(data._u32[3]) + zext(extract(pair.second, 3) & 0x3fff0)); + set_vr(op.rt, make_load_ls(addr)); + return; + } + } + } + + value_t addr = eval(zext((extract(a, 3) + extract(b, 3)) & 0x3fff0)); + set_vr(op.rt, make_load_ls(addr)); + } + + void STQA(spu_opcode_t op) + { + value_t addr = eval((get_imm(op.i16, false) << 2) & 0x3fff0); + make_store_ls(addr, get_vr(op.rt)); + } + + void LQA(spu_opcode_t op) + { + value_t addr = eval((get_imm(op.i16, false) << 2) & 0x3fff0); + set_vr(op.rt, make_load_ls(addr)); + } + + llvm::Value* get_pc_as_u64(u32 addr) + { + return m_ir->CreateAdd(m_ir->CreateZExt(m_base_pc, get_type()), m_ir->getInt64(addr - m_base)); + } + + void STQR(spu_opcode_t op) // + { + value_t addr; + addr.value = m_interp_magn ? m_ir->CreateZExt(m_interp_pc, get_type()) : get_pc_as_u64(m_pos); + addr = eval(((get_imm(op.i16, false) << 2) + addr) & (m_interp_magn ? 0x3fff0 : ~0xf)); + make_store_ls(addr, get_vr(op.rt)); + } + + void LQR(spu_opcode_t op) // + { + value_t addr; + addr.value = m_interp_magn ? m_ir->CreateZExt(m_interp_pc, get_type()) : get_pc_as_u64(m_pos); + addr = eval(((get_imm(op.i16, false) << 2) + addr) & (m_interp_magn ? 0x3fff0 : ~0xf)); + set_vr(op.rt, make_load_ls(addr)); + } + + void STQD(spu_opcode_t op) + { + if (m_finfo && m_finfo->fn) + { + if (op.rt <= s_reg_sp || (op.rt >= s_reg_80 && op.rt <= s_reg_127)) + { + if (m_block->bb->reg_save_dom[op.rt] && get_reg_raw(op.rt) == m_finfo->load[op.rt]) + { + return; + } + } + } + + value_t addr = eval(zext(extract(get_vr(op.ra), 3) & 0x3fff0) + (get_imm(op.si10) << 4)); + make_store_ls(addr, get_vr(op.rt)); + } + + void LQD(spu_opcode_t op) + { + value_t addr = eval(zext(extract(get_vr(op.ra), 3) & 0x3fff0) + (get_imm(op.si10) << 4)); + set_vr(op.rt, make_load_ls(addr)); + } + + void make_halt(value_t cond) + { + const auto next = llvm::BasicBlock::Create(m_context, "", m_function); + const auto halt = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->CreateCondBr(cond.value, halt, next, m_md_unlikely); + m_ir->SetInsertPoint(halt); + if (m_interp_magn) + m_ir->CreateStore(m_function->getArg(2), spu_ptr(&spu_thread::pc)); + else + update_pc(); + const auto ptr = _ptr(m_memptr, 0xffdead00); + m_ir->CreateStore(m_ir->getInt32("HALT"_u32), ptr); + m_ir->CreateBr(next); + m_ir->SetInsertPoint(next); + } + + void HGT(spu_opcode_t op) + { + const auto cond = eval(extract(get_vr(op.ra), 3) > extract(get_vr(op.rb), 3)); + make_halt(cond); + } + + void HEQ(spu_opcode_t op) + { + const auto cond = eval(extract(get_vr(op.ra), 3) == extract(get_vr(op.rb), 3)); + make_halt(cond); + } + + void HLGT(spu_opcode_t op) + { + const auto cond = eval(extract(get_vr(op.ra), 3) > extract(get_vr(op.rb), 3)); + make_halt(cond); + } + + void HGTI(spu_opcode_t op) + { + const auto cond = eval(extract(get_vr(op.ra), 3) > get_imm(op.si10)); + make_halt(cond); + } + + void HEQI(spu_opcode_t op) + { + const auto cond = eval(extract(get_vr(op.ra), 3) == get_imm(op.si10)); + make_halt(cond); + } + + void HLGTI(spu_opcode_t op) + { + const auto cond = eval(extract(get_vr(op.ra), 3) > get_imm(op.si10)); + make_halt(cond); + } + + void HBR([[maybe_unused]] spu_opcode_t op) // + { + // TODO: use the hint. + } + + void HBRA([[maybe_unused]] spu_opcode_t op) // + { + // TODO: use the hint. + } + + void HBRR([[maybe_unused]] spu_opcode_t op) // + { + // TODO: use the hint. + } + + // TODO + static u32 exec_check_interrupts(spu_thread* _spu, u32 addr) + { + _spu->set_interrupt_status(true); + + if (_spu->ch_events.load().count) + { + _spu->interrupts_enabled = false; + _spu->srr0 = addr; + + // Test for BR/BRA instructions (they are equivalent at zero pc) + const u32 br = _spu->_ref(0); + + if ((br & 0xfd80007f) == 0x30000000) + { + return (br >> 5) & 0x3fffc; + } + + return 0; + } + + return addr; + } + + llvm::BasicBlock* add_block_indirect(spu_opcode_t op, value_t addr, bool ret = true) + { + if (m_interp_magn) + { + m_interp_bblock = llvm::BasicBlock::Create(m_context, "", m_function); + + const auto cblock = m_ir->GetInsertBlock(); + const auto result = llvm::BasicBlock::Create(m_context, "", m_function); + const auto e_exec = llvm::BasicBlock::Create(m_context, "", m_function); + const auto d_test = llvm::BasicBlock::Create(m_context, "", m_function); + const auto d_exec = llvm::BasicBlock::Create(m_context, "", m_function); + const auto d_done = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->SetInsertPoint(result); + m_ir->CreateCondBr(get_imm(op.e).value, e_exec, d_test, m_md_unlikely); + m_ir->SetInsertPoint(e_exec); + const auto e_addr = call("spu_check_interrupts", &exec_check_interrupts, m_thread, addr.value); + m_ir->CreateBr(d_test); + m_ir->SetInsertPoint(d_test); + const auto target = m_ir->CreatePHI(get_type(), 2); + target->addIncoming(addr.value, result); + target->addIncoming(e_addr, e_exec); + m_ir->CreateCondBr(get_imm(op.d).value, d_exec, d_done, m_md_unlikely); + m_ir->SetInsertPoint(d_exec); + m_ir->CreateStore(m_ir->getFalse(), spu_ptr(&spu_thread::interrupts_enabled)); + m_ir->CreateBr(d_done); + m_ir->SetInsertPoint(d_done); + m_ir->CreateBr(m_interp_bblock); + m_ir->SetInsertPoint(cblock); + m_interp_pc = target; + return result; + } + + if (llvm::isa(addr.value)) + { + // Fixed branch excludes the possibility it's a function return (TODO) + ret = false; + } + + if (m_finfo && m_finfo->fn && op.opcode) + { + const auto cblock = m_ir->GetInsertBlock(); + const auto result = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->SetInsertPoint(result); + ret_function(); + m_ir->SetInsertPoint(cblock); + return result; + } + + // Load stack addr if necessary + value_t sp; + + if (ret && g_cfg.core.spu_block_size != spu_block_size_type::safe) + { + if (op.opcode) + { + sp = eval(extract(get_reg_fixed(1), 3) & 0x3fff0); + } + else + { + sp.value = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::gpr, 1, &v128::_u32, 3)); + } + } + + const auto cblock = m_ir->GetInsertBlock(); + const auto result = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->SetInsertPoint(result); + + if (op.e) + { + addr.value = call("spu_check_interrupts", &exec_check_interrupts, m_thread, addr.value); + } + + if (op.d) + { + m_ir->CreateStore(m_ir->getFalse(), spu_ptr(&spu_thread::interrupts_enabled)); + } + + m_ir->CreateStore(addr.value, spu_ptr(&spu_thread::pc)); + + if (ret && g_cfg.core.spu_block_size >= spu_block_size_type::mega) + { + // Compare address stored in stack mirror with addr + const auto stack0 = eval(zext(sp) + ::offset32(&spu_thread::stack_mirror)); + const auto stack1 = eval(stack0 + 8); + const auto _ret = m_ir->CreateLoad(get_type(), m_ir->CreateGEP(get_type(), m_thread, stack0.value)); + const auto link = m_ir->CreateLoad(get_type(), m_ir->CreateGEP(get_type(), m_thread, stack1.value)); + const auto fail = llvm::BasicBlock::Create(m_context, "", m_function); + const auto done = llvm::BasicBlock::Create(m_context, "", m_function); + const auto next = llvm::BasicBlock::Create(m_context, "", m_function); + m_ir->CreateCondBr(m_ir->CreateICmpEQ(addr.value, m_ir->CreateTrunc(link, get_type())), next, fail, m_md_likely); + m_ir->SetInsertPoint(next); + const auto cmp2 = m_ir->CreateLoad(get_type(), m_ir->CreateGEP(get_type(), m_lsptr, addr.value)); + m_ir->CreateCondBr(m_ir->CreateICmpEQ(cmp2, m_ir->CreateTrunc(_ret, get_type())), done, fail, m_md_likely); + m_ir->SetInsertPoint(done); + + // Clear stack mirror and return by tail call to the provided return address + m_ir->CreateStore(splat(-1).eval(m_ir), m_ir->CreateGEP(get_type(), m_thread, stack0.value)); + const auto targ = m_ir->CreateAdd(m_ir->CreateLShr(_ret, 32), get_segment_base()); + const auto type = m_finfo->chunk->getFunctionType(); + const auto fval = m_ir->CreateIntToPtr(targ, type->getPointerTo()); + tail_chunk({type, fval}, m_ir->CreateTrunc(m_ir->CreateLShr(link, 32), get_type())); + m_ir->SetInsertPoint(fail); + } + + if (g_cfg.core.spu_block_size >= spu_block_size_type::mega) + { + // Try to load chunk address from the function table + const auto fail = llvm::BasicBlock::Create(m_context, "", m_function); + const auto done = llvm::BasicBlock::Create(m_context, "", m_function); + const auto ad32 = m_ir->CreateSub(addr.value, m_base_pc); + m_ir->CreateCondBr(m_ir->CreateICmpULT(ad32, m_ir->getInt32(m_size)), done, fail, m_md_likely); + m_ir->SetInsertPoint(done); + + const auto ad64 = m_ir->CreateZExt(ad32, get_type()); + const auto pptr = dyn_cast(m_ir->CreateGEP(m_function_table->getValueType(), m_function_table, {m_ir->getInt64(0), m_ir->CreateLShr(ad64, 2, "", true)})); + tail_chunk({m_dispatch->getFunctionType(), m_ir->CreateLoad(pptr->getResultElementType(), pptr)}); + m_ir->SetInsertPoint(fail); + } + + tail_chunk(nullptr); + m_ir->SetInsertPoint(cblock); + return result; + } + + llvm::BasicBlock* add_block_next() + { + if (m_interp_magn) + { + const auto cblock = m_ir->GetInsertBlock(); + m_ir->SetInsertPoint(m_interp_bblock); + const auto target = m_ir->CreatePHI(get_type(), 2); + target->addIncoming(m_interp_pc_next, cblock); + target->addIncoming(m_interp_pc, m_interp_bblock->getSinglePredecessor()); + m_ir->SetInsertPoint(cblock); + m_interp_pc = target; + return m_interp_bblock; + } + + return add_block(m_pos + 4); + } + + void BIZ(spu_opcode_t op) // + { + if (m_block) m_block->block_end = m_ir->GetInsertBlock(); + + const auto rt = get_vr(op.rt); + + // Checking for zero doesn't care about the order of the bytes, + // so load the data before it's byteswapped + if (auto [ok, as] = match_expr(rt, byteswap(match())); ok) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto cond = eval(extract(bitcast(as), 0) == 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + return; + } + + const auto ox = get_vr(op.rt); + + // Instead of extracting the value generated by orx, just test the input to orx with ptest + if (auto [ok, as] = match_expr(ox, orx(match())); ok) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto a = extract(bitcast(as), 0); + const auto b = extract(bitcast(as), 1); + const auto cond = eval((a | b) == 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + return; + } + + + // Check sign bit instead (optimization) + if (match_vr(op.rt, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + const auto a = get_vr(op.rt); + const auto cond = eval(bitcast(trunc(a)) >= 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + return true; + } + + return false; + })) + { + return; + } + + const auto cond = eval(extract(get_vr(op.rt), 3) == 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + } + + void BINZ(spu_opcode_t op) // + { + if (m_block) m_block->block_end = m_ir->GetInsertBlock(); + + const auto rt = get_vr(op.rt); + + // Checking for zero doesn't care about the order of the bytes, + // so load the data before it's byteswapped + if (auto [ok, as] = match_expr(rt, byteswap(match())); ok) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto cond = eval(extract(bitcast(as), 0) != 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + return; + } + + const auto ox = get_vr(op.rt); + + // Instead of extracting the value generated by orx, just test the input to orx with ptest + if (auto [ok, as] = match_expr(ox, orx(match())); ok) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto a = extract(bitcast(as), 0); + const auto b = extract(bitcast(as), 1); + const auto cond = eval((a | b) != 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + return; + } + + + // Check sign bit instead (optimization) + if (match_vr(op.rt, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + const auto a = get_vr(op.rt); + const auto cond = eval(bitcast(trunc(a)) < 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + return true; + } + + return false; + })) + { + return; + } + + const auto cond = eval(extract(get_vr(op.rt), 3) != 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + } + + void BIHZ(spu_opcode_t op) // + { + if (m_block) m_block->block_end = m_ir->GetInsertBlock(); + + // Check sign bits of 2 vector elements (optimization) + if (match_vr(op.rt, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + const auto a = get_vr(op.rt); + const auto cond = eval((bitcast(trunc(a)) & 0x3000) == 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + return true; + } + + return false; + })) + { + return; + } + + const auto cond = eval(extract(get_vr(op.rt), 6) == 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + } + + void BIHNZ(spu_opcode_t op) // + { + if (m_block) m_block->block_end = m_ir->GetInsertBlock(); + + // Check sign bits of 2 vector elements (optimization) + if (match_vr(op.rt, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + const auto a = get_vr(op.rt); + const auto cond = eval((bitcast(trunc(a)) & 0x3000) != 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + return true; + } + + return false; + })) + { + return; + } + + const auto cond = eval(extract(get_vr(op.rt), 6) != 0); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(cond.value, target, add_block_next()); + } + + void BI(spu_opcode_t op) // + { + if (m_block) m_block->block_end = m_ir->GetInsertBlock(); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + + if (m_interp_magn) + { + m_ir->CreateBr(add_block_indirect(op, addr)); + return; + } + + // Create jump table if necessary (TODO) + const auto tfound = m_targets.find(m_pos); + + if (op.d && tfound != m_targets.end() && tfound->second.size() == 1 && tfound->second[0] == spu_branch_target(m_pos, 1)) + { + // Interrupts-disable pattern + m_ir->CreateStore(m_ir->getFalse(), spu_ptr(&spu_thread::interrupts_enabled)); + return; + } + + if (!op.d && !op.e && tfound != m_targets.end() && tfound->second.size() > 1) + { + // Shift aligned address for switch + const auto addrfx = m_ir->CreateSub(addr.value, m_base_pc); + const auto sw_arg = m_ir->CreateLShr(addrfx, 2, "", true); + + // Initialize jump table targets + std::map targets; + + for (u32 target : tfound->second) + { + if (m_block_info[target / 4]) + { + targets.emplace(target, nullptr); + } + } + + // Initialize target basic blocks + for (auto& pair : targets) + { + pair.second = add_block(pair.first); + } + + if (targets.empty()) + { + // Emergency exit + spu_log.error("[%s] [0x%05x] No jump table targets at 0x%05x (%u)", m_hash, m_entry, m_pos, tfound->second.size()); + m_ir->CreateBr(add_block_indirect(op, addr)); + return; + } + + // Get jump table bounds (optimization) + const u32 start = targets.begin()->first; + const u32 end = targets.rbegin()->first + 4; + + // Emit switch instruction aiming for a jumptable in the end (indirectbr could guarantee it) + const auto sw = m_ir->CreateSwitch(sw_arg, llvm::BasicBlock::Create(m_context, "", m_function), (end - start) / 4); + + for (u32 pos = start; pos < end; pos += 4) + { + if (m_block_info[pos / 4] && targets.count(pos)) + { + const auto found = targets.find(pos); + + if (found != targets.end()) + { + sw->addCase(m_ir->getInt32(pos / 4 - m_base / 4), found->second); + continue; + } + } + + sw->addCase(m_ir->getInt32(pos / 4 - m_base / 4), sw->getDefaultDest()); + } + + // Exit function on unexpected target + m_ir->SetInsertPoint(sw->getDefaultDest()); + m_ir->CreateStore(addr.value, spu_ptr(&spu_thread::pc)); + + if (m_finfo && m_finfo->fn) + { + // Can't afford external tail call in true functions + m_ir->CreateStore(m_ir->getInt32("BIJT"_u32), _ptr(m_memptr, 0xffdead20)); + m_ir->CreateCall(m_test_state, {m_thread}); + m_ir->CreateBr(sw->getDefaultDest()); + } + else + { + tail_chunk(nullptr); + } + } + else + { + // Simple indirect branch + m_ir->CreateBr(add_block_indirect(op, addr)); + } + } + + void BISL(spu_opcode_t op) // + { + if (m_block) m_block->block_end = m_ir->GetInsertBlock(); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + set_link(op); + m_ir->CreateBr(add_block_indirect(op, addr, false)); + } + + void IRET(spu_opcode_t op) // + { + if (m_block) m_block->block_end = m_ir->GetInsertBlock(); + value_t srr0; + srr0.value = m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::srr0)); + m_ir->CreateBr(add_block_indirect(op, srr0)); + } + + void BISLED(spu_opcode_t op) // + { + if (m_block) m_block->block_end = m_ir->GetInsertBlock(); + const auto addr = eval(extract(get_vr(op.ra), 3) & 0x3fffc); + set_link(op); + const auto mask = m_ir->CreateTrunc(m_ir->CreateLShr(m_ir->CreateLoad(get_type(), spu_ptr(&spu_thread::ch_events), true), 32), get_type()); + const auto res = call("spu_get_events", &exec_get_events, m_thread, mask); + const auto target = add_block_indirect(op, addr); + m_ir->CreateCondBr(m_ir->CreateICmpNE(res, m_ir->getInt32(0)), target, add_block_next()); + } + + void BRZ(spu_opcode_t op) // + { + if (m_interp_magn) + { + value_t target; + target.value = m_interp_pc; + target = eval((target + (get_imm(op.i16, false) << 2)) & 0x3fffc); + m_interp_pc = m_ir->CreateSelect(eval(extract(get_vr(op.rt), 3) == 0).value, target.value, m_interp_pc_next); + return; + } + + const u32 target = spu_branch_target(m_pos, op.i16); + + const auto rt = get_vr(op.rt); + + // Checking for zero doesn't care about the order of the bytes, + // so load the data before it's byteswapped + if (auto [ok, as] = match_expr(rt, byteswap(match())); ok) + { + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto cond = eval(extract(bitcast(as), 0) == 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + return; + } + } + + const auto ox = get_vr(op.rt); + + // Instead of extracting the value generated by orx, just test the input to orx with ptest + if (auto [ok, as] = match_expr(ox, orx(match())); ok) + { + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto a = extract(bitcast(as), 0); + const auto b = extract(bitcast(as), 1); + const auto cond = eval((a | b) == 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + return; + } + } + + + // Check sign bit instead (optimization) + if (match_vr(op.rt, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto a = get_vr(op.rt); + const auto cond = eval(bitcast(trunc(a)) >= 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + return true; + } + } + + return false; + })) + { + return; + } + + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto cond = eval(extract(get_vr(op.rt), 3) == 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + } + } + + void BRNZ(spu_opcode_t op) // + { + if (m_interp_magn) + { + value_t target; + target.value = m_interp_pc; + target = eval((target + (get_imm(op.i16, false) << 2)) & 0x3fffc); + m_interp_pc = m_ir->CreateSelect(eval(extract(get_vr(op.rt), 3) != 0).value, target.value, m_interp_pc_next); + return; + } + + const u32 target = spu_branch_target(m_pos, op.i16); + + const auto rt = get_vr(op.rt); + + // Checking for zero doesn't care about the order of the bytes, + // so load the data before it's byteswapped + if (auto [ok, as] = match_expr(rt, byteswap(match())); ok) + { + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto cond = eval(extract(bitcast(as), 0) != 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + return; + } + } + + const auto ox = get_vr(op.rt); + + // Instead of extracting the value generated by orx, just test the input to orx with ptest + if (auto [ok, as] = match_expr(ox, orx(match())); ok) + { + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto a = extract(bitcast(as), 0); + const auto b = extract(bitcast(as), 1); + const auto cond = eval((a | b) != 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + return; + } + } + + // Check sign bit instead (optimization) + if (match_vr(op.rt, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto a = get_vr(op.rt); + const auto cond = eval(bitcast(trunc(a)) < 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + return true; + } + } + + return false; + })) + { + return; + } + + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto cond = eval(extract(get_vr(op.rt), 3) != 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + } + } + + void BRHZ(spu_opcode_t op) // + { + if (m_interp_magn) + { + value_t target; + target.value = m_interp_pc; + target = eval((target + (get_imm(op.i16, false) << 2)) & 0x3fffc); + m_interp_pc = m_ir->CreateSelect(eval(extract(get_vr(op.rt), 6) == 0).value, target.value, m_interp_pc_next); + return; + } + + const u32 target = spu_branch_target(m_pos, op.i16); + + // Check sign bits of 2 vector elements (optimization) + if (match_vr(op.rt, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto a = get_vr(op.rt); + const auto cond = eval((bitcast(trunc(a)) & 0x3000) == 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + return true; + } + } + + return false; + })) + { + return; + } + + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto cond = eval(extract(get_vr(op.rt), 6) == 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + } + } + + void BRHNZ(spu_opcode_t op) // + { + if (m_interp_magn) + { + value_t target; + target.value = m_interp_pc; + target = eval((target + (get_imm(op.i16, false) << 2)) & 0x3fffc); + m_interp_pc = m_ir->CreateSelect(eval(extract(get_vr(op.rt), 6) != 0).value, target.value, m_interp_pc_next); + return; + } + + const u32 target = spu_branch_target(m_pos, op.i16); + + // Check sign bits of 2 vector elements (optimization) + if (match_vr(op.rt, [&](auto c, auto MP) + { + using VT = typename decltype(MP)::type; + + if (auto [ok, x] = match_expr(c, sext(match]>())); ok) + { + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto a = get_vr(op.rt); + const auto cond = eval((bitcast(trunc(a)) & 0x3000) != 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + return true; + } + } + + return false; + })) + { + return; + } + + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + const auto cond = eval(extract(get_vr(op.rt), 6) != 0); + m_ir->CreateCondBr(cond.value, add_block(target), add_block(m_pos + 4)); + } + } + + void BRA(spu_opcode_t op) // + { + if (m_interp_magn) + { + m_interp_pc = eval((get_imm(op.i16, false) << 2) & 0x3fffc).value; + return; + } + + const u32 target = spu_branch_target(0, op.i16); + + m_block->block_end = m_ir->GetInsertBlock(); + m_ir->CreateBr(add_block(target, true)); + } + + void BRASL(spu_opcode_t op) // + { + set_link(op); + BRA(op); + } + + void BR(spu_opcode_t op) // + { + if (m_interp_magn) + { + value_t target; + target.value = m_interp_pc; + target = eval((target + (get_imm(op.i16, false) << 2)) & 0x3fffc); + m_interp_pc = target.value; + return; + } + + const u32 target = spu_branch_target(m_pos, op.i16); + + if (target != m_pos + 4) + { + m_block->block_end = m_ir->GetInsertBlock(); + m_ir->CreateBr(add_block(target)); + } + } + + void BRSL(spu_opcode_t op) // + { + set_link(op); + + const u32 target = spu_branch_target(m_pos, op.i16); + + if (m_finfo && m_finfo->fn && target != m_pos + 4) + { + if (auto fn = add_function(target)->fn) + { + call_function(fn); + return; + } + else + { + spu_log.fatal("[0x%x] Can't add function 0x%x", m_pos, target); + return; + } + } + + BR(op); + } + + void set_link(spu_opcode_t op) + { + if (m_interp_magn) + { + value_t next; + next.value = m_interp_pc_next; + set_vr(op.rt, insert(splat(0), 3, next)); + return; + } + + set_vr(op.rt, insert(splat(0), 3, value(get_pc(m_pos + 4)) & 0x3fffc)); + + if (m_finfo && m_finfo->fn) + { + return; + } + + if (g_cfg.core.spu_block_size >= spu_block_size_type::mega && m_block_info[m_pos / 4 + 1] && m_entry_info[m_pos / 4 + 1]) + { + // Store the return function chunk address at the stack mirror + const auto pfunc = add_function(m_pos + 4); + const auto stack0 = eval(zext(extract(get_reg_fixed(1), 3) & 0x3fff0) + ::offset32(&spu_thread::stack_mirror)); + const auto stack1 = eval(stack0 + 8); + const auto rel_ptr = m_ir->CreateSub(m_ir->CreatePtrToInt(pfunc->chunk, get_type()), get_segment_base()); + const auto ptr_plus_op = m_ir->CreateOr(m_ir->CreateShl(rel_ptr, 32), m_ir->getInt64(m_next_op)); + const auto base_plus_pc = m_ir->CreateOr(m_ir->CreateShl(m_ir->CreateZExt(m_base_pc, get_type()), 32), m_ir->getInt64(m_pos + 4)); + m_ir->CreateStore(ptr_plus_op, m_ir->CreateGEP(get_type(), m_thread, stack0.value)); + m_ir->CreateStore(base_plus_pc, m_ir->CreateGEP(get_type(), m_thread, stack1.value)); + } + } + + llvm::Value* get_segment_base() + { + const auto type = llvm::FunctionType::get(get_type(), {}, false); + const auto func = llvm::cast(m_module->getOrInsertFunction("spu_segment_base", type).getCallee()); + m_engine->updateGlobalMapping("spu_segment_base", reinterpret_cast(jit_runtime::alloc(0, 0))); + return m_ir->CreatePtrToInt(func, get_type()); + } + + static decltype(&spu_llvm_recompiler::UNK) decode(u32 op); +}; + +std::unique_ptr spu_recompiler_base::make_llvm_recompiler(u8 magn) +{ + return std::make_unique(magn); +} + +const spu_decoder s_spu_llvm_decoder; + +decltype(&spu_llvm_recompiler::UNK) spu_llvm_recompiler::decode(u32 op) +{ + return s_spu_llvm_decoder.decode(op); +} + +#else + +std::unique_ptr spu_recompiler_base::make_llvm_recompiler(u8 magn) +{ + if (magn) + { + return nullptr; + } + + fmt::throw_exception("LLVM is not available in this build."); +} + +#endif // LLVM_AVAILABLE + +struct spu_llvm_worker +{ + lf_queue> registered; + + void operator()() + { + // SPU LLVM Recompiler instance + const auto compiler = spu_recompiler_base::make_llvm_recompiler(); + compiler->init(); + + // Fake LS + std::vector> ls(0x10000); + + for (auto slice = registered.pop_all();; [&] + { + if (slice) + { + slice.pop_front(); + } + + if (slice || thread_ctrl::state() == thread_state::aborting) + { + return; + } + + thread_ctrl::wait_on(utils::bless>(®istered)[1], 0); + slice = registered.pop_all(); + }()) + { + auto* prog = slice.get(); + + if (thread_ctrl::state() == thread_state::aborting) + { + break; + } + + if (!prog) + { + continue; + } + + if (!prog->second) + { + break; + } + + const auto& func = *prog->second; + + // Get data start + const u32 start = func.lower_bound; + const u32 size0 = ::size32(func.data); + + // Initialize LS with function data only + for (u32 i = 0, pos = start; i < size0; i++, pos += 4) + { + ls[pos / 4] = std::bit_cast>(func.data[i]); + } + + // Call analyser + spu_program func2 = compiler->analyse(ls.data(), func.entry_point); + + if (func2 != func) + { + spu_log.error("[0x%05x] SPU Analyser failed, %u vs %u", func2.entry_point, func2.data.size(), size0); + } + else if (const auto target = compiler->compile(std::move(func2))) + { + // Redirect old function (TODO: patch in multiple places) + const s64 rel = reinterpret_cast(target) - prog->first - 5; + + union + { + u8 bytes[8]; + u64 result; + }; + + bytes[0] = 0xe9; // jmp rel32 + std::memcpy(bytes + 1, &rel, 4); + bytes[5] = 0x90; + bytes[6] = 0x90; + bytes[7] = 0x90; + + atomic_storage::release(*reinterpret_cast(prog->first), result); + } + else + { + spu_log.fatal("[0x%05x] Compilation failed.", func.entry_point); + return; + } + + // Clear fake LS + std::memset(ls.data() + start / 4, 0, 4 * (size0 - 1)); + } + } +}; + +// SPU LLVM recompiler thread context +struct spu_llvm +{ + // Workload + lf_queue> registered; + atomic_ptr> m_workers; + + spu_llvm() + { + // Dependency + g_fxo->init(); + } + + void operator()() + { + if (g_cfg.core.spu_decoder != spu_decoder_type::llvm) + { + return; + } + + // To compile (hash -> item) + std::unordered_multimap> enqueued; + + // Mini-profiler (hash -> number of occurrences) + std::unordered_map, value_hash> samples; + + // For synchronization with profiler thread + stx::init_mutex prof_mutex; + + named_thread profiler("SPU LLVM Profiler"sv, [&]() + { + while (thread_ctrl::state() != thread_state::aborting) + { + { + // Lock if enabled + const auto lock = prof_mutex.access(); + + if (!lock) + { + // Wait when the profiler is disabled + prof_mutex.wait_for_initialized(); + continue; + } + + // Collect profiling samples + idm::select>([&](u32 /*id*/, spu_thread& spu) + { + const u64 name = atomic_storage::load(spu.block_hash); + + if (auto state = +spu.state; !::is_paused(state) && !::is_stopped(state) && cpu_flag::wait - state) + { + const auto found = std::as_const(samples).find(name); + + if (found != std::as_const(samples).end()) + { + const_cast&>(found->second)++; + } + } + }); + } + + // Sleep for a short period if enabled + thread_ctrl::wait_for(20, false); + } + }); + + u32 worker_count = 1; + + if (uint hc = utils::get_thread_count(); hc >= 12) + { + worker_count = hc - 10; + } + + u32 worker_index = 0; + + m_workers = make_single>("SPUW.", worker_count); + auto workers_ptr = m_workers.load(); + auto& workers = *workers_ptr; + + while (thread_ctrl::state() != thread_state::aborting) + { + for (const auto& pair : registered.pop_all()) + { + enqueued.emplace(pair); + + // Interrupt and kick profiler thread + const auto lock = prof_mutex.init_always([&]{}); + + // Register new blocks to collect samples + samples.emplace(pair.first, 0); + } + + if (enqueued.empty()) + { + // Interrupt profiler thread and put it to sleep + static_cast(prof_mutex.reset()); + thread_ctrl::wait_on(utils::bless>(®istered)[1], 0); + continue; + } + + // Find the most used enqueued item + u64 sample_max = 0; + auto found_it = enqueued.begin(); + + for (auto it = enqueued.begin(), end = enqueued.end(); it != end; ++it) + { + const u64 cur = ::at32(std::as_const(samples), it->first); + + if (cur > sample_max) + { + sample_max = cur; + found_it = it; + } + } + + // Start compiling + const spu_program& func = found_it->second->data; + + // Old function pointer (pre-recompiled) + const spu_function_t _old = found_it->second->compiled; + + // Remove item from the queue + enqueued.erase(found_it); + + // Push the workload + (workers.begin() + (worker_index++ % worker_count))->registered.push(reinterpret_cast(_old), &func); + } + + static_cast(prof_mutex.init_always([&]{ samples.clear(); })); + + m_workers.reset(); + + for (u32 i = 0; i < worker_count; i++) + { + (workers.begin() + i)->operator=(thread_state::aborting); + } + } + + spu_llvm& operator=(thread_state) + { + if (const auto workers = m_workers.load()) + { + for (u32 i = 0; i < workers->size(); i++) + { + (workers->begin() + i)->operator=(thread_state::aborting); + } + } + + return *this; + } + + static constexpr auto thread_name = "SPU LLVM"sv; +}; + +using spu_llvm_thread = named_thread; + +struct spu_fast : public spu_recompiler_base +{ + virtual void init() override + { + if (!m_spurt) + { + m_spurt = &g_fxo->get(); + } + } + + virtual spu_function_t compile(spu_program&& _func) override + { + const auto add_loc = m_spurt->add_empty(std::move(_func)); + + if (!add_loc) + { + return nullptr; + } + + if (add_loc->compiled) + { + return add_loc->compiled; + } + + const spu_program& func = add_loc->data; + + if (g_cfg.core.spu_debug && !add_loc->logged.exchange(1)) + { + std::string log; + this->dump(func, log); + fs::write_file(m_spurt->get_cache_path() + "spu.log", fs::create + fs::write + fs::append, log); + } + + // Allocate executable area with necessary size + const auto result = jit_runtime::alloc(22 + 1 + 9 + ::size32(func.data) * (16 + 16) + 36 + 47, 16); + + if (!result) + { + return nullptr; + } + + m_pos = func.lower_bound; + m_size = ::size32(func.data) * 4; + + { + sha1_context ctx; + u8 output[20]; + + sha1_starts(&ctx); + sha1_update(&ctx, reinterpret_cast(func.data.data()), func.data.size() * 4); + sha1_finish(&ctx, output); + + be_t hash_start; + std::memcpy(&hash_start, output, sizeof(hash_start)); + m_hash_start = hash_start; + } + + u8* raw = result; + + // 8-byte intruction for patching (long NOP) + *raw++ = 0x0f; + *raw++ = 0x1f; + *raw++ = 0x84; + *raw++ = 0; + *raw++ = 0; + *raw++ = 0; + *raw++ = 0; + *raw++ = 0; + + // mov rax, m_hash_start + *raw++ = 0x48; + *raw++ = 0xb8; + std::memcpy(raw, &m_hash_start, sizeof(m_hash_start)); + raw += 8; + + // Update block_hash: mov [r13 + spu_thread::m_block_hash], rax + *raw++ = 0x49; + *raw++ = 0x89; + *raw++ = 0x45; + *raw++ = ::narrow(::offset32(&spu_thread::block_hash)); + + // Load PC: mov eax, [r13 + spu_thread::pc] + *raw++ = 0x41; + *raw++ = 0x8b; + *raw++ = 0x45; + *raw++ = ::narrow(::offset32(&spu_thread::pc)); + + // Get LS address starting from PC: lea rcx, [rbp + rax] + *raw++ = 0x48; + *raw++ = 0x8d; + *raw++ = 0x4c; + *raw++ = 0x05; + *raw++ = 0x00; + + // Verification (slow) + for (u32 i = 0; i < func.data.size(); i++) + { + if (!func.data[i]) + { + continue; + } + + // cmp dword ptr [rcx + off], opc + *raw++ = 0x81; + *raw++ = 0xb9; + const u32 off = i * 4; + const u32 opc = func.data[i]; + std::memcpy(raw + 0, &off, 4); + std::memcpy(raw + 4, &opc, 4); + raw += 8; + + // jne tr_dispatch + const s64 rel = reinterpret_cast(spu_runtime::tr_dispatch) - reinterpret_cast(raw) - 6; + *raw++ = 0x0f; + *raw++ = 0x85; + std::memcpy(raw + 0, &rel, 4); + raw += 4; + } + + // trap + //*raw++ = 0xcc; + + // Secondary prologue: sub rsp,0x28 + *raw++ = 0x48; + *raw++ = 0x83; + *raw++ = 0xec; + *raw++ = 0x28; + + // Fix args: xchg r13,rbp + *raw++ = 0x49; + *raw++ = 0x87; + *raw++ = 0xed; + + // mov r12d, eax + *raw++ = 0x41; + *raw++ = 0x89; + *raw++ = 0xc4; + + // mov esi, 0x7f0 + *raw++ = 0xbe; + *raw++ = 0xf0; + *raw++ = 0x07; + *raw++ = 0x00; + *raw++ = 0x00; + + // lea rdi, [rbp + spu_thread::gpr] + *raw++ = 0x48; + *raw++ = 0x8d; + *raw++ = 0x7d; + *raw++ = ::narrow(::offset32(&spu_thread::gpr)); + + // Save base pc: mov [rbp + spu_thread::base_pc], eax + *raw++ = 0x89; + *raw++ = 0x45; + *raw++ = ::narrow(::offset32(&spu_thread::base_pc)); + + // inc block_counter + *raw++ = 0x48; + *raw++ = 0xff; + *raw++ = 0x85; + const u32 blc_off = ::offset32(&spu_thread::block_counter); + std::memcpy(raw, &blc_off, 4); + raw += 4; + + // lea r14, [local epilogue] + *raw++ = 0x4c; + *raw++ = 0x8d; + *raw++ = 0x35; + const u32 epi_off = ::size32(func.data) * 16; + std::memcpy(raw, &epi_off, 4); + raw += 4; + + // Instructions (each instruction occupies fixed number of bytes) + for (u32 i = 0; i < func.data.size(); i++) + { + const u32 pos = m_pos + i * 4; + + if (!func.data[i]) + { + // Save pc: mov [rbp + spu_thread::pc], r12d + *raw++ = 0x44; + *raw++ = 0x89; + *raw++ = 0x65; + *raw++ = ::narrow(::offset32(&spu_thread::pc)); + + // Epilogue: add rsp,0x28 + *raw++ = 0x48; + *raw++ = 0x83; + *raw++ = 0xc4; + *raw++ = 0x28; + + // ret (TODO) + *raw++ = 0xc3; + std::memset(raw, 0xcc, 16 - 9); + raw += 16 - 9; + continue; + } + + // Fix endianness + const spu_opcode_t op{std::bit_cast>(func.data[i])}; + + switch (auto type = g_spu_itype.decode(op.opcode)) + { + case spu_itype::BRZ: + case spu_itype::BRHZ: + case spu_itype::BRNZ: + case spu_itype::BRHNZ: + { + const u32 target = spu_branch_target(pos, op.i16); + + if (0 && target >= m_pos && target < m_pos + m_size) + { + *raw++ = type == spu_itype::BRHZ || type == spu_itype::BRHNZ ? 0x66 : 0x90; + *raw++ = 0x83; + *raw++ = 0xbd; + const u32 off = ::offset32(&spu_thread::gpr, op.rt) + 12; + std::memcpy(raw, &off, 4); + raw += 4; + *raw++ = 0x00; + + *raw++ = 0x0f; + *raw++ = type == spu_itype::BRZ || type == spu_itype::BRHZ ? 0x84 : 0x85; + const u32 dif = (target - (pos + 4)) / 4 * 16 + 2; + std::memcpy(raw, &dif, 4); + raw += 4; + + *raw++ = 0x66; + *raw++ = 0x90; + break; + } + + [[fallthrough]]; + } + default: + { + // Ballast: mov r15d, pos + *raw++ = 0x41; + *raw++ = 0xbf; + std::memcpy(raw, &pos, 4); + raw += 4; + + // mov ebx, opc + *raw++ = 0xbb; + std::memcpy(raw, &op, 4); + raw += 4; + + // call spu_* (specially built interpreter function) + const s64 rel = spu_runtime::g_interpreter_table[static_cast(type)] - reinterpret_cast(raw) - 5; + *raw++ = 0xe8; + std::memcpy(raw, &rel, 4); + raw += 4; + break; + } + } + } + + // Local dispatcher/epilogue: fix stack after branch instruction, then dispatch or return + + // add rsp, 8 + *raw++ = 0x48; + *raw++ = 0x83; + *raw++ = 0xc4; + *raw++ = 0x08; + + // and rsp, -16 + *raw++ = 0x48; + *raw++ = 0x83; + *raw++ = 0xe4; + *raw++ = 0xf0; + + // lea rax, [r12 - size] + *raw++ = 0x49; + *raw++ = 0x8d; + *raw++ = 0x84; + *raw++ = 0x24; + const u32 msz = 0u - m_size; + std::memcpy(raw, &msz, 4); + raw += 4; + + // sub eax, [rbp + spu_thread::base_pc] + *raw++ = 0x2b; + *raw++ = 0x45; + *raw++ = ::narrow(::offset32(&spu_thread::base_pc)); + + // cmp eax, (0 - size) + *raw++ = 0x3d; + std::memcpy(raw, &msz, 4); + raw += 4; + + // jb epilogue + *raw++ = 0x72; + *raw++ = +12; + + // movsxd rax, eax + *raw++ = 0x48; + *raw++ = 0x63; + *raw++ = 0xc0; + + // shl rax, 2 + *raw++ = 0x48; + *raw++ = 0xc1; + *raw++ = 0xe0; + *raw++ = 0x02; + + // add rax, r14 + *raw++ = 0x4c; + *raw++ = 0x01; + *raw++ = 0xf0; + + // jmp rax + *raw++ = 0xff; + *raw++ = 0xe0; + + // Save pc: mov [rbp + spu_thread::pc], r12d + *raw++ = 0x44; + *raw++ = 0x89; + *raw++ = 0x65; + *raw++ = ::narrow(::offset32(&spu_thread::pc)); + + // Epilogue: add rsp,0x28 ; ret + *raw++ = 0x48; + *raw++ = 0x83; + *raw++ = 0xc4; + *raw++ = 0x28; + *raw++ = 0xc3; + + const auto fn = reinterpret_cast(result); + + // Install pointer carefully + const bool added = !add_loc->compiled && add_loc->compiled.compare_and_swap_test(nullptr, fn); + + // Check hash against allowed bounds + const bool inverse_bounds = g_cfg.core.spu_llvm_lower_bound > g_cfg.core.spu_llvm_upper_bound; + + if ((!inverse_bounds && (m_hash_start < g_cfg.core.spu_llvm_lower_bound || m_hash_start > g_cfg.core.spu_llvm_upper_bound)) || + (inverse_bounds && (m_hash_start < g_cfg.core.spu_llvm_lower_bound && m_hash_start > g_cfg.core.spu_llvm_upper_bound))) + { + spu_log.error("[Debug] Skipped function %s", fmt::base57(be_t{m_hash_start})); + } + else if (added) + { + // Send work to LLVM compiler thread + g_fxo->get().registered.push(m_hash_start, add_loc); + } + + // Rebuild trampoline if necessary + if (!m_spurt->rebuild_ubertrampoline(func.data[0])) + { + return nullptr; + } + + if (added) + { + add_loc->compiled.notify_all(); + } + + return fn; + } +}; + +std::unique_ptr spu_recompiler_base::make_fast_llvm_recompiler() +{ + return std::make_unique(); +}