diff --git a/Utilities/Thread.cpp b/Utilities/Thread.cpp index e7011b7a8b..e9e72c3878 100644 --- a/Utilities/Thread.cpp +++ b/Utilities/Thread.cpp @@ -1001,24 +1001,11 @@ namespace rsx bool handle_access_violation(u32 addr, bool is_writing, x64_context* context) { - const auto cpu = get_current_cpu_thread(); - - if (cpu) - { - cpu->state += cpu_flag::is_waiting; - } - g_tls_fault_all++; if (rsx::g_access_violation_handler && rsx::g_access_violation_handler(addr, is_writing)) { g_tls_fault_rsx++; - - if (cpu) - { - cpu->state -= cpu_flag::is_waiting; - } - return true; } @@ -1147,23 +1134,23 @@ bool handle_access_violation(u32 addr, bool is_writing, x64_context* context) } } - if (cpu) - { - cpu->state -= cpu_flag::is_waiting; - } - // skip processed instruction RIP(context) += i_size; g_tls_fault_spu++; return true; } + if (vm::check_addr(addr, d_size)) + { + return true; + } + // TODO: allow recovering from a page fault as a feature of PS3 virtual memory - if (cpu) + if (const auto cpu = get_current_cpu_thread()) { LOG_FATAL(MEMORY, "Access violation %s location 0x%x", is_writing ? "writing" : "reading", addr); cpu->state += cpu_flag::dbg_pause; - cpu->test_state(); + cpu->check_state(); } return true; diff --git a/rpcs3/Emu/CPU/CPUThread.cpp b/rpcs3/Emu/CPU/CPUThread.cpp index 0d029f6537..9fef3dbf4e 100644 --- a/rpcs3/Emu/CPU/CPUThread.cpp +++ b/rpcs3/Emu/CPU/CPUThread.cpp @@ -1,13 +1,12 @@ #include "stdafx.h" #include "Emu/System.h" +#include "Emu/Memory/vm.h" #include "CPUThread.h" -#include - DECLARE(cpu_thread::g_threads_created){0}; DECLARE(cpu_thread::g_threads_deleted){0}; -template<> +template <> void fmt_class_string::format(std::string& out, u64 arg) { format_enum(out, arg, [](cpu_flag f) @@ -19,11 +18,11 @@ void fmt_class_string::format(std::string& out, u64 arg) case cpu_flag::suspend: return "s"; case cpu_flag::ret: return "ret"; case cpu_flag::signal: return "sig"; + case cpu_flag::memory: return "mem"; case cpu_flag::dbg_global_pause: return "G-PAUSE"; case cpu_flag::dbg_global_stop: return "G-EXIT"; case cpu_flag::dbg_pause: return "PAUSE"; case cpu_flag::dbg_step: return "STEP"; - case cpu_flag::is_waiting: return "w"; case cpu_flag::__bitset_enum_max: break; } @@ -66,7 +65,6 @@ void cpu_thread::on_task() } state -= cpu_flag::ret; - state += cpu_flag::is_waiting; continue; } @@ -82,6 +80,7 @@ void cpu_thread::on_stop() cpu_thread::~cpu_thread() { + vm::cleanup_unlock(*this); g_threads_deleted++; } @@ -94,12 +93,23 @@ cpu_thread::cpu_thread(u32 id) bool cpu_thread::check_state() { bool cpu_sleep_called = false; + bool cpu_flag_memory = false; while (true) { + if (test(state, cpu_flag::memory) && state.test_and_reset(cpu_flag::memory)) + { + cpu_flag_memory = true; + + if (auto& ptr = vm::g_tls_locked) + { + ptr->compare_and_swap(this, nullptr); + ptr = nullptr; + } + } + if (test(state, cpu_flag::exit + cpu_flag::dbg_global_stop)) { - state += cpu_flag::is_waiting; return true; } @@ -110,20 +120,10 @@ bool cpu_thread::check_state() if (!test(state, cpu_state_pause)) { - if (test(state, cpu_flag::is_waiting)) - { - state -= cpu_flag::is_waiting; - } - + if (cpu_flag_memory) vm::passive_lock(*this); break; } - - if (!state.test_and_set(cpu_flag::is_waiting)) - { - continue; - } - - if (test(state & cpu_flag::suspend) && !cpu_sleep_called) + else if (!cpu_sleep_called) { cpu_sleep(); cpu_sleep_called = true; @@ -137,7 +137,6 @@ bool cpu_thread::check_state() if (test(state_, cpu_flag::ret + cpu_flag::stop)) { - state += cpu_flag::is_waiting; return true; } @@ -147,7 +146,6 @@ bool cpu_thread::check_state() state -= cpu_flag::dbg_step; } - state -= cpu_flag::is_waiting; return false; } diff --git a/rpcs3/Emu/CPU/CPUThread.h b/rpcs3/Emu/CPU/CPUThread.h index b086776b22..fa69f2d421 100644 --- a/rpcs3/Emu/CPU/CPUThread.h +++ b/rpcs3/Emu/CPU/CPUThread.h @@ -11,14 +11,13 @@ enum class cpu_flag : u32 suspend, // Thread suspended ret, // Callback return requested signal, // Thread received a signal (HLE) + memory, // Thread must unlock memory mutex dbg_global_pause, // Emulation paused dbg_global_stop, // Emulation stopped dbg_pause, // Thread paused dbg_step, // Thread forced to pause after one step (one instruction, etc) - is_waiting, // Informational, self-maintained - __bitset_enum_max }; @@ -38,7 +37,7 @@ public: cpu_thread(u32 id); // Public thread state - atomic_t> state{cpu_flag::stop + cpu_flag::is_waiting}; + atomic_t> state{+cpu_flag::stop}; // Process thread state, return true if the checker must return bool check_state(); diff --git a/rpcs3/Emu/Cell/MFC.cpp b/rpcs3/Emu/Cell/MFC.cpp index c0b1c504f1..20ca75e149 100644 --- a/rpcs3/Emu/Cell/MFC.cpp +++ b/rpcs3/Emu/Cell/MFC.cpp @@ -69,7 +69,7 @@ std::string mfc_thread::get_name() const void mfc_thread::cpu_task() { - state -= cpu_flag::is_waiting; + vm::passive_lock(*this); u32 no_updates = 0; @@ -78,8 +78,6 @@ void mfc_thread::cpu_task() // Add or remove destroyed SPU threads while (m_spuq.size()) { - state += cpu_flag::is_waiting; - auto& thread_ptr = m_spuq[0]; // Look for deleted threads if nullptr received @@ -145,8 +143,7 @@ void mfc_thread::cpu_task() no_updates = 0; // Store unconditionally - state += cpu_flag::is_waiting; - writer_lock lock(vm::g_mutex); + vm::writer_lock lock(0); data = to_write; vm::reservation_update(cmd.eal, 128); vm::notify(cmd.eal, 128); @@ -259,8 +256,6 @@ void mfc_thread::cpu_task() if (no_updates++) { - state += cpu_flag::is_waiting; - if (no_updates >= 3) { if (m_spuq.size()) @@ -298,16 +293,20 @@ void mfc_thread::cpu_task() if (no_updates) { + vm::temporary_unlock(*this); thread_ctrl::wait_for(100); } } else { - reader_lock lock(vm::g_mutex); + vm::reader_lock lock; vm::notify_all(); } } } + + vm::passive_unlock(*this); + state += cpu_flag::stop; } void mfc_thread::add_spu(spu_ptr _spu) diff --git a/rpcs3/Emu/Cell/Modules/cellAudio.cpp b/rpcs3/Emu/Cell/Modules/cellAudio.cpp index 81e9bb4152..d23dbd6186 100644 --- a/rpcs3/Emu/Cell/Modules/cellAudio.cpp +++ b/rpcs3/Emu/Cell/Modules/cellAudio.cpp @@ -16,6 +16,21 @@ logs::channel cellAudio("cellAudio", logs::level::notice); cfg::bool_entry g_cfg_audio_dump_to_file(cfg::root.audio, "Dump to file"); cfg::bool_entry g_cfg_audio_convert_to_u16(cfg::root.audio, "Convert to 16 bit"); +void audio_config::on_init(const std::shared_ptr& _this) +{ + m_buffer.set(vm::alloc(AUDIO_PORT_OFFSET * AUDIO_PORT_COUNT, vm::main)); + m_indexes.set(vm::alloc(sizeof(u64) * AUDIO_PORT_COUNT, vm::main)); + + for (u32 i = 0; i < AUDIO_PORT_COUNT; i++) + { + ports[i].number = i; + ports[i].addr = m_buffer + AUDIO_PORT_OFFSET * i; + ports[i].index = m_indexes + i; + } + + named_thread::on_init(_this); +} + void audio_config::on_task() { AudioDumper m_dump(g_cfg_audio_dump_to_file ? 2 : 0); // Init AudioDumper for 2 channels if enabled diff --git a/rpcs3/Emu/Cell/Modules/cellAudio.h b/rpcs3/Emu/Cell/Modules/cellAudio.h index 9aa41299c5..3a55f47ccf 100644 --- a/rpcs3/Emu/Cell/Modules/cellAudio.h +++ b/rpcs3/Emu/Cell/Modules/cellAudio.h @@ -125,12 +125,14 @@ class audio_config final : public named_thread std::string get_name() const override { return "Audio Thread"; } - vm::var> m_buffer{ AUDIO_PORT_OFFSET * AUDIO_PORT_COUNT }; - vm::var> m_indexes{ AUDIO_PORT_COUNT }; + vm::ptr m_buffer = vm::null; + vm::ptr m_indexes = vm::null; u64 m_counter{}; public: + void on_init(const std::shared_ptr&) override; + const u64 start_time = get_system_time(); std::array ports; @@ -139,17 +141,13 @@ public: semaphore<> mutex; - audio_config() - { - for (u32 i = 0; i < AUDIO_PORT_COUNT; i++) - { - ports[i].number = i; - ports[i].addr = m_buffer + AUDIO_PORT_OFFSET * i; - ports[i].index = m_indexes + i; - } - } + audio_config() = default; - ~audio_config() = default; + ~audio_config() + { + vm::dealloc_verbose_nothrow(m_buffer.addr()); + vm::dealloc_verbose_nothrow(m_indexes.addr()); + } audio_port* open_port() { diff --git a/rpcs3/Emu/Cell/Modules/cellGcmSys.cpp b/rpcs3/Emu/Cell/Modules/cellGcmSys.cpp index 585bb3fb37..01c32f38f3 100644 --- a/rpcs3/Emu/Cell/Modules/cellGcmSys.cpp +++ b/rpcs3/Emu/Cell/Modules/cellGcmSys.cpp @@ -1323,12 +1323,10 @@ s32 cellGcmCallback(ppu_thread& ppu, vm::ptr context, u32 co if (isInCommandBufferExcept(getPos, newCommandBuffer.first, newCommandBuffer.second)) break; - ppu.state += cpu_flag::is_waiting; ppu.test_state(); busy_wait(); } - ppu.test_state(); return CELL_OK; } diff --git a/rpcs3/Emu/Cell/Modules/sys_lwcond_.cpp b/rpcs3/Emu/Cell/Modules/sys_lwcond_.cpp index 5568e04150..8293ed86f8 100644 --- a/rpcs3/Emu/Cell/Modules/sys_lwcond_.cpp +++ b/rpcs3/Emu/Cell/Modules/sys_lwcond_.cpp @@ -58,6 +58,7 @@ s32 sys_lwcond_signal(ppu_thread& ppu, vm::ptr lwcond) // call the syscall if (s32 res = _sys_lwcond_signal(ppu, lwcond->lwcond_queue, lwmutex->sleep_queue, -1, 1)) { + ppu.test_state(); lwmutex->all_info--; return res == CELL_EPERM ? CELL_OK : res; @@ -85,6 +86,7 @@ s32 sys_lwcond_signal(ppu_thread& ppu, vm::ptr lwcond) // call the syscall if (s32 res = _sys_lwcond_signal(ppu, lwcond->lwcond_queue, lwmutex->sleep_queue, -1, 3)) { + ppu.test_state(); lwmutex->all_info--; // unlock the lightweight mutex @@ -119,6 +121,7 @@ s32 sys_lwcond_signal_all(ppu_thread& ppu, vm::ptr lwcond) return res; } + ppu.test_state(); lwmutex->all_info += res; return CELL_OK; @@ -140,6 +143,8 @@ s32 sys_lwcond_signal_all(ppu_thread& ppu, vm::ptr lwcond) // if locking succeeded, call the syscall s32 res = _sys_lwcond_signal_all(ppu, lwcond->lwcond_queue, lwmutex->sleep_queue, 1); + ppu.test_state(); + if (res > 0) { lwmutex->all_info += res; @@ -173,6 +178,7 @@ s32 sys_lwcond_signal_to(ppu_thread& ppu, vm::ptr lwcond, u32 ppu_ // call the syscall if (s32 res = _sys_lwcond_signal(ppu, lwcond->lwcond_queue, lwmutex->sleep_queue, ppu_thread_id, 1)) { + ppu.test_state(); lwmutex->all_info--; return res; @@ -200,6 +206,7 @@ s32 sys_lwcond_signal_to(ppu_thread& ppu, vm::ptr lwcond, u32 ppu_ // call the syscall if (s32 res = _sys_lwcond_signal(ppu, lwcond->lwcond_queue, lwmutex->sleep_queue, ppu_thread_id, 3)) { + ppu.test_state(); lwmutex->all_info--; // unlock the lightweight mutex diff --git a/rpcs3/Emu/Cell/PPUFunction.h b/rpcs3/Emu/Cell/PPUFunction.h index cd7ac19451..193c812081 100644 --- a/rpcs3/Emu/Cell/PPUFunction.h +++ b/rpcs3/Emu/Cell/PPUFunction.h @@ -9,6 +9,7 @@ using ppu_function_t = void(*)(ppu_thread&); const auto old_f = ppu.last_function;\ ppu.last_function = #func;\ ppu_func_detail::do_call(ppu, func);\ + ppu.test_state();\ ppu.last_function = old_f;\ })) diff --git a/rpcs3/Emu/Cell/PPUThread.cpp b/rpcs3/Emu/Cell/PPUThread.cpp index 42888bb481..a1186b8a9e 100644 --- a/rpcs3/Emu/Cell/PPUThread.cpp +++ b/rpcs3/Emu/Cell/PPUThread.cpp @@ -216,6 +216,23 @@ extern void ppu_breakpoint(u32 addr) } } +void ppu_thread::on_init(const std::shared_ptr& _this) +{ + if (!stack_addr) + { + const_cast(stack_addr) = vm::alloc(stack_size, vm::stack); + + if (!stack_addr) + { + fmt::throw_exception("Out of stack memory (size=0x%x)" HERE, stack_size); + } + + gpr[1] = ::align(stack_addr + stack_size, 0x200) - 0x200; + + cpu_thread::on_init(_this); + } +} + std::string ppu_thread::get_name() const { return fmt::format("PPU[0x%x] Thread (%s)", id, m_name); @@ -453,19 +470,12 @@ ppu_thread::ppu_thread(const std::string& name, u32 prio, u32 stack) : cpu_thread(idm::last_id()) , prio(prio) , stack_size(std::max(stack, 0x4000)) - , stack_addr(vm::alloc(stack_size, vm::stack)) + , stack_addr(0) , start_time(get_system_time()) , m_name(name) { - if (!stack_addr) - { - fmt::throw_exception("Out of stack memory (size=0x%x)" HERE, stack_size); - } - - gpr[1] = ::align(stack_addr + stack_size, 0x200) - 0x200; - // Trigger the scheduler - state += cpu_flag::suspend; + state += cpu_flag::suspend + cpu_flag::memory; } void ppu_thread::cmd_push(cmd64 cmd) @@ -710,8 +720,7 @@ extern bool ppu_stwcx(ppu_thread& ppu, u32 addr, u32 reg_value) return false; } - ppu.state += cpu_flag::is_waiting; - writer_lock lock(vm::g_mutex); + vm::writer_lock lock(0); const bool result = ppu.rtime == vm::reservation_acquire(addr, sizeof(u32)) && data.compare_and_swap_test(static_cast(ppu.rdata), reg_value); @@ -722,7 +731,6 @@ extern bool ppu_stwcx(ppu_thread& ppu, u32 addr, u32 reg_value) } ppu.raddr = 0; - ppu.state -= cpu_flag::is_waiting; return result; } @@ -736,8 +744,7 @@ extern bool ppu_stdcx(ppu_thread& ppu, u32 addr, u64 reg_value) return false; } - ppu.state += cpu_flag::is_waiting; - writer_lock lock(vm::g_mutex); + vm::writer_lock lock(0); const bool result = ppu.rtime == vm::reservation_acquire(addr, sizeof(u64)) && data.compare_and_swap_test(ppu.rdata, reg_value); @@ -748,7 +755,6 @@ extern bool ppu_stdcx(ppu_thread& ppu, u32 addr, u64 reg_value) } ppu.raddr = 0; - ppu.state -= cpu_flag::is_waiting; return result; } diff --git a/rpcs3/Emu/Cell/PPUThread.h b/rpcs3/Emu/Cell/PPUThread.h index c6b67339de..fd1494a848 100644 --- a/rpcs3/Emu/Cell/PPUThread.h +++ b/rpcs3/Emu/Cell/PPUThread.h @@ -25,6 +25,7 @@ public: static const u32 id_step = 1; static const u32 id_count = 2048; + virtual void on_init(const std::shared_ptr&) override; virtual std::string get_name() const override; virtual std::string dump() const override; virtual void cpu_task() override; diff --git a/rpcs3/Emu/Cell/RawSPUThread.cpp b/rpcs3/Emu/Cell/RawSPUThread.cpp index a22b04f73b..bae8f297c6 100644 --- a/rpcs3/Emu/Cell/RawSPUThread.cpp +++ b/rpcs3/Emu/Cell/RawSPUThread.cpp @@ -32,7 +32,7 @@ void RawSPUThread::on_init(const std::shared_ptr& _this) const_cast(index) = id; const_cast(offset) = verify(HERE, vm::falloc(RAW_SPU_BASE_ADDR + RAW_SPU_OFFSET * index, 0x40000)); - SPUThread::on_init(_this); + cpu_thread::on_init(_this); } } diff --git a/rpcs3/Emu/Cell/SPUThread.cpp b/rpcs3/Emu/Cell/SPUThread.cpp index 50774fa886..513c1131d2 100644 --- a/rpcs3/Emu/Cell/SPUThread.cpp +++ b/rpcs3/Emu/Cell/SPUThread.cpp @@ -20,8 +20,6 @@ #include #include -#include -#include #ifdef _MSC_VER bool operator ==(const u128& lhs, const u128& rhs) @@ -134,6 +132,16 @@ spu_imm_table_t::spu_imm_table_t() } } +void SPUThread::on_init(const std::shared_ptr& _this) +{ + if (!offset) + { + const_cast(offset) = verify("SPU LS" HERE, vm::alloc(0x40000, vm::main)); + + cpu_thread::on_init(_this); + } +} + std::string SPUThread::get_name() const { return fmt::format("%sSPU[0x%x] Thread (%s)", offset >= RAW_SPU_BASE_ADDR ? "Raw" : "", id, m_name); @@ -259,7 +267,7 @@ SPUThread::SPUThread(const std::string& name, u32 index, lv2_spu_group* group) : cpu_thread(idm::last_id()) , m_name(name) , index(index) - , offset(verify("SPU LS" HERE, vm::alloc(0x40000, vm::main))) + , offset(0) , group(group) { } @@ -516,7 +524,7 @@ void SPUThread::process_mfc_cmd() if (is_polling || UNLIKELY(vm::reservation_acquire(raddr, 128) != rtime)) { // TODO: vm::check_addr - reader_lock lock(vm::g_mutex); + vm::reader_lock lock; rtime = vm::reservation_acquire(raddr, 128); rdata = data; } @@ -537,9 +545,8 @@ void SPUThread::process_mfc_cmd() if (raddr == ch_mfc_cmd.eal && rtime == vm::reservation_acquire(raddr, 128) && rdata == data) { - lv2_obj::lock_all(); - // TODO: vm::check_addr + vm::writer_lock lock; if (rtime == vm::reservation_acquire(raddr, 128) && rdata == data) { @@ -549,8 +556,6 @@ void SPUThread::process_mfc_cmd() vm::reservation_update(raddr, 128); vm::notify(raddr, 128); } - - lv2_obj::unlock_all(); } if (result) @@ -583,7 +588,7 @@ void SPUThread::process_mfc_cmd() // Store unconditionally // TODO: vm::check_addr - writer_lock lock(vm::g_mutex); + vm::writer_lock lock(0); data = to_write; vm::reservation_update(ch_mfc_cmd.eal, 128); vm::notify(ch_mfc_cmd.eal, 128); @@ -616,7 +621,7 @@ void SPUThread::process_mfc_cmd() // Try to process small transfers immediately if (ch_mfc_cmd.size <= 256 && mfc_queue.size() == 0) { - std::shared_lock lock(vm::g_mutex, std::try_to_lock); + vm::reader_lock lock(vm::try_to_lock); if (!lock) { @@ -647,7 +652,7 @@ void SPUThread::process_mfc_cmd() { if (ch_mfc_cmd.size <= 16 * 8 && mfc_queue.size() == 0 && (ch_stall_mask & (1u << ch_mfc_cmd.tag)) == 0) { - std::shared_lock lock(vm::g_mutex, std::try_to_lock); + vm::reader_lock lock(vm::try_to_lock); if (!lock) { @@ -732,7 +737,7 @@ void SPUThread::process_mfc_cmd() // Enqueue verify(HERE), mfc_queue.try_push(ch_mfc_cmd); - if (test(mfc->state, cpu_flag::is_waiting)) + //if (test(mfc->state, cpu_flag::is_waiting)) { mfc->notify(); } @@ -1191,7 +1196,7 @@ bool SPUThread::set_ch_value(u32 ch, u32 value) { auto mfc = fxm::check_unlocked(); - if (test(mfc->state, cpu_flag::is_waiting)) + //if (test(mfc->state, cpu_flag::is_waiting)) { mfc->notify(); } @@ -1245,7 +1250,7 @@ bool SPUThread::set_ch_value(u32 ch, u32 value) { auto mfc = fxm::check_unlocked(); - if (test(mfc->state, cpu_flag::is_waiting)) + //if (test(mfc->state, cpu_flag::is_waiting)) { mfc->notify(); } diff --git a/rpcs3/Emu/Cell/SPUThread.h b/rpcs3/Emu/Cell/SPUThread.h index 7d94bccada..b3a53f3c3f 100644 --- a/rpcs3/Emu/Cell/SPUThread.h +++ b/rpcs3/Emu/Cell/SPUThread.h @@ -504,6 +504,7 @@ public: class SPUThread : public cpu_thread { public: + virtual void on_init(const std::shared_ptr&) override; virtual std::string get_name() const override; virtual std::string dump() const override; virtual void cpu_task() override; diff --git a/rpcs3/Emu/Cell/lv2/lv2.cpp b/rpcs3/Emu/Cell/lv2/lv2.cpp index 8d702dc3cc..911190dc95 100644 --- a/rpcs3/Emu/Cell/lv2/lv2.cpp +++ b/rpcs3/Emu/Cell/lv2/lv2.cpp @@ -1047,7 +1047,6 @@ void lv2_obj::sleep_timeout(named_thread& thread, u64 timeout) if (!test(val, cpu_flag::signal)) { val += cpu_flag::suspend; - val += cpu_flag::is_waiting; } }); @@ -1171,84 +1170,6 @@ void lv2_obj::awake(cpu_thread& cpu, u32 prio) schedule_all(); } -void lv2_obj::lock_all() -{ - std::size_t count = 0; - std::array array; - - { - semaphore_lock lock(g_mutex); - - if (g_pending.empty() || g_pending.front()) - { - if (auto mfc = fxm::check_unlocked()) - { - if (!mfc->state.test_and_set(cpu_flag::suspend)) - { - array.at(count++) = mfc; - } - } - } - - for (std::size_t i = 0, x = g_ppu.size(); i < x; i++) - { - const auto target = g_ppu[i]; - - if (!target->state.test_and_set(cpu_flag::suspend)) - { - g_pending.emplace_back(target); - } - } - - for (cpu_thread* target : g_pending) - { - if (target && !test(target->state, cpu_flag::is_waiting)) - { - array.at(count++) = target; - } - } - - g_pending.emplace_front(nullptr); - } - - vm::g_mutex.lock(); - - for (std::size_t i = 0; i < count; i++) - { - while (!test(array[i]->state, cpu_flag::is_waiting)) - { - busy_wait(); - } - } -} - -void lv2_obj::unlock_all() -{ - vm::g_mutex.unlock(); - - semaphore_lock lock(g_mutex); - - if (!g_pending.empty() && !g_pending.front()) - { - g_pending.pop_front(); - - if (g_pending.empty() || g_pending.front()) - { - if (auto mfc = fxm::check_unlocked()) - { - const auto old_state = mfc->state.fetch_sub(cpu_flag::suspend); - - if (!test(old_state, cpu_flag::stop) && test(old_state, cpu_flag::is_waiting)) - { - mfc->notify(); - } - } - } - } - - schedule_all(); -} - void lv2_obj::cleanup() { g_ppu.clear(); @@ -1295,13 +1216,10 @@ void lv2_obj::schedule_all() break; } } - - // Check memory - //reader_lock lock(vm::g_mutex); - //vm::notify(0, -1); } void ppu_thread::cpu_sleep() { + vm::temporary_unlock(*this); lv2_obj::awake(*this); } diff --git a/rpcs3/Emu/Cell/lv2/sys_cond.cpp b/rpcs3/Emu/Cell/lv2/sys_cond.cpp index c13269dd88..d52c6357c4 100644 --- a/rpcs3/Emu/Cell/lv2/sys_cond.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_cond.cpp @@ -99,9 +99,7 @@ error_code sys_cond_signal(ppu_thread& ppu, u32 cond_id) if (cond.ret) { - ppu.state += cpu_flag::is_waiting; cond->awake(*cond.ret); - ppu.test_state(); } return CELL_OK; @@ -140,9 +138,7 @@ error_code sys_cond_signal_all(ppu_thread& ppu, u32 cond_id) if (cond.ret) { - ppu.state += cpu_flag::is_waiting; cond->awake(*cond.ret); - ppu.test_state(); } return CELL_OK; @@ -184,9 +180,7 @@ error_code sys_cond_signal_to(ppu_thread& ppu, u32 cond_id, u32 thread_id) if (cond.ret && cond.ret != (cpu_thread*)(1)) { - ppu.state += cpu_flag::is_waiting; cond->awake(*cond.ret); - ppu.test_state(); } else if (!cond.ret) { @@ -283,6 +277,5 @@ error_code sys_cond_wait(ppu_thread& ppu, u32 cond_id, u64 timeout) // Restore the recursive value cond->mutex->lock_count = cond.ret; - ppu.test_state(); return not_an_error(ppu.gpr[3]); } diff --git a/rpcs3/Emu/Cell/lv2/sys_event.cpp b/rpcs3/Emu/Cell/lv2/sys_event.cpp index c58d19b815..ce90ac7118 100644 --- a/rpcs3/Emu/Cell/lv2/sys_event.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_event.cpp @@ -172,7 +172,6 @@ error_code sys_event_queue_destroy(ppu_thread& ppu, u32 equeue_id, s32 mode) if (queue->type == SYS_PPU_QUEUE) { static_cast(*cpu).gpr[3] = CELL_ECANCELED; - ppu.state += cpu_flag::is_waiting; queue->awake(*cpu); } else @@ -184,7 +183,6 @@ error_code sys_event_queue_destroy(ppu_thread& ppu, u32 equeue_id, s32 mode) } } - ppu.test_state(); return CELL_OK; } @@ -295,7 +293,6 @@ error_code sys_event_queue_receive(ppu_thread& ppu, u32 equeue_id, vm::ptr(eport_id, [&](lv2_event_port& port) -> CellError { if (const auto queue = port.queue.lock()) @@ -455,6 +450,5 @@ error_code sys_event_port_send(ppu_thread& ppu, u32 eport_id, u64 data1, u64 dat return port.ret; } - ppu.test_state(); return CELL_OK; } diff --git a/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp b/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp index fad907f410..c9c57c21f5 100644 --- a/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp @@ -221,13 +221,6 @@ error_code sys_event_flag_trywait(u32 id, u64 bitptn, u32 mode, vm::ptr res error_code sys_event_flag_set(u32 id, u64 bitptn) { // Warning: may be called from SPU thread. - auto cpu = get_current_cpu_thread(); - - if (cpu && cpu->id_type() != 1) - { - cpu = nullptr; - } - sys_event_flag.trace("sys_event_flag_set(id=0x%x, bitptn=0x%llx)", id, bitptn); const auto flag = idm::get(id); @@ -282,10 +275,6 @@ error_code sys_event_flag_set(u32 id, u64 bitptn) { return CELL_OK; } - else if (cpu) - { - cpu->state += cpu_flag::is_waiting; - } // Remove waiters const auto tail = std::remove_if(flag->sq.begin(), flag->sq.end(), [&](cpu_thread* cpu) @@ -305,7 +294,6 @@ error_code sys_event_flag_set(u32 id, u64 bitptn) flag->sq.erase(tail, flag->sq.end()); } - if (cpu) cpu->test_state(); return CELL_OK; } @@ -352,8 +340,6 @@ error_code sys_event_flag_cancel(ppu_thread& ppu, u32 id, vm::ptr num) // Signal all threads to return CELL_ECANCELED while (auto thread = flag->schedule(flag->sq, flag->protocol)) { - ppu.state += cpu_flag::is_waiting; - auto& ppu = static_cast(*thread); ppu.gpr[3] = CELL_ECANCELED; diff --git a/rpcs3/Emu/Cell/lv2/sys_interrupt.cpp b/rpcs3/Emu/Cell/lv2/sys_interrupt.cpp index c8278e93bf..54305c8f55 100644 --- a/rpcs3/Emu/Cell/lv2/sys_interrupt.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_interrupt.cpp @@ -144,6 +144,8 @@ error_code _sys_interrupt_thread_disestablish(ppu_thread& ppu, u32 ih, vm::ptrawake(*cond.ret); - ppu.test_state(); } else if (mode == 2) { @@ -212,15 +210,9 @@ error_code _sys_lwcond_signal_all(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id for (auto cpu : threads) { - ppu.state += cpu_flag::is_waiting; cond->awake(*cpu); } - if (threads.size()) - { - ppu.test_state(); - } - if (mode == 1) { // Mode 1: return the amount of threads (TODO) @@ -269,7 +261,6 @@ error_code _sys_lwcond_queue_wait(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id if (cond.ret) { - ppu.state += cpu_flag::is_waiting; cond->awake(*cond.ret); } @@ -312,6 +303,5 @@ error_code _sys_lwcond_queue_wait(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id } // Return cause - ppu.test_state(); return not_an_error(ppu.gpr[3]); } diff --git a/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp b/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp index 2c51ead3e7..e88647e14b 100644 --- a/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp @@ -138,7 +138,6 @@ error_code _sys_lwmutex_lock(ppu_thread& ppu, u32 lwmutex_id, u64 timeout) } } - ppu.test_state(); return not_an_error(ppu.gpr[3]); } @@ -196,9 +195,7 @@ error_code _sys_lwmutex_unlock(ppu_thread& ppu, u32 lwmutex_id) if (mutex.ret) { - ppu.state += cpu_flag::is_waiting; mutex->awake(*mutex.ret); - ppu.test_state(); } return CELL_OK; diff --git a/rpcs3/Emu/Cell/lv2/sys_mutex.cpp b/rpcs3/Emu/Cell/lv2/sys_mutex.cpp index 0912fdb7af..d5d5990233 100644 --- a/rpcs3/Emu/Cell/lv2/sys_mutex.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_mutex.cpp @@ -170,7 +170,6 @@ error_code sys_mutex_lock(ppu_thread& ppu, u32 mutex_id, u64 timeout) } } - ppu.test_state(); return not_an_error(ppu.gpr[3]); } @@ -221,7 +220,6 @@ error_code sys_mutex_unlock(ppu_thread& ppu, u32 mutex_id) if (auto cpu = mutex->reown()) { - ppu.state += cpu_flag::is_waiting; mutex->awake(*cpu); } } @@ -230,6 +228,5 @@ error_code sys_mutex_unlock(ppu_thread& ppu, u32 mutex_id) return mutex.ret; } - ppu.test_state(); return CELL_OK; } diff --git a/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp b/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp index 1c61ec1579..79c7cbd01d 100644 --- a/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp @@ -13,10 +13,11 @@ logs::channel sys_ppu_thread("sys_ppu_thread", logs::level::notice); void _sys_ppu_thread_exit(ppu_thread& ppu, u64 errorcode) { + vm::temporary_unlock(ppu); + sys_ppu_thread.trace("_sys_ppu_thread_exit(errorcode=0x%llx)", errorcode); ppu.state += cpu_flag::exit; - ppu.state += cpu_flag::is_waiting; // Get joiner ID const u32 jid = ppu.joiner.fetch_op([](u32& value) @@ -65,13 +66,13 @@ void sys_ppu_thread_yield(ppu_thread& ppu) { sys_ppu_thread.trace("sys_ppu_thread_yield()"); - ppu.state += cpu_flag::is_waiting; lv2_obj::awake(ppu, -4); - ppu.test_state(); } error_code sys_ppu_thread_join(ppu_thread& ppu, u32 thread_id, vm::ptr vptr) { + vm::temporary_unlock(ppu); + sys_ppu_thread.trace("sys_ppu_thread_join(thread_id=0x%x, vptr=*0x%x)", thread_id, vptr); const auto thread = idm::get(thread_id, [&](ppu_thread& thread) -> CellError @@ -134,8 +135,6 @@ error_code sys_ppu_thread_join(ppu_thread& ppu, u32 thread_id, vm::ptr vptr // Cleanup idm::remove(thread->id); - - ppu.test_state(); return CELL_OK; } @@ -211,7 +210,6 @@ error_code sys_ppu_thread_set_priority(ppu_thread& ppu, u32 thread_id, s32 prio) { if (thread.prio != prio && thread.prio.exchange(prio) != prio) { - ppu.state += cpu_flag::is_waiting; lv2_obj::awake(thread, prio); } }); @@ -221,7 +219,6 @@ error_code sys_ppu_thread_set_priority(ppu_thread& ppu, u32 thread_id, s32 prio) return CELL_ESRCH; } - ppu.test_state(); return CELL_OK; } @@ -339,7 +336,6 @@ error_code sys_ppu_thread_start(ppu_thread& ppu, u32 thread_id) const auto thread = idm::get(thread_id, [&](ppu_thread& thread) { - ppu.state += cpu_flag::is_waiting; lv2_obj::awake(thread, -2); }); @@ -358,7 +354,6 @@ error_code sys_ppu_thread_start(ppu_thread& ppu, u32 thread_id) thread->notify(); } - ppu.test_state(); return CELL_OK; } diff --git a/rpcs3/Emu/Cell/lv2/sys_process.cpp b/rpcs3/Emu/Cell/lv2/sys_process.cpp index 812a969693..a1ba49f54e 100644 --- a/rpcs3/Emu/Cell/lv2/sys_process.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_process.cpp @@ -46,8 +46,10 @@ s32 sys_process_getppid() return 0; } -s32 sys_process_exit(s32 status) +s32 sys_process_exit(ppu_thread& ppu, s32 status) { + vm::temporary_unlock(ppu); + sys_process.warning("sys_process_exit(status=0x%x)", status); Emu.CallAfter([]() diff --git a/rpcs3/Emu/Cell/lv2/sys_process.h b/rpcs3/Emu/Cell/lv2/sys_process.h index be4951f978..b403482fce 100644 --- a/rpcs3/Emu/Cell/lv2/sys_process.h +++ b/rpcs3/Emu/Cell/lv2/sys_process.h @@ -37,7 +37,7 @@ s32 _sys_process_get_paramsfo(vm::ps3::ptr buffer); s32 sys_process_get_sdk_version(u32 pid, vm::ps3::ptr version); s32 sys_process_get_status(u64 unk); s32 sys_process_is_spu_lock_line_reservation_address(u32 addr, u64 flags); -s32 sys_process_exit(s32 errorcode); +s32 sys_process_exit(ppu_thread& ppu, s32 errorcode); s32 sys_process_kill(u32 pid); s32 sys_process_wait_for_child(u32 pid, vm::ps3::ptr status, u64 unk); s32 sys_process_wait_for_child2(u64 unk1, u64 unk2, u64 unk3, u64 unk4, u64 unk5, u64 unk6); diff --git a/rpcs3/Emu/Cell/lv2/sys_rwlock.cpp b/rpcs3/Emu/Cell/lv2/sys_rwlock.cpp index 0581aa3bec..226c9180a9 100644 --- a/rpcs3/Emu/Cell/lv2/sys_rwlock.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_rwlock.cpp @@ -155,7 +155,6 @@ error_code sys_rwlock_rlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) } } - ppu.test_state(); return not_an_error(ppu.gpr[3]); } @@ -241,8 +240,6 @@ error_code sys_rwlock_runlock(ppu_thread& ppu, u32 rw_lock_id) { if (const auto cpu = rwlock->schedule(rwlock->wq, rwlock->protocol)) { - ppu.state += cpu_flag::is_waiting; - rwlock->owner = cpu->id << 1 | !rwlock->wq.empty(); rwlock->awake(*cpu); @@ -256,7 +253,6 @@ error_code sys_rwlock_runlock(ppu_thread& ppu, u32 rw_lock_id) } } - ppu.test_state(); return CELL_OK; } @@ -344,8 +340,6 @@ error_code sys_rwlock_wlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) while (auto cpu = rwlock->schedule(rwlock->rq, SYS_SYNC_PRIORITY)) { - ppu.state += cpu_flag::is_waiting; - rwlock->awake(*cpu); } @@ -364,7 +358,6 @@ error_code sys_rwlock_wlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) } } - ppu.test_state(); return not_an_error(ppu.gpr[3]); } @@ -426,8 +419,6 @@ error_code sys_rwlock_wunlock(ppu_thread& ppu, u32 rw_lock_id) if (auto cpu = rwlock->schedule(rwlock->wq, rwlock->protocol)) { - ppu.state += cpu_flag::is_waiting; - rwlock->owner = cpu->id << 1 | !rwlock->wq.empty(); rwlock->awake(*cpu); @@ -438,7 +429,6 @@ error_code sys_rwlock_wunlock(ppu_thread& ppu, u32 rw_lock_id) while (auto cpu = rwlock->schedule(rwlock->rq, SYS_SYNC_PRIORITY)) { - ppu.state += cpu_flag::is_waiting; rwlock->awake(*cpu); } @@ -450,10 +440,5 @@ error_code sys_rwlock_wunlock(ppu_thread& ppu, u32 rw_lock_id) } } - if (rwlock.ret & 1) - { - ppu.test_state(); - } - return CELL_OK; } diff --git a/rpcs3/Emu/Cell/lv2/sys_semaphore.cpp b/rpcs3/Emu/Cell/lv2/sys_semaphore.cpp index a9a767c5c9..22c609432a 100644 --- a/rpcs3/Emu/Cell/lv2/sys_semaphore.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_semaphore.cpp @@ -158,7 +158,6 @@ error_code sys_semaphore_wait(ppu_thread& ppu, u32 sem_id, u64 timeout) } } - ppu.test_state(); return not_an_error(ppu.gpr[3]); } @@ -247,14 +246,10 @@ error_code sys_semaphore_post(ppu_thread& ppu, u32 sem_id, s32 count) // Wake threads for (s32 i = std::min(-std::min(val, 0), count); i > 0; i--) { - const auto cpu = verify(HERE, sem->schedule(sem->sq, sem->protocol)); - - ppu.state += cpu_flag::is_waiting; - sem->awake(*cpu); + sem->awake(*verify(HERE, sem->schedule(sem->sq, sem->protocol))); } } - ppu.test_state(); return CELL_OK; } diff --git a/rpcs3/Emu/Cell/lv2/sys_spu.cpp b/rpcs3/Emu/Cell/lv2/sys_spu.cpp index fb879745f0..51cfb923f2 100644 --- a/rpcs3/Emu/Cell/lv2/sys_spu.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_spu.cpp @@ -233,7 +233,7 @@ error_code sys_spu_thread_group_destroy(u32 id) error_code sys_spu_thread_group_start(ppu_thread& ppu, u32 id) { - ppu.state += cpu_flag::is_waiting; + vm::temporary_unlock(ppu); sys_spu.warning("sys_spu_thread_group_start(id=0x%x)", id); @@ -291,7 +291,6 @@ error_code sys_spu_thread_group_start(ppu_thread& ppu, u32 id) } } - ppu.test_state(); return CELL_OK; } @@ -480,6 +479,8 @@ error_code sys_spu_thread_group_terminate(u32 id, s32 value) error_code sys_spu_thread_group_join(ppu_thread& ppu, u32 id, vm::ptr cause, vm::ptr status) { + vm::temporary_unlock(ppu); + sys_spu.warning("sys_spu_thread_group_join(id=0x%x, cause=*0x%x, status=*0x%x)", id, cause, status); const auto group = idm::get(id); @@ -489,48 +490,60 @@ error_code sys_spu_thread_group_join(ppu_thread& ppu, u32 id, vm::ptr cause return CELL_ESRCH; } - semaphore_lock lock(group->mutex); + u32 join_state = 0; + s32 exit_value = 0; - if (group->run_state < SPU_THREAD_GROUP_STATUS_INITIALIZED) { - return CELL_ESTAT; - } + semaphore_lock lock(group->mutex); - if (group->join_state.fetch_or(SPU_TGJSF_IS_JOINING) & SPU_TGJSF_IS_JOINING) - { - // another PPU thread is joining this thread group - return CELL_EBUSY; - } - - lv2_obj::sleep(ppu); - - while ((group->join_state & ~SPU_TGJSF_IS_JOINING) == 0) - { - bool stopped = true; - - for (auto& t : group->threads) + if (group->run_state < SPU_THREAD_GROUP_STATUS_INITIALIZED) { - if (t) + return CELL_ESTAT; + } + + if (group->join_state.fetch_or(SPU_TGJSF_IS_JOINING) & SPU_TGJSF_IS_JOINING) + { + // another PPU thread is joining this thread group + return CELL_EBUSY; + } + + lv2_obj::sleep(ppu); + + while ((group->join_state & ~SPU_TGJSF_IS_JOINING) == 0) + { + bool stopped = true; + + for (auto& t : group->threads) { - if ((t->status & SPU_STATUS_STOPPED_BY_STOP) == 0) + if (t) { - stopped = false; - break; + if ((t->status & SPU_STATUS_STOPPED_BY_STOP) == 0) + { + stopped = false; + break; + } } } + + if (stopped) + { + break; + } + + // TODO + group->cv.wait(lock, 1000); + thread_ctrl::test(); } - if (stopped) - { - break; - } - - // TODO - group->cv.wait(lock, 1000); - thread_ctrl::test(); + join_state = group->join_state; + exit_value = group->exit_status; + group->join_state &= ~SPU_TGJSF_IS_JOINING; + group->run_state = SPU_THREAD_GROUP_STATUS_INITIALIZED; // hack } + + ppu.test_state(); - switch (group->join_state & ~SPU_TGJSF_IS_JOINING) + switch (join_state & ~SPU_TGJSF_IS_JOINING) { case 0: { @@ -557,9 +570,7 @@ error_code sys_spu_thread_group_join(ppu_thread& ppu, u32 id, vm::ptr cause { *status = group->exit_status; } - - group->join_state &= ~SPU_TGJSF_IS_JOINING; - group->run_state = SPU_THREAD_GROUP_STATUS_INITIALIZED; // hack + return CELL_OK; } diff --git a/rpcs3/Emu/Cell/lv2/sys_sync.h b/rpcs3/Emu/Cell/lv2/sys_sync.h index 8dfc2ad827..0943706b2b 100644 --- a/rpcs3/Emu/Cell/lv2/sys_sync.h +++ b/rpcs3/Emu/Cell/lv2/sys_sync.h @@ -4,6 +4,7 @@ #include "Utilities/sema.h" #include "Utilities/cond.h" +#include "Emu/Memory/vm.h" #include "Emu/CPU/CPUThread.h" #include "Emu/Cell/ErrorCodes.h" @@ -111,7 +112,7 @@ struct lv2_obj static void sleep(cpu_thread& thread, u64 timeout = 0) { - thread.state += cpu_flag::is_waiting; + vm::temporary_unlock(thread); sleep_timeout(thread, timeout); } @@ -123,8 +124,6 @@ struct lv2_obj awake(thread, -1); } - static void lock_all(); - static void unlock_all(); static void cleanup(); private: diff --git a/rpcs3/Emu/Cell/lv2/sys_timer.cpp b/rpcs3/Emu/Cell/lv2/sys_timer.cpp index a8db729a4b..64602334ef 100644 --- a/rpcs3/Emu/Cell/lv2/sys_timer.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_timer.cpp @@ -281,6 +281,8 @@ error_code sys_timer_disconnect_event_queue(u32 timer_id) error_code sys_timer_sleep(ppu_thread& ppu, u32 sleep_time) { + vm::temporary_unlock(ppu); + sys_timer.trace("sys_timer_sleep(sleep_time=%d) -> sys_timer_usleep()", sleep_time); return sys_timer_usleep(ppu, sleep_time * u64{1000000}); @@ -288,6 +290,8 @@ error_code sys_timer_sleep(ppu_thread& ppu, u32 sleep_time) error_code sys_timer_usleep(ppu_thread& ppu, u64 sleep_time) { + vm::temporary_unlock(ppu); + sys_timer.trace("sys_timer_usleep(sleep_time=0x%llx)", sleep_time); u64 passed = 0; diff --git a/rpcs3/Emu/Memory/vm.cpp b/rpcs3/Emu/Memory/vm.cpp index 7d86261063..b0cf8aec88 100644 --- a/rpcs3/Emu/Memory/vm.cpp +++ b/rpcs3/Emu/Memory/vm.cpp @@ -38,9 +38,162 @@ namespace vm // Registered waiters std::deque g_waiters; - // Memory mutex + // Memory mutex core shared_mutex g_mutex; + // Memory mutex acknowledgement + thread_local atomic_t* g_tls_locked = nullptr; + + // Memory mutex: passive locks + std::array, 32> g_locks; + + static void _register_lock(cpu_thread* _cpu) + { + for (u32 i = 0;; i = (i + 1) % g_locks.size()) + { + if (!g_locks[i] && g_locks[i].compare_and_swap_test(nullptr, _cpu)) + { + g_tls_locked = g_locks.data() + i; + return; + } + } + } + + void passive_lock(cpu_thread& cpu) + { + if (g_tls_locked && *g_tls_locked == &cpu) + { + return; + } + + ::reader_lock lock(g_mutex); + + _register_lock(&cpu); + } + + void passive_unlock(cpu_thread& cpu) + { + if (g_tls_locked) + { + g_tls_locked->compare_and_swap_test(&cpu, nullptr); + ::reader_lock lock(g_mutex); + g_tls_locked = nullptr; + } + } + + void cleanup_unlock(cpu_thread& cpu) noexcept + { + if (g_tls_locked && cpu.get() == thread_ctrl::get_current()) + { + g_tls_locked = nullptr; + } + + for (u32 i = 0; i < g_locks.size(); i++) + { + if (g_locks[i] == &cpu) + { + g_locks[i].compare_and_swap_test(&cpu, nullptr); + return; + } + } + } + + void temporary_unlock(cpu_thread& cpu) noexcept + { + if (g_tls_locked && g_tls_locked->compare_and_swap_test(&cpu, nullptr)) + { + cpu.state.test_and_set(cpu_flag::memory); + } + } + + reader_lock::reader_lock() + : locked(true) + { + auto cpu = get_current_cpu_thread(); + + if (!cpu || !g_tls_locked || !g_tls_locked->compare_and_swap_test(cpu, nullptr)) + { + cpu = nullptr; + } + + g_mutex.lock_shared(); + + if (cpu) + { + _register_lock(cpu); + cpu->state -= cpu_flag::memory; + } + } + + reader_lock::reader_lock(const try_to_lock_t&) + : locked(g_mutex.try_lock_shared()) + { + } + + reader_lock::~reader_lock() + { + if (locked) + { + g_mutex.unlock_shared(); + } + } + + writer_lock::writer_lock(int full) + : locked(true) + { + auto cpu = get_current_cpu_thread(); + + if (!cpu || !g_tls_locked || !g_tls_locked->compare_and_swap_test(cpu, nullptr)) + { + cpu = nullptr; + } + + g_mutex.lock(); + + if (full) + { + for (auto& lock : g_locks) + { + if (cpu_thread* ptr = lock) + { + ptr->state.test_and_set(cpu_flag::memory); + } + } + + for (auto& lock : g_locks) + { + while (cpu_thread* ptr = lock) + { + if (test(ptr->state, cpu_flag::dbg_global_stop + cpu_flag::exit)) + { + break; + } + + busy_wait(); + } + } + } + + if (cpu) + { + _register_lock(cpu); + cpu->state -= cpu_flag::memory; + } + } + + writer_lock::writer_lock(const try_to_lock_t&) + : locked(g_mutex.try_lock()) + { + } + + writer_lock::~writer_lock() + { + if (locked) + { + g_mutex.unlock(); + } + } + // Page information struct memory_page { @@ -73,39 +226,6 @@ namespace vm } }; - template - struct mem_lock - { - cpu_thread* thread; - T lock; - - template - mem_lock(X&& mtx) - : thread(find_thread()) - , lock(std::forward(mtx)) - { - } - - ~mem_lock() - { - if (thread) - { - thread->state -= cpu_flag::is_waiting; - } - } - - static cpu_thread* find_thread() - { - if (auto cpu = get_current_cpu_thread()) - { - cpu->state += cpu_flag::is_waiting; - return cpu; - } - - return nullptr; - } - }; - // Memory pages std::array g_pages{}; @@ -124,7 +244,7 @@ namespace vm void waiter::init() { // Register waiter - writer_lock lock(g_mutex); + writer_lock lock(0); g_waiters.emplace_back(this); } @@ -156,18 +276,15 @@ namespace vm waiter::~waiter() { - if (owner) + // Unregister waiter + writer_lock lock(0); + + // Find waiter + const auto found = std::find(g_waiters.cbegin(), g_waiters.cend(), this); + + if (found != g_waiters.cend()) { - // Unregister waiter - writer_lock lock(g_mutex); - - // Find waiter - const auto found = std::find(g_waiters.cbegin(), g_waiters.cend(), this); - - if (found != g_waiters.cend()) - { - g_waiters.erase(found); - } + g_waiters.erase(found); } } @@ -209,14 +326,11 @@ namespace vm #ifdef _WIN32 auto protection = flags & page_writable ? PAGE_READWRITE : (flags & page_readable ? PAGE_READONLY : PAGE_NOACCESS); - if (!::VirtualAlloc(real_addr, size, MEM_COMMIT, protection)) + verify(__func__), ::VirtualAlloc(real_addr, size, MEM_COMMIT, protection); #else auto protection = flags & page_writable ? PROT_WRITE | PROT_READ : (flags & page_readable ? PROT_READ : PROT_NONE); - if (::mprotect(real_addr, size, protection)) + verify(__func__), !::mprotect(real_addr, size, protection), !::madvise(real_addr, size, MADV_WILLNEED); #endif - { - fmt::throw_exception("System failure (addr=0x%x, size=0x%x, flags=0x%x)" HERE, addr, size, flags); - } for (u32 i = addr / 4096; i < addr / 4096 + size / 4096; i++) { @@ -229,7 +343,7 @@ namespace vm bool page_protect(u32 addr, u32 size, u8 flags_test, u8 flags_set, u8 flags_clear) { - mem_lock lock(g_mutex); + writer_lock lock(0); if (!size || (size | addr) % 4096) { @@ -277,14 +391,11 @@ namespace vm DWORD old; auto protection = start_value & page_writable ? PAGE_READWRITE : (start_value & page_readable ? PAGE_READONLY : PAGE_NOACCESS); - if (!::VirtualProtect(vm::base(start * 4096), page_size, protection, &old)) + verify(__func__), ::VirtualProtect(vm::base(start * 4096), page_size, protection, &old); #else auto protection = start_value & page_writable ? PROT_WRITE | PROT_READ : (start_value & page_readable ? PROT_READ : PROT_NONE); - if (::mprotect(vm::base(start * 4096), page_size, protection)) + verify(__func__), !::mprotect(vm::base(start * 4096), page_size, protection); #endif - { - fmt::throw_exception("System failure (addr=0x%x, size=0x%x, flags_test=0x%x, flags_set=0x%x, flags_clear=0x%x)" HERE, addr, size, flags_test, flags_set, flags_clear); - } } start_value = new_val; @@ -321,13 +432,10 @@ namespace vm void* real_addr = vm::base(addr); #ifdef _WIN32 - if (!::VirtualFree(real_addr, size, MEM_DECOMMIT)) + verify(__func__), ::VirtualFree(real_addr, size, MEM_DECOMMIT); #else - if (::madvise(real_addr, size, MADV_REMOVE) || ::mprotect(real_addr, size, PROT_NONE)) + verify(__func__), ::mmap(real_addr, size, PROT_NONE, MAP_FIXED | MAP_ANON | MAP_PRIVATE, -1, 0); #endif - { - fmt::throw_exception("System failure (addr=0x%x, size=0x%x)" HERE, addr, size); - } } bool check_addr(u32 addr, u32 size, u8 flags) @@ -428,7 +536,7 @@ namespace vm block_t::~block_t() { - mem_lock lock(g_mutex); + writer_lock lock; // Deallocate all memory for (auto& entry : m_map) @@ -439,7 +547,7 @@ namespace vm u32 block_t::alloc(u32 size, u32 align, u32 sup) { - mem_lock lock(g_mutex); + writer_lock lock; // Align to minimal page size size = ::align(size, 4096); @@ -481,7 +589,7 @@ namespace vm u32 block_t::falloc(u32 addr, u32 size, u32 sup) { - mem_lock lock(g_mutex); + writer_lock lock; // align to minimal page size size = ::align(size, 4096); @@ -502,7 +610,7 @@ namespace vm u32 block_t::dealloc(u32 addr, u32* sup_out) { - mem_lock lock(g_mutex); + writer_lock lock; const auto found = m_map.find(addr); @@ -530,7 +638,7 @@ namespace vm u32 block_t::used() { - mem_lock lock(g_mutex); + reader_lock lock; u32 result = 0; @@ -544,7 +652,7 @@ namespace vm std::shared_ptr map(u32 addr, u32 size, u64 flags) { - mem_lock lock(g_mutex); + writer_lock lock(0); if (!size || (size | addr) % 4096) { @@ -581,7 +689,7 @@ namespace vm std::shared_ptr unmap(u32 addr, bool must_be_empty) { - mem_lock lock(g_mutex); + writer_lock lock(0); for (auto it = g_locations.begin(); it != g_locations.end(); it++) { @@ -603,7 +711,7 @@ namespace vm std::shared_ptr get(memory_location_t location, u32 addr) { - mem_lock lock(g_mutex); + reader_lock lock; if (location != any) { diff --git a/rpcs3/Emu/Memory/vm.h b/rpcs3/Emu/Memory/vm.h index afc3d97b52..bfc88c7323 100644 --- a/rpcs3/Emu/Memory/vm.h +++ b/rpcs3/Emu/Memory/vm.h @@ -4,16 +4,13 @@ #include #include -#include "Utilities/mutex.h" - class named_thread; +class cpu_thread; namespace vm { extern u8* const g_base_addr; - extern shared_mutex g_mutex; - enum memory_location_t : uint { main, @@ -60,22 +57,62 @@ namespace vm // Address type enum addr_t : u32 {}; + extern thread_local atomic_t* g_tls_locked; + + // Register reader + void passive_lock(cpu_thread& cpu); + + // Unregister reader + void passive_unlock(cpu_thread& cpu); + + // Unregister reader (foreign thread) + void cleanup_unlock(cpu_thread& cpu) noexcept; + + // Optimization (set cpu_flag::memory) + void temporary_unlock(cpu_thread& cpu) noexcept; + + constexpr struct try_to_lock_t{} try_to_lock{}; + + struct reader_lock final + { + const bool locked; + + reader_lock(const reader_lock&) = delete; + reader_lock(); + reader_lock(const try_to_lock_t&); + ~reader_lock(); + + explicit operator bool() const { return locked; } + }; + + struct writer_lock final + { + const bool locked; + + writer_lock(const writer_lock&) = delete; + writer_lock(int full = 1); + writer_lock(const try_to_lock_t&); + ~writer_lock(); + + explicit operator bool() const { return locked; } + }; + // Get reservation status for further atomic update: last update timestamp u64 reservation_acquire(u32 addr, u32 size); // End atomic update void reservation_update(u32 addr, u32 size); - // Check and notify memory change at address + // Check and notify memory changes at address void notify(u32 addr, u32 size); + // Check and notify memory changes void notify_all(); // Change memory protection of specified memory region bool page_protect(u32 addr, u32 size, u8 flags_test = 0, u8 flags_set = 0, u8 flags_clear = 0); - // Check if existing memory range is allocated. Checking address before using it is very unsafe. - // Return value may be wrong. Even if it's true and correct, actual memory protection may be read-only and no-access. + // Check flags for specified memory range (unsafe) bool check_addr(u32 addr, u32 size = 1, u8 flags = page_allocated); // Search and map memory in specified memory location (don't pass alignment smaller than 4096) diff --git a/rpcs3/Emu/PSP2/ARMv7Interpreter.cpp b/rpcs3/Emu/PSP2/ARMv7Interpreter.cpp index 5b7446eab7..65c3eed74c 100644 --- a/rpcs3/Emu/PSP2/ARMv7Interpreter.cpp +++ b/rpcs3/Emu/PSP2/ARMv7Interpreter.cpp @@ -2091,7 +2091,7 @@ void arm_interpreter::STREX(ARMv7Thread& cpu, const u32 op, const u32 cond) return; } - writer_lock lock(vm::g_mutex); + vm::writer_lock lock(0); const bool result = cpu.rtime == vm::reservation_acquire(addr, cpu.rtime) && data.compare_and_swap_test(cpu.rdata, value);