diff --git a/rpcs3/Emu/Cell/Modules/cellAudio.cpp b/rpcs3/Emu/Cell/Modules/cellAudio.cpp index 10bd68e82f..7ecf9f1b2c 100644 --- a/rpcs3/Emu/Cell/Modules/cellAudio.cpp +++ b/rpcs3/Emu/Cell/Modules/cellAudio.cpp @@ -606,6 +606,7 @@ void cell_audio_thread::advance(u64 timestamp) for (u32 i = 0; i < queue_count; i++) { + lv2_obj::notify_all_t notify; queues[i]->send(event_sources[i], 0, 0, 0); } } diff --git a/rpcs3/Emu/Cell/SPUThread.cpp b/rpcs3/Emu/Cell/SPUThread.cpp index 1c56e4cd56..7f4049cd83 100644 --- a/rpcs3/Emu/Cell/SPUThread.cpp +++ b/rpcs3/Emu/Cell/SPUThread.cpp @@ -1362,6 +1362,7 @@ void spu_thread::cpu_return() if (ensure(group->running)-- == 1) { { + lv2_obj::notify_all_t notify; std::lock_guard lock(group->mutex); group->run_state = SPU_THREAD_GROUP_STATUS_INITIALIZED; @@ -4269,6 +4270,8 @@ bool spu_thread::set_ch_value(u32 ch, u32 value) state += cpu_flag::wait; + lv2_obj::notify_all_t notify; + const u32 code = value >> 24; { if (code < 64) diff --git a/rpcs3/Emu/Cell/lv2/lv2.cpp b/rpcs3/Emu/Cell/lv2/lv2.cpp index 9a6ec2cee3..05767bed8f 100644 --- a/rpcs3/Emu/Cell/lv2/lv2.cpp +++ b/rpcs3/Emu/Cell/lv2/lv2.cpp @@ -1192,6 +1192,7 @@ DECLARE(lv2_obj::g_ppu){}; DECLARE(lv2_obj::g_pending){}; thread_local DECLARE(lv2_obj::g_to_notify){}; +thread_local DECLARE(lv2_obj::g_postpone_notify_barrier){}; thread_local DECLARE(lv2_obj::g_to_awake); // Scheduler queue for timeouts (wait until -> thread) @@ -1205,31 +1206,57 @@ namespace cpu_counter void remove(cpu_thread*) noexcept; } -void lv2_obj::sleep(cpu_thread& cpu, const u64 timeout, bool notify_later) +void lv2_obj::sleep(cpu_thread& cpu, const u64 timeout) { // Should already be performed when using this flag - if (!notify_later) + if (!g_postpone_notify_barrier) { - vm::temporary_unlock(cpu); - cpu_counter::remove(&cpu); + prepare_for_sleep(cpu); } { std::lock_guard lock{g_mutex}; - sleep_unlocked(cpu, timeout, notify_later); + sleep_unlocked(cpu, timeout); + + if (!g_to_awake.empty()) + { + // Schedule pending entries + awake_unlocked({}); + } + + schedule_all(); } + + if (!g_postpone_notify_barrier) + { + notify_all(); + } + g_to_awake.clear(); } -bool lv2_obj::awake(cpu_thread* const thread, bool notify_later, s32 prio) +bool lv2_obj::awake(cpu_thread* thread, s32 prio) { - // Too risky to postpone it in case the notified thread may wait for this thread to free its passive lock - if (!notify_later) + bool result = false; { - vm::temporary_unlock(); + std::lock_guard lock(g_mutex); + result = awake_unlocked(thread, prio); + schedule_all(); } - std::lock_guard lock(g_mutex); - return awake_unlocked(thread, notify_later, prio); + if (result) + { + if (auto cpu = cpu_thread::get_current(); cpu && cpu->is_paused()) + { + vm::temporary_unlock(); + } + } + + if (!g_postpone_notify_barrier) + { + notify_all(); + } + + return result; } bool lv2_obj::yield(cpu_thread& thread) @@ -1241,10 +1268,10 @@ bool lv2_obj::yield(cpu_thread& thread) ppu->raddr = 0; // Clear reservation } - return awake(&thread, false, yield_cmd); + return awake(&thread, yield_cmd); } -void lv2_obj::sleep_unlocked(cpu_thread& thread, u64 timeout, bool notify_later) +void lv2_obj::sleep_unlocked(cpu_thread& thread, u64 timeout) { const u64 start_time = get_guest_system_time(); @@ -1341,19 +1368,9 @@ void lv2_obj::sleep_unlocked(cpu_thread& thread, u64 timeout, bool notify_later) } } } - - if (!g_to_awake.empty()) - { - // Schedule pending entries - awake_unlocked({}, notify_later); - } - else - { - schedule_all(notify_later); - } } -bool lv2_obj::awake_unlocked(cpu_thread* cpu, bool notify_later, s32 prio) +bool lv2_obj::awake_unlocked(cpu_thread* cpu, s32 prio) { // Check thread type AUDIT(!cpu || cpu->id_type() == 1); @@ -1498,9 +1515,9 @@ bool lv2_obj::awake_unlocked(cpu_thread* cpu, bool notify_later, s32 prio) // Emplace current thread if (!emplace_thread(cpu)) { - if (notify_later) + if (g_postpone_notify_barrier) { - // notify_later includes common optimizations regarding syscalls + // This flag includes common optimizations regarding syscalls // one of which is to allow a lock-free version of syscalls with awake behave as semaphore post: always notifies the thread, even if it hasn't slept yet cpu->state += cpu_flag::signal; } @@ -1515,7 +1532,7 @@ bool lv2_obj::awake_unlocked(cpu_thread* cpu, bool notify_later, s32 prio) // Emplace threads from list if (!emplace_thread(_cpu)) { - if (notify_later) + if (g_postpone_notify_barrier) { _cpu->state += cpu_flag::signal; } @@ -1557,7 +1574,6 @@ bool lv2_obj::awake_unlocked(cpu_thread* cpu, bool notify_later, s32 prio) } } - schedule_all(notify_later); return changed_queue; } @@ -1569,11 +1585,11 @@ void lv2_obj::cleanup() g_pending = 0; } -void lv2_obj::schedule_all(bool notify_later) +void lv2_obj::schedule_all() { if (!g_pending && g_to_sleep.empty()) { - usz notify_later_idx = notify_later ? 0 : std::size(g_to_notify) - 1; + usz notify_later_idx = 0; auto target = +g_ppu; @@ -1592,7 +1608,7 @@ void lv2_obj::schedule_all(bool notify_later) } else { - g_to_notify[notify_later_idx++] = target; + g_to_notify[notify_later_idx++] = &target->state; } } } diff --git a/rpcs3/Emu/Cell/lv2/sys_cond.cpp b/rpcs3/Emu/Cell/lv2/sys_cond.cpp index f12bab35eb..6e571d01ba 100644 --- a/rpcs3/Emu/Cell/lv2/sys_cond.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_cond.cpp @@ -142,12 +142,10 @@ error_code sys_cond_signal(ppu_thread& ppu, u32 cond_id) sys_cond.trace("sys_cond_signal(cond_id=0x%x)", cond_id); - const auto cond = idm::check(cond_id, [&](lv2_cond& cond) + const auto cond = idm::check(cond_id, [&, notify = lv2_obj::notify_all_t()](lv2_cond& cond) { if (atomic_storage::load(cond.sq)) { - lv2_obj::notify_all_t notify; - std::lock_guard lock(cond.mutex->mutex); if (const auto cpu = cond.schedule(cond.sq, cond.mutex->protocol)) @@ -162,7 +160,7 @@ error_code sys_cond_signal(ppu_thread& ppu, u32 cond_id) if (cond.mutex->try_own(*cpu)) { - cond.awake(cpu, true); + cond.awake(cpu); } } } @@ -182,12 +180,10 @@ error_code sys_cond_signal_all(ppu_thread& ppu, u32 cond_id) sys_cond.trace("sys_cond_signal_all(cond_id=0x%x)", cond_id); - const auto cond = idm::check(cond_id, [&](lv2_cond& cond) + const auto cond = idm::check(cond_id, [&, notify = lv2_obj::notify_all_t()](lv2_cond& cond) { if (atomic_storage::load(cond.sq)) { - lv2_obj::notify_all_t notify; - std::lock_guard lock(cond.mutex->mutex); for (auto cpu = +cond.sq; cpu; cpu = cpu->next_cpu) @@ -213,7 +209,7 @@ error_code sys_cond_signal_all(ppu_thread& ppu, u32 cond_id) if (result) { - cond.awake(result, true); + cond.awake(result); } } }); @@ -232,7 +228,7 @@ error_code sys_cond_signal_to(ppu_thread& ppu, u32 cond_id, u32 thread_id) sys_cond.trace("sys_cond_signal_to(cond_id=0x%x, thread_id=0x%x)", cond_id, thread_id); - const auto cond = idm::check(cond_id, [&](lv2_cond& cond) -> int + const auto cond = idm::check(cond_id, [&, notify = lv2_obj::notify_all_t()](lv2_cond& cond) -> int { if (!idm::check_unlocked>(thread_id)) { @@ -241,8 +237,6 @@ error_code sys_cond_signal_to(ppu_thread& ppu, u32 cond_id, u32 thread_id) if (atomic_storage::load(cond.sq)) { - lv2_obj::notify_all_t notify; - std::lock_guard lock(cond.mutex->mutex); for (auto cpu = +cond.sq; cpu; cpu = cpu->next_cpu) @@ -259,7 +253,7 @@ error_code sys_cond_signal_to(ppu_thread& ppu, u32 cond_id, u32 thread_id) if (cond.mutex->try_own(*cpu)) { - cond.awake(cpu, true); + cond.awake(cpu); } return 1; @@ -294,14 +288,14 @@ error_code sys_cond_wait(ppu_thread& ppu, u32 cond_id, u64 timeout) auto& sstate = *ppu.optional_savestate_state; - const auto cond = idm::get(cond_id, [&](lv2_cond& cond) -> s64 + const auto cond = idm::get(cond_id, [&, notify = lv2_obj::notify_all_t()](lv2_cond& cond) -> s64 { if (!ppu.loaded_from_savestate && atomic_storage::load(cond.mutex->control.raw().owner) != ppu.id) { return -1; } - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); std::lock_guard lock(cond.mutex->mutex); @@ -320,7 +314,7 @@ error_code sys_cond_wait(ppu_thread& ppu, u32 cond_id, u64 timeout) lv2_obj::emplace(cond.sq, &ppu); } - cond.sleep(ppu, timeout, true); + cond.sleep(ppu, timeout); return static_cast(syscall_state >> 32); } @@ -342,7 +336,7 @@ error_code sys_cond_wait(ppu_thread& ppu, u32 cond_id, u64 timeout) lv2_obj::emplace(cond.sq, &ppu); // Sleep current thread and schedule mutex waiter - cond.sleep(ppu, timeout, true); + cond.sleep(ppu, timeout); // Save the recursive value return count; diff --git a/rpcs3/Emu/Cell/lv2/sys_event.cpp b/rpcs3/Emu/Cell/lv2/sys_event.cpp index 41fab64e6d..8ba38e586c 100644 --- a/rpcs3/Emu/Cell/lv2/sys_event.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_event.cpp @@ -109,10 +109,8 @@ std::shared_ptr lv2_event_queue::find(u64 ipc_key) extern void resume_spu_thread_group_from_waiting(spu_thread& spu); -CellError lv2_event_queue::send(lv2_event event, bool notify_later) +CellError lv2_event_queue::send(lv2_event event) { - lv2_obj::notify_all_t notify; - std::lock_guard lock(mutex); if (!exists) @@ -153,7 +151,7 @@ CellError lv2_event_queue::send(lv2_event event, bool notify_later) std::tie(ppu.gpr[4], ppu.gpr[5], ppu.gpr[6], ppu.gpr[7]) = event; - awake(&ppu, notify_later); + awake(&ppu); } else { @@ -414,14 +412,14 @@ error_code sys_event_queue_receive(ppu_thread& ppu, u32 equeue_id, vm::ptr(equeue_id, [&](lv2_event_queue& queue) -> CellError + const auto queue = idm::get(equeue_id, [&, notify = lv2_obj::notify_all_t()](lv2_event_queue& queue) -> CellError { if (queue.type != SYS_PPU_QUEUE) { return CELL_EINVAL; } - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); std::lock_guard lock(queue.mutex); @@ -435,7 +433,7 @@ error_code sys_event_queue_receive(ppu_thread& ppu, u32 equeue_id, vm::ptr(eport_id, [&](lv2_event_port& port) -> CellError + const auto port = idm::check(eport_id, [&, notify = lv2_obj::notify_all_t()](lv2_event_port& port) -> CellError { if (lv2_obj::check(port.queue)) { const u64 source = port.name ? port.name : (s64{process_getpid()} << 32) | u64{eport_id}; - return port.queue->send(source, data1, data2, data3, true); + return port.queue->send(source, data1, data2, data3); } return CELL_ENOTCONN; diff --git a/rpcs3/Emu/Cell/lv2/sys_event.h b/rpcs3/Emu/Cell/lv2/sys_event.h index 9f01a235d7..90fd092f55 100644 --- a/rpcs3/Emu/Cell/lv2/sys_event.h +++ b/rpcs3/Emu/Cell/lv2/sys_event.h @@ -103,11 +103,11 @@ struct lv2_event_queue final : public lv2_obj static void save_ptr(utils::serial&, lv2_event_queue*); static std::shared_ptr load_ptr(utils::serial& ar, std::shared_ptr& queue); - CellError send(lv2_event event, bool notify_later = false); + CellError send(lv2_event event); - CellError send(u64 source, u64 d1, u64 d2, u64 d3, bool notify_later = false) + CellError send(u64 source, u64 d1, u64 d2, u64 d3) { - return send(std::make_tuple(source, d1, d2, d3), notify_later); + return send(std::make_tuple(source, d1, d2, d3)); } // Get event queue by its global key diff --git a/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp b/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp index 7e14daacad..1a489638e8 100644 --- a/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp @@ -141,7 +141,7 @@ error_code sys_event_flag_wait(ppu_thread& ppu, u32 id, u64 bitptn, u32 mode, vm return CELL_EINVAL; } - const auto flag = idm::get(id, [&](lv2_event_flag& flag) -> CellError + const auto flag = idm::get(id, [&, notify = lv2_obj::notify_all_t()](lv2_event_flag& flag) -> CellError { if (flag.pattern.fetch_op([&](u64& pat) { @@ -152,7 +152,7 @@ error_code sys_event_flag_wait(ppu_thread& ppu, u32 id, u64 bitptn, u32 mode, vm return {}; } - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); std::lock_guard lock(flag.mutex); @@ -169,7 +169,7 @@ error_code sys_event_flag_wait(ppu_thread& ppu, u32 id, u64 bitptn, u32 mode, vm return CELL_EPERM; } - flag.sleep(ppu, timeout, true); + flag.sleep(ppu, timeout); lv2_obj::emplace(flag.sq, &ppu); return CELL_EBUSY; }); @@ -314,7 +314,7 @@ error_code sys_event_flag_set(cpu_thread& cpu, u32 id, u64 bitptn) return CELL_OK; } - if (true) + if (lv2_obj::notify_all_t notify; true) { std::lock_guard lock(flag->mutex); @@ -454,6 +454,8 @@ error_code sys_event_flag_cancel(ppu_thread& ppu, u32 id, vm::ptr num) u32 value = 0; { + lv2_obj::notify_all_t notify; + std::lock_guard lock(flag->mutex); for (auto cpu = +flag->sq; cpu; cpu = cpu->next_cpu) diff --git a/rpcs3/Emu/Cell/lv2/sys_lwcond.cpp b/rpcs3/Emu/Cell/lv2/sys_lwcond.cpp index 16c4996ed5..a8606b1610 100644 --- a/rpcs3/Emu/Cell/lv2/sys_lwcond.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_lwcond.cpp @@ -101,7 +101,7 @@ error_code _sys_lwcond_signal(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id, u6 fmt::throw_exception("Unknown mode (%d)", mode); } - const auto cond = idm::check(lwcond_id, [&](lv2_lwcond& cond) -> int + const auto cond = idm::check(lwcond_id, [&, notify = lv2_obj::notify_all_t()](lv2_lwcond& cond) -> int { ppu_thread* cpu = nullptr; @@ -129,8 +129,6 @@ error_code _sys_lwcond_signal(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id, u6 if (atomic_storage::load(cond.sq)) { - lv2_obj::notify_all_t notify; - std::lock_guard lock(cond.mutex); if (cpu) @@ -189,7 +187,7 @@ error_code _sys_lwcond_signal(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id, u6 if (result) { - cond.awake(result, true); + cond.awake(result); } return 1; @@ -238,7 +236,7 @@ error_code _sys_lwcond_signal_all(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id fmt::throw_exception("Unknown mode (%d)", mode); } - const auto cond = idm::check(lwcond_id, [&](lv2_lwcond& cond) -> s32 + const auto cond = idm::check(lwcond_id, [&, notify = lv2_obj::notify_all_t()](lv2_lwcond& cond) -> s32 { lv2_lwmutex* mutex{}; @@ -254,8 +252,6 @@ error_code _sys_lwcond_signal_all(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id if (atomic_storage::load(cond.sq)) { - lv2_obj::notify_all_t notify; - std::lock_guard lock(cond.mutex); u32 result = 0; @@ -293,7 +289,7 @@ error_code _sys_lwcond_signal_all(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id if (result && mode == 2) { - lv2_obj::awake_all(true); + lv2_obj::awake_all(); } return result; @@ -328,7 +324,7 @@ error_code _sys_lwcond_queue_wait(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id auto& sstate = *ppu.optional_savestate_state; - const auto cond = idm::get(lwcond_id, [&](lv2_lwcond& cond) + const auto cond = idm::get(lwcond_id, [&, notify = lv2_obj::notify_all_t()](lv2_lwcond& cond) { mutex = idm::get_unlocked(lwmutex_id); @@ -340,7 +336,7 @@ error_code _sys_lwcond_queue_wait(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id // Increment lwmutex's lwcond's waiters count mutex->lwcond_waiters++; - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); std::lock_guard lock(cond.mutex); @@ -377,7 +373,7 @@ error_code _sys_lwcond_queue_wait(ppu_thread& ppu, u32 lwcond_id, u32 lwmutex_id } // Sleep current thread and schedule lwmutex waiter - cond.sleep(ppu, timeout, true); + cond.sleep(ppu, timeout); }); if (!cond || !mutex) diff --git a/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp b/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp index 4eef449bc4..f2c292858b 100644 --- a/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp @@ -139,7 +139,7 @@ error_code _sys_lwmutex_lock(ppu_thread& ppu, u32 lwmutex_id, u64 timeout) ppu.gpr[3] = CELL_OK; - const auto mutex = idm::get(lwmutex_id, [&](lv2_lwmutex& mutex) + const auto mutex = idm::get(lwmutex_id, [&, notify = lv2_obj::notify_all_t()](lv2_lwmutex& mutex) { if (s32 signal = mutex.lv2_control.fetch_op([](auto& data) { @@ -160,7 +160,7 @@ error_code _sys_lwmutex_lock(ppu_thread& ppu, u32 lwmutex_id, u64 timeout) return true; } - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); if (s32 signal = mutex.try_own(&ppu)) { @@ -172,7 +172,7 @@ error_code _sys_lwmutex_lock(ppu_thread& ppu, u32 lwmutex_id, u64 timeout) return true; } - mutex.sleep(ppu, timeout, true); + mutex.sleep(ppu, timeout); return false; }); @@ -290,15 +290,13 @@ error_code _sys_lwmutex_unlock(ppu_thread& ppu, u32 lwmutex_id) sys_lwmutex.trace("_sys_lwmutex_unlock(lwmutex_id=0x%x)", lwmutex_id); - const auto mutex = idm::check(lwmutex_id, [&](lv2_lwmutex& mutex) + const auto mutex = idm::check(lwmutex_id, [&, notify = lv2_obj::notify_all_t()](lv2_lwmutex& mutex) { if (mutex.try_unlock(false)) { return; } - lv2_obj::notify_all_t notify; - std::lock_guard lock(mutex.mutex); if (const auto cpu = mutex.reown()) @@ -309,7 +307,7 @@ error_code _sys_lwmutex_unlock(ppu_thread& ppu, u32 lwmutex_id) return; } - mutex.awake(cpu, true); + mutex.awake(cpu); return; } }); @@ -328,15 +326,13 @@ error_code _sys_lwmutex_unlock2(ppu_thread& ppu, u32 lwmutex_id) sys_lwmutex.warning("_sys_lwmutex_unlock2(lwmutex_id=0x%x)", lwmutex_id); - const auto mutex = idm::check(lwmutex_id, [&](lv2_lwmutex& mutex) + const auto mutex = idm::check(lwmutex_id, [&, notify = lv2_obj::notify_all_t()](lv2_lwmutex& mutex) { if (mutex.try_unlock(true)) { return; } - lv2_obj::notify_all_t notify; - std::lock_guard lock(mutex.mutex); if (const auto cpu = mutex.reown(true)) @@ -348,7 +344,7 @@ error_code _sys_lwmutex_unlock2(ppu_thread& ppu, u32 lwmutex_id) } static_cast(cpu)->gpr[3] = CELL_EBUSY; - mutex.awake(cpu, true); + mutex.awake(cpu); return; } }); diff --git a/rpcs3/Emu/Cell/lv2/sys_mutex.cpp b/rpcs3/Emu/Cell/lv2/sys_mutex.cpp index 97f211ee56..8c64a7e3d0 100644 --- a/rpcs3/Emu/Cell/lv2/sys_mutex.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_mutex.cpp @@ -139,7 +139,7 @@ error_code sys_mutex_lock(ppu_thread& ppu, u32 mutex_id, u64 timeout) sys_mutex.trace("sys_mutex_lock(mutex_id=0x%x, timeout=0x%llx)", mutex_id, timeout); - const auto mutex = idm::get(mutex_id, [&](lv2_mutex& mutex) + const auto mutex = idm::get(mutex_id, [&, notify = lv2_obj::notify_all_t()](lv2_mutex& mutex) { CellError result = mutex.try_lock(ppu); @@ -160,7 +160,7 @@ error_code sys_mutex_lock(ppu_thread& ppu, u32 mutex_id, u64 timeout) if (result == CELL_EBUSY) { - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); if (mutex.try_own(ppu)) { @@ -168,7 +168,7 @@ error_code sys_mutex_lock(ppu_thread& ppu, u32 mutex_id, u64 timeout) } else { - mutex.sleep(ppu, timeout, true); + mutex.sleep(ppu, timeout); } } @@ -292,14 +292,12 @@ error_code sys_mutex_unlock(ppu_thread& ppu, u32 mutex_id) sys_mutex.trace("sys_mutex_unlock(mutex_id=0x%x)", mutex_id); - const auto mutex = idm::check(mutex_id, [&](lv2_mutex& mutex) -> CellError + const auto mutex = idm::check(mutex_id, [&, notify = lv2_obj::notify_all_t()](lv2_mutex& mutex) -> CellError { auto result = mutex.try_unlock(ppu); if (result == CELL_EBUSY) { - lv2_obj::notify_all_t notify; - std::lock_guard lock(mutex.mutex); if (auto cpu = mutex.reown()) @@ -310,7 +308,7 @@ error_code sys_mutex_unlock(ppu_thread& ppu, u32 mutex_id) return {}; } - mutex.awake(cpu, true); + mutex.awake(cpu); } result = {}; diff --git a/rpcs3/Emu/Cell/lv2/sys_net.cpp b/rpcs3/Emu/Cell/lv2/sys_net.cpp index 30de34df56..8433d46bbe 100644 --- a/rpcs3/Emu/Cell/lv2/sys_net.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_net.cpp @@ -358,7 +358,7 @@ error_code sys_net_bnet_accept(ppu_thread& ppu, s32 s, vm::ptr sys_net_sockaddr sn_addr{}; std::shared_ptr new_socket{}; - const auto sock = idm::check(s, [&](lv2_socket& sock) + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) { const auto [success, res, res_socket, res_addr] = sock.accept(); @@ -391,6 +391,7 @@ error_code sys_net_bnet_accept(ppu_thread& ppu, s32 s, vm::ptr return false; }); + lv2_obj::prepare_for_sleep(ppu); lv2_obj::sleep(ppu); return false; }); @@ -482,7 +483,7 @@ error_code sys_net_bnet_bind(ppu_thread& ppu, s32 s, vm::cptr return -SYS_NET_EAFNOSUPPORT; } - const auto sock = idm::check(s, [&](lv2_socket& sock) -> s32 + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) -> s32 { return sock.bind(sn_addr); }); @@ -519,7 +520,7 @@ error_code sys_net_bnet_connect(ppu_thread& ppu, s32 s, vm::ptr(s, [&](lv2_socket& sock) + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) { const auto success = sock.connect(sn_addr); @@ -544,8 +545,8 @@ error_code sys_net_bnet_connect(ppu_thread& ppu, s32 s, vm::ptr(s, [&](lv2_socket& sock) -> s32 + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) -> s32 { auto [res, sn_addr] = sock.getpeername(); @@ -650,7 +651,7 @@ error_code sys_net_bnet_getsockname(ppu_thread& ppu, s32 s, vm::ptr(s, [&](lv2_socket& sock) -> s32 + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) -> s32 { auto [res, sn_addr] = sock.getsockname(); @@ -708,7 +709,7 @@ error_code sys_net_bnet_getsockopt(ppu_thread& ppu, s32 s, s32 level, s32 optnam return -SYS_NET_EINVAL; } - const auto sock = idm::check(s, [&](lv2_socket& sock) -> s32 + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) -> s32 { if (len < sizeof(s32)) { @@ -750,7 +751,7 @@ error_code sys_net_bnet_listen(ppu_thread& ppu, s32 s, s32 backlog) return -SYS_NET_EINVAL; } - const auto sock = idm::check(s, [&](lv2_socket& sock) -> s32 + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) -> s32 { return sock.listen(backlog); }); @@ -788,7 +789,7 @@ error_code sys_net_bnet_recvfrom(ppu_thread& ppu, s32 s, vm::ptr buf, u32 s32 result = 0; sys_net_sockaddr sn_addr{}; - const auto sock = idm::check(s, [&](lv2_socket& sock) + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) { const auto success = sock.recvfrom(flags, len); @@ -832,6 +833,7 @@ error_code sys_net_bnet_recvfrom(ppu_thread& ppu, s32 s, vm::ptr buf, u32 return false; }); + lv2_obj::prepare_for_sleep(ppu); lv2_obj::sleep(ppu); return false; }); @@ -929,7 +931,7 @@ error_code sys_net_bnet_sendto(ppu_thread& ppu, s32 s, vm::cptr buf, u32 l const std::vector buf_copy(vm::_ptr(buf.addr()), vm::_ptr(buf.addr()) + len); s32 result{}; - const auto sock = idm::check(s, [&](lv2_socket& sock) + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) { auto success = sock.sendto(flags, buf_copy, sn_addr); @@ -958,6 +960,7 @@ error_code sys_net_bnet_sendto(ppu_thread& ppu, s32 s, vm::cptr buf, u32 l return false; }); + lv2_obj::prepare_for_sleep(ppu); lv2_obj::sleep(ppu); return false; }); @@ -1036,7 +1039,7 @@ error_code sys_net_bnet_setsockopt(ppu_thread& ppu, s32 s, s32 level, s32 optnam std::vector optval_copy(vm::_ptr(optval.addr()), vm::_ptr(optval.addr() + optlen)); - const auto sock = idm::check(s, [&](lv2_socket& sock) -> s32 + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) -> s32 { return sock.setsockopt(level, optname, optval_copy); }); @@ -1065,7 +1068,7 @@ error_code sys_net_bnet_shutdown(ppu_thread& ppu, s32 s, s32 how) return -SYS_NET_EINVAL; } - const auto sock = idm::check(s, [&](lv2_socket& sock) -> s32 + const auto sock = idm::check(s, [&, notify = lv2_obj::notify_all_t()](lv2_socket& sock) -> s32 { return sock.shutdown(how); }); @@ -1181,6 +1184,8 @@ error_code sys_net_bnet_poll(ppu_thread& ppu, vm::ptr fds, s32 n { fds_buf.assign(fds.get_ptr(), fds.get_ptr() + nfds); + lv2_obj::prepare_for_sleep(ppu); + std::unique_lock nw_lock(g_fxo->get().s_nw_mutex); std::shared_lock lock(id_manager::g_mutex); diff --git a/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp b/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp index 1d86d0b698..9d22591d48 100644 --- a/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp @@ -76,6 +76,9 @@ void _sys_ppu_thread_exit(ppu_thread& ppu, u64 errorcode) // Avoid cases where cleaning causes the destructor to be called inside IDM lock scope (for performance) std::shared_ptr old_ppu; + lv2_obj::notify_all_t notify; + lv2_obj::prepare_for_sleep(ppu); + std::lock_guard lock(id_manager::g_mutex); // Get joiner ID @@ -105,6 +108,7 @@ void _sys_ppu_thread_exit(ppu_thread& ppu, u64 errorcode) // Unqueue lv2_obj::sleep(ppu); + notify.cleanup(); // Remove suspend state (TODO) ppu.state -= cpu_flag::suspend; @@ -138,11 +142,11 @@ s32 sys_ppu_thread_yield(ppu_thread& ppu) error_code sys_ppu_thread_join(ppu_thread& ppu, u32 thread_id, vm::ptr vptr) { - ppu.state += cpu_flag::wait; + lv2_obj::prepare_for_sleep(ppu); sys_ppu_thread.trace("sys_ppu_thread_join(thread_id=0x%x, vptr=*0x%x)", thread_id, vptr); - auto thread = idm::get>(thread_id, [&](ppu_thread& thread) -> CellError + auto thread = idm::get>(thread_id, [&, notify = lv2_obj::notify_all_t()](ppu_thread& thread) -> CellError { if (&ppu == &thread) { @@ -173,6 +177,7 @@ error_code sys_ppu_thread_join(ppu_thread& ppu, u32 thread_id, vm::ptr vptr if (!result) { + lv2_obj::prepare_for_sleep(ppu); lv2_obj::sleep(ppu); } else if (result == CELL_EAGAIN) @@ -180,6 +185,7 @@ error_code sys_ppu_thread_join(ppu_thread& ppu, u32 thread_id, vm::ptr vptr thread.joiner.notify_one(); } + notify.cleanup(); return result; }); @@ -471,7 +477,7 @@ error_code sys_ppu_thread_start(ppu_thread& ppu, u32 thread_id) sys_ppu_thread.trace("sys_ppu_thread_start(thread_id=0x%x)", thread_id); - const auto thread = idm::get>(thread_id, [&](ppu_thread& thread) -> CellError + const auto thread = idm::get>(thread_id, [&, notify = lv2_obj::notify_all_t()](ppu_thread& thread) -> CellError { if (!thread.state.test_and_reset(cpu_flag::stop)) { @@ -479,7 +485,7 @@ error_code sys_ppu_thread_start(ppu_thread& ppu, u32 thread_id) return CELL_EBUSY; } - lv2_obj::awake(&thread); + ensure(lv2_obj::awake(&thread)); thread.cmd_list ({ diff --git a/rpcs3/Emu/Cell/lv2/sys_rwlock.cpp b/rpcs3/Emu/Cell/lv2/sys_rwlock.cpp index f393b4e620..b7f37d99e0 100644 --- a/rpcs3/Emu/Cell/lv2/sys_rwlock.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_rwlock.cpp @@ -102,7 +102,7 @@ error_code sys_rwlock_rlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) sys_rwlock.trace("sys_rwlock_rlock(rw_lock_id=0x%x, timeout=0x%llx)", rw_lock_id, timeout); - const auto rwlock = idm::get(rw_lock_id, [&](lv2_rwlock& rwlock) + const auto rwlock = idm::get(rw_lock_id, [&, notify = lv2_obj::notify_all_t()](lv2_rwlock& rwlock) { const s64 val = rwlock.owner; @@ -114,7 +114,7 @@ error_code sys_rwlock_rlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) } } - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); std::lock_guard lock(rwlock.mutex); @@ -132,7 +132,7 @@ error_code sys_rwlock_rlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) if (_old > 0 || _old & 1) { - rwlock.sleep(ppu, timeout, true); + rwlock.sleep(ppu, timeout); lv2_obj::emplace(rwlock.rq, &ppu); return false; } @@ -276,6 +276,8 @@ error_code sys_rwlock_runlock(ppu_thread& ppu, u32 rw_lock_id) return CELL_ESRCH; } + lv2_obj::notify_all_t notify; + if (rwlock.ret) { return CELL_OK; @@ -330,7 +332,7 @@ error_code sys_rwlock_wlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) sys_rwlock.trace("sys_rwlock_wlock(rw_lock_id=0x%x, timeout=0x%llx)", rw_lock_id, timeout); - const auto rwlock = idm::get(rw_lock_id, [&](lv2_rwlock& rwlock) -> s64 + const auto rwlock = idm::get(rw_lock_id, [&, notify = lv2_obj::notify_all_t()](lv2_rwlock& rwlock) -> s64 { const s64 val = rwlock.owner; @@ -346,7 +348,7 @@ error_code sys_rwlock_wlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) return val; } - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); std::lock_guard lock(rwlock.mutex); @@ -364,7 +366,7 @@ error_code sys_rwlock_wlock(ppu_thread& ppu, u32 rw_lock_id, u64 timeout) if (_old != 0) { - rwlock.sleep(ppu, timeout, true); + rwlock.sleep(ppu, timeout); lv2_obj::emplace(rwlock.wq, &ppu); } @@ -532,7 +534,7 @@ error_code sys_rwlock_wunlock(ppu_thread& ppu, u32 rw_lock_id) return CELL_EPERM; } - if (rwlock.ret & 1) + if (lv2_obj::notify_all_t notify; rwlock.ret & 1) { std::lock_guard lock(rwlock->mutex); diff --git a/rpcs3/Emu/Cell/lv2/sys_semaphore.cpp b/rpcs3/Emu/Cell/lv2/sys_semaphore.cpp index b73344a589..72e35b2156 100644 --- a/rpcs3/Emu/Cell/lv2/sys_semaphore.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_semaphore.cpp @@ -111,7 +111,7 @@ error_code sys_semaphore_wait(ppu_thread& ppu, u32 sem_id, u64 timeout) sys_semaphore.trace("sys_semaphore_wait(sem_id=0x%x, timeout=0x%llx)", sem_id, timeout); - const auto sem = idm::get(sem_id, [&](lv2_sema& sema) + const auto sem = idm::get(sem_id, [&, notify = lv2_obj::notify_all_t()](lv2_sema& sema) { const s32 val = sema.val; @@ -123,13 +123,13 @@ error_code sys_semaphore_wait(ppu_thread& ppu, u32 sem_id, u64 timeout) } } - lv2_obj::notify_all_t notify(ppu); + lv2_obj::prepare_for_sleep(ppu); std::lock_guard lock(sema.mutex); if (sema.val-- <= 0) { - sema.sleep(ppu, timeout, true); + sema.sleep(ppu, timeout); lv2_obj::emplace(sema.sq, &ppu); return false; } @@ -275,6 +275,8 @@ error_code sys_semaphore_post(ppu_thread& ppu, u32 sem_id, s32 count) return CELL_EINVAL; } + lv2_obj::notify_all_t notify; + if (sem.ret) { return CELL_OK; diff --git a/rpcs3/Emu/Cell/lv2/sys_spu.cpp b/rpcs3/Emu/Cell/lv2/sys_spu.cpp index 57450aafe3..fe49e647db 100644 --- a/rpcs3/Emu/Cell/lv2/sys_spu.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_spu.cpp @@ -1399,6 +1399,8 @@ error_code sys_spu_thread_group_join(ppu_thread& ppu, u32 id, vm::ptr cause do { + lv2_obj::prepare_for_sleep(ppu); + std::unique_lock lock(group->mutex); const auto state = +group->run_state; @@ -1433,8 +1435,11 @@ error_code sys_spu_thread_group_join(ppu_thread& ppu, u32 id, vm::ptr cause group->waiter = &ppu; } - lv2_obj::sleep(ppu); - lock.unlock(); + { + lv2_obj::notify_all_t notify; + lv2_obj::sleep(ppu); + lock.unlock(); + } while (auto state = +ppu.state) { diff --git a/rpcs3/Emu/Cell/lv2/sys_sync.h b/rpcs3/Emu/Cell/lv2/sys_sync.h index 88a24893b5..7a84d3eba5 100644 --- a/rpcs3/Emu/Cell/lv2/sys_sync.h +++ b/rpcs3/Emu/Cell/lv2/sys_sync.h @@ -225,17 +225,17 @@ public: private: // Remove the current thread from the scheduling queue, register timeout - static void sleep_unlocked(cpu_thread&, u64 timeout, bool notify_later); + static void sleep_unlocked(cpu_thread&, u64 timeout); // Schedule the thread - static bool awake_unlocked(cpu_thread*, bool notify_later = false, s32 prio = enqueue_cmd); + static bool awake_unlocked(cpu_thread*, s32 prio = enqueue_cmd); public: static constexpr u64 max_timeout = u64{umax} / 1000; - static void sleep(cpu_thread& cpu, const u64 timeout = 0, bool notify_later = false); + static void sleep(cpu_thread& cpu, const u64 timeout = 0); - static bool awake(cpu_thread* const thread, bool notify_later = false, s32 prio = enqueue_cmd); + static bool awake(cpu_thread* thread, s32 prio = enqueue_cmd); // Returns true on successful context switch, false otherwise static bool yield(cpu_thread& thread); @@ -243,12 +243,12 @@ public: static void set_priority(cpu_thread& thread, s32 prio) { ensure(prio + 512u < 3712); - awake(&thread, false, prio); + awake(&thread, prio); } - static inline void awake_all(bool notify_later = false) + static inline void awake_all() { - awake({}, notify_later); + awake({}); g_to_awake.clear(); } @@ -503,37 +503,32 @@ public: if (!cpu) { g_to_notify[0] = nullptr; + g_postpone_notify_barrier = false; return; } - if (cpu->state & cpu_flag::signal) - { - cpu->state.notify_one(cpu_flag::suspend + cpu_flag::signal); - } + // Note: by the time of notification the thread could have been deallocated which is why the direct function is used + // TODO: Pass a narrower mask + atomic_wait_engine::notify_one(cpu, 4, atomic_wait::default_mask>); } } - template + // Can be called before the actual sleep call in order to move it out of mutex scope + static inline void prepare_for_sleep(cpu_thread& cpu) + { + vm::temporary_unlock(cpu); + cpu_counter::remove(&cpu); + } + struct notify_all_t { - notify_all_t() noexcept = default; - - notify_all_t(T& cpu) noexcept + notify_all_t() noexcept { - vm::temporary_unlock(cpu); - cpu_counter::remove(&cpu); + g_postpone_notify_barrier = true; } ~notify_all_t() noexcept { - if constexpr (!std::is_base_of_v) - { - if (auto cpu = cpu_thread::get_current(); cpu && cpu->is_paused()) - { - vm::temporary_unlock(*cpu); - } - } - lv2_obj::notify_all(); } }; @@ -551,8 +546,11 @@ private: // Waiting for the response from static u32 g_pending; - // Pending list of threads to notify - static thread_local std::add_pointer_t g_to_notify[4]; + // Pending list of threads to notify (cpu_thread::state ptr) + static thread_local std::add_pointer_t g_to_notify[4]; - static void schedule_all(bool notify_later); + // If a notify_all_t object exists locally, postpone notifications to the destructor of it (not recursive, notifies on the first destructor for safety) + static thread_local bool g_postpone_notify_barrier; + + static void schedule_all(); }; diff --git a/rpcs3/Emu/Cell/lv2/sys_timer.cpp b/rpcs3/Emu/Cell/lv2/sys_timer.cpp index 99aafe9c19..8a1f54c26c 100644 --- a/rpcs3/Emu/Cell/lv2/sys_timer.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_timer.cpp @@ -60,6 +60,8 @@ u64 lv2_timer::check() // If aborting, perform the last accurate check for event if (_now >= next) { + lv2_obj::notify_all_t notify; + std::lock_guard lock(mutex); if (next = expire; _now < next) diff --git a/rpcs3/Emu/Memory/vm.cpp b/rpcs3/Emu/Memory/vm.cpp index 86ed89c666..de626e8b7a 100644 --- a/rpcs3/Emu/Memory/vm.cpp +++ b/rpcs3/Emu/Memory/vm.cpp @@ -382,11 +382,16 @@ namespace vm void temporary_unlock(cpu_thread& cpu) noexcept { - if (!(cpu.state & cpu_flag::wait)) cpu.state += cpu_flag::wait; + bs_t add_state = cpu_flag::wait; if (g_tls_locked && g_tls_locked->compare_and_swap_test(&cpu, nullptr)) { - cpu.state += cpu_flag::memory; + add_state += cpu_flag::memory; + } + + if (add_state - cpu.state) + { + cpu.state += add_state; } } diff --git a/rpcs3/util/atomic.hpp b/rpcs3/util/atomic.hpp index 82559ede48..7e4712f683 100644 --- a/rpcs3/util/atomic.hpp +++ b/rpcs3/util/atomic.hpp @@ -315,10 +315,11 @@ private: friend class atomic_wait::list; static void wait(const void* data, u32 size, u128 old_value, u64 timeout, u128 mask, atomic_wait::info* ext = nullptr); + +public: static void notify_one(const void* data, u32 size, u128 mask128); static void notify_all(const void* data, u32 size, u128 mask128); -public: static void set_wait_callback(bool(*cb)(const void* data, u64 attempts, u64 stamp0)); static void set_notify_callback(void(*cb)(const void* data, u64 progress));