diff --git a/rpcs3/Emu/Cell/Modules/cellGem.cpp b/rpcs3/Emu/Cell/Modules/cellGem.cpp
index 47a70bb22a..f3251c1170 100644
--- a/rpcs3/Emu/Cell/Modules/cellGem.cpp
+++ b/rpcs3/Emu/Cell/Modules/cellGem.cpp
@@ -658,9 +658,9 @@ static void ds3_input_to_pad(const u32 port_no, be_t<u16>& digital_buttons, be_t
 			continue;
 		}
 
-		if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode))
+		if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode); btn.has_value() && btn.value())
 		{
-			switch (btn.value())
+			switch (btn.value()->btn_id())
 			{
 			case gem_btn::start:
 				digital_buttons |= CELL_GEM_CTRL_START;
diff --git a/rpcs3/Emu/Io/Buzz.cpp b/rpcs3/Emu/Io/Buzz.cpp
index 5eeaa6f4e4..ee917e427d 100644
--- a/rpcs3/Emu/Io/Buzz.cpp
+++ b/rpcs3/Emu/Io/Buzz.cpp
@@ -92,9 +92,9 @@ void usb_device_buzz::interrupt_transfer(u32 buf_size, u8* buf, u32 /*endpoint*/
 				continue;
 			}
 
-			if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode))
+			if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode); btn.has_value() && btn.value())
 			{
-				switch (btn.value())
+				switch (btn.value()->btn_id())
 				{
 				case buzz_btn::red:
 					buf[2 + (0 + 5 * index) / 8] |= 1 << ((0 + 5 * index) % 8); // Red
diff --git a/rpcs3/Emu/Io/GHLtar.cpp b/rpcs3/Emu/Io/GHLtar.cpp
index a69a31dacf..14d6d74dd4 100644
--- a/rpcs3/Emu/Io/GHLtar.cpp
+++ b/rpcs3/Emu/Io/GHLtar.cpp
@@ -125,9 +125,9 @@ void usb_device_ghltar::interrupt_transfer(u32 buf_size, u8* buf, u32 /*endpoint
 			continue;
 		}
 
-		if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode))
+		if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode); btn.has_value() && btn.value())
 		{
-			switch (btn.value())
+			switch (btn.value()->btn_id())
 			{
 			case ghltar_btn::w1:
 				buf[0] += 0x01; // W1
diff --git a/rpcs3/Emu/Io/Turntable.cpp b/rpcs3/Emu/Io/Turntable.cpp
index c0cc19fabb..62bde59af4 100644
--- a/rpcs3/Emu/Io/Turntable.cpp
+++ b/rpcs3/Emu/Io/Turntable.cpp
@@ -134,9 +134,9 @@ void usb_device_turntable::interrupt_transfer(u32 buf_size, u8* buf, u32 /*endpo
 		if (!button.m_pressed)
 			continue;
 
-		if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode))
+		if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode); btn.has_value() && btn.value())
 		{
-			switch (btn.value())
+			switch (btn.value()->btn_id())
 			{
 			case turntable_btn::blue:
 				buf[0] |= 0x01;   // Square Button
diff --git a/rpcs3/Emu/Io/emulated_pad_config.h b/rpcs3/Emu/Io/emulated_pad_config.h
index a2ceb6e79d..a3e66981e4 100644
--- a/rpcs3/Emu/Io/emulated_pad_config.h
+++ b/rpcs3/Emu/Io/emulated_pad_config.h
@@ -32,13 +32,13 @@ struct emulated_pad_config : cfg::node
 {
 	using cfg::node::node;
 
-	std::map<u32, std::map<u32, T>> buttons;
+	std::map<u32, std::map<u32, const cfg_pad_btn<T>*>> button_map;
 
-	std::optional<T> find_button(u32 offset, u32 keycode) const
+	std::optional<const cfg_pad_btn<T>*> find_button(u32 offset, u32 keycode) const
 	{
-		if (const auto it = buttons.find(offset); it != buttons.cend())
+		if (const auto it = button_map.find(offset); it != button_map.cend())
 		{
-			if (const auto it2 = it->second.find(keycode); it2 != it->second.cend())
+			if (const auto it2 = it->second.find(keycode); it2 != it->second.cend() && it2->second)
 			{
 				return it2->second;
 			}
@@ -47,19 +47,19 @@ struct emulated_pad_config : cfg::node
 		return std::nullopt;
 	}
 
-	void set_button(const cfg_pad_btn<T>& pbtn, T bbtn)
+	void init_button(cfg_pad_btn<T>* pbtn)
 	{
-		const u32 offset = pad_button_offset(pbtn);
-		const u32 keycode = pad_button_keycode(pbtn);
-		buttons[(offset >> 8) & 0xFF][keycode & 0xFF] = bbtn;
+		if (!pbtn) return;
+		const u32 offset = pad_button_offset(pbtn->get());
+		const u32 keycode = pad_button_keycode(pbtn->get());
+		button_map[(offset >> 8) & 0xFF][keycode & 0xFF] = std::as_const(pbtn);
 	}
 
-	void set_buttons()
+	void init_buttons()
 	{
 		for (const auto& n : get_nodes())
 		{
-			const auto& node = static_cast<cfg_pad_btn<T>*>(n);
-			set_button(*node, node->btn_id());
+			init_button(static_cast<cfg_pad_btn<T>*>(n));
 		}
 	}
 };
@@ -108,7 +108,7 @@ struct emulated_pads_config : cfg::node
 	
 		for (T* player : players)
 		{
-			player->set_buttons();
+			player->init_buttons();
 		}
 	
 		return result;
diff --git a/rpcs3/Emu/Io/usio.cpp b/rpcs3/Emu/Io/usio.cpp
index fe329b7e51..42acff76cf 100644
--- a/rpcs3/Emu/Io/usio.cpp
+++ b/rpcs3/Emu/Io/usio.cpp
@@ -173,9 +173,9 @@ void usb_device_usio::translate_input()
 
 		for (const Button& button : pad->m_buttons)
 		{
-			if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode))
+			if (const auto btn = cfg->find_button(button.m_offset, button.m_outKeyCode); btn.has_value() && btn.value())
 			{
-				switch (btn.value())
+				switch (btn.value()->btn_id())
 				{
 				case usio_btn::test:
 					if (player != 0) break;