RPCS3/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2024-11-22 10:42:36 +01:00
Code Issues Releases Wiki Activity

cellRec: Don't present frames/samples if external audio/video is used

Browse Source
This commit is contained in:
Megamouse 2023-11-14 21:51:42 +01:00
parent 5fa77b04ea
commit 4c14290694
6 changed files with 97 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
rpcs3/Emu/Cell/Modules/cellRec.cpp
View File

@ -136,6 +136,23 @@ struct rec_param
video_input, audio_input, audio_input_mix_vol, reduce_memsize, show_xmb, filename, metadata_filename, spurs_param.pSpurs, spurs_param.spu_usage_rate,
priority, movie_metadata.to_string(), scene_metadata.to_string());
}
bool use_external_audio() const
{
return audio_input != CELL_REC_PARAM_AUDIO_INPUT_DISABLE // != DISABLE means that cellRec will add samples on its own
&& audio_input_mix_vol > CELL_REC_PARAM_AUDIO_INPUT_MIX_VOL_MIN; // We need to mix cellRec audio with internal audio
}
bool use_internal_audio() const
{
return audio_input == CELL_REC_PARAM_AUDIO_INPUT_DISABLE // DISABLE means that cellRec won't add samples on its own
|| audio_input_mix_vol < CELL_REC_PARAM_AUDIO_INPUT_MIX_VOL_MAX; // We need to mix cellRec audio with internal audio
}
bool use_internal_video() const
{
return video_input == CELL_REC_PARAM_VIDEO_INPUT_DISABLE; // DISABLE means that cellRec won't add frames on its own
}
};
constexpr u32 rec_framerate = 30; // Always 30 fps
@ -146,7 +163,7 @@ public:
rec_video_sink() : utils::video_sink()
{
m_framerate = rec_framerate;
m_sample_rate = 44100; // TODO
m_sample_rate = 48000; // TODO
}
void stop(bool flush = true) override
@ -173,6 +190,20 @@ public:
return {};
}
encoder_sample get_sample()
{
std::lock_guard lock(m_mtx);
if (!m_samples_to_encode.empty())
{
encoder_sample block = std::move(m_samples_to_encode.front());
m_samples_to_encode.pop_front();
return block;
}
return {};
}
};
struct rec_info
@ -201,11 +232,12 @@ struct rec_info
return pos;
}
std::shared_ptr<rec_video_sink> video_sink;
std::shared_ptr<rec_video_sink> ringbuffer_sink;
std::shared_ptr<utils::video_encoder> encoder;
std::unique_ptr<named_thread<std::function<void()>>> video_provider_thread;
atomic_t<bool> paused = false;
s64 last_pts = -1;
s64 last_video_pts = -1;
s64 last_audio_pts = -1;
// Video parameters
utils::video_encoder::frame_format output_format{};
@ -523,13 +555,13 @@ void rec_info::start_video_provider()
video_provider_thread = std::make_unique<named_thread<std::function<void()>>>("cellRec video provider", [this]()
{
const bool use_internal_audio = param.audio_input == CELL_REC_PARAM_AUDIO_INPUT_DISABLE || param.audio_input_mix_vol < 100;
const bool use_external_audio = param.audio_input != CELL_REC_PARAM_AUDIO_INPUT_DISABLE && param.audio_input_mix_vol > 0;
const bool use_external_video = param.video_input != CELL_REC_PARAM_VIDEO_INPUT_DISABLE;
const bool use_internal_audio = param.use_internal_audio();
const bool use_external_audio = param.use_external_audio();
const bool use_external_video = !param.use_internal_video();
const bool use_ring_buffer = param.ring_sec > 0;
const usz frame_size = input_format.pitch * input_format.height;
cellRec.notice("video_provider_thread: use_ring_buffer=%d, video_ringbuffer_size=%d, audio_ringbuffer_size=%d, ring_sec=%d, frame_size=%d, use_external_video=%d, use_external_audio=%d, use_internal_audio=%d", use_ring_buffer, video_ringbuffer.size(), audio_ringbuffer.size(), param.ring_sec, frame_size, use_external_video, use_external_audio, use_internal_audio);
cellRec.notice("video_provider_thread: use_ring_buffer=%d, video_ringbuffer_size=%d, audio_ringbuffer_size=%d, ring_sec=%d, frame_size=%d, use_internal_video=%d, use_external_audio=%d, use_internal_audio=%d", use_ring_buffer, video_ringbuffer.size(), audio_ringbuffer.size(), param.ring_sec, frame_size, encoder->use_internal_video, use_external_audio, encoder->use_internal_audio);
while (thread_ctrl::state() != thread_state::aborting && encoder)
{
@ -555,18 +587,25 @@ void rec_info::start_video_provider()
continue;
}
const usz timestamp_ms = (get_system_time() - recording_time_start - pause_time_total) / 1000;
// We only care for new video frames or audio samples that can be properly encoded, so we check the timestamps and pts.
const usz timestamp_us = get_system_time() - recording_time_start - pause_time_total;
const usz timestamp_ms = timestamp_us / 1000;
/////////////////
// VIDEO //
/////////////////
// We only care for new video frames that can be properly encoded
// TODO: wait for flip before adding a frame
if (use_external_video)
{
if (const s64 pts = encoder->get_pts(timestamp_ms); pts > last_pts)
// The video frames originate from cellRec instead of our render pipeline.
if (const s64 pts = encoder->get_pts(timestamp_ms); pts > last_video_pts)
{
if (video_input_buffer)
{
if (use_ring_buffer)
{
// The video frames originate from cellRec and are stored in a ringbuffer.
utils::video_sink::encoder_frame& frame_data = video_ringbuffer[next_video_ring_pos()];
frame_data.pts = pts;
frame_data.width = input_format.width;
@ -578,29 +617,35 @@ void rec_info::start_video_provider()
}
else
{
// The video frames originate from cellRec and are pushed to the encoder immediately.
std::vector<u8> frame(frame_size);
std::memcpy(frame.data(), video_input_buffer.get_ptr(), frame.size());
encoder->add_frame(frame, input_format.pitch, input_format.width, input_format.height, input_format.av_pixel_format, timestamp_ms);
}
}
last_pts = pts;
last_video_pts = pts;
}
}
else if (use_ring_buffer && video_sink)
else if (use_ring_buffer && ringbuffer_sink)
{
utils::video_sink::encoder_frame frame = video_sink->get_frame();
// The video frames originate from our render pipeline and are stored in a ringbuffer.
utils::video_sink::encoder_frame frame = ringbuffer_sink->get_frame();
if (const s64 pts = encoder->get_pts(frame.timestamp_ms); pts > last_pts && frame.data.size() > 0)
if (const s64 pts = encoder->get_pts(frame.timestamp_ms); pts > last_video_pts && frame.data.size() > 0)
{
ensure(frame.data.size() == frame_size);
utils::video_sink::encoder_frame& frame_data = video_ringbuffer[next_video_ring_pos()];
frame_data = std::move(frame);
frame_data.pts = pts;
last_pts = pts;
last_video_pts = pts;
video_ring_frame_count++;
}
}
//else
//{
// The video frames originate from our render pipeline and are directly encoded by the encoder video sink itself.
//}
if (use_internal_audio)
{
@ -657,6 +702,9 @@ void rec_info::stop_video_provider(bool flush)
video_provider_thread.reset();
}
// Flush the ringbuffer if necessary.
// This should only happen if the video sink is not the encoder itself.
// In this case the encoder should have been idle until now.
if (flush && param.ring_sec > 0 && !video_ringbuffer.empty())
{
cellRec.notice("Flushing video ringbuffer.");
@ -919,11 +967,11 @@ error_code cellRecOpen(vm::cptr<char> pDirName, vm::cptr<char> pFileName, vm::cp
if (opt.value.audio_input == CELL_REC_PARAM_AUDIO_INPUT_DISABLE)
{
rec.param.audio_input_mix_vol = 0;
rec.param.audio_input_mix_vol = CELL_REC_PARAM_AUDIO_INPUT_MIX_VOL_MIN;
}
else
{
rec.param.audio_input_mix_vol = 100;
rec.param.audio_input_mix_vol = CELL_REC_PARAM_AUDIO_INPUT_MIX_VOL_MAX;
}
break;
}
@ -1043,7 +1091,7 @@ error_code cellRecOpen(vm::cptr<char> pDirName, vm::cptr<char> pFileName, vm::cp
rec.cb = cb;
rec.cbUserData = cbUserData;
rec.last_pts = -1;
rec.last_video_pts = -1;
rec.audio_ringbuffer.clear();
rec.video_ringbuffer.clear();
rec.video_ring_frame_count = 0;
@ -1065,10 +1113,15 @@ error_code cellRecOpen(vm::cptr<char> pDirName, vm::cptr<char> pFileName, vm::cp
rec.audio_ringbuffer.resize(audio_ring_buffer_size);
rec.audio_ring_step = audio_size_per_sample;
rec.video_ringbuffer.resize(video_ring_buffer_size, {});
rec.video_sink = std::make_shared<rec_video_sink>();
rec.ringbuffer_sink = std::make_shared<rec_video_sink>();
rec.ringbuffer_sink->use_internal_audio = rec.param.use_internal_audio();
rec.ringbuffer_sink->use_internal_video = rec.param.use_internal_video();
}
rec.encoder = std::make_shared<utils::video_encoder>();
rec.encoder->use_internal_audio = rec.param.use_internal_audio() && !rec.ringbuffer_sink;
rec.encoder->use_internal_video = rec.param.use_internal_video() && !rec.ringbuffer_sink;
rec.encoder->set_path(vfs::get(rec.param.filename));
rec.encoder->set_framerate(rec.fps);
rec.encoder->set_video_bitrate(rec.video_bps);
@ -1110,9 +1163,9 @@ error_code cellRecClose(s32 isDiscard)
rec.stop_video_provider(false);
rec.encoder->stop(false);
if (rec.video_sink)
if (rec.ringbuffer_sink)
{
rec.video_sink->stop(false);
rec.ringbuffer_sink->stop(false);
}
if (fs::is_file(rec.param.filename))
@ -1132,9 +1185,9 @@ error_code cellRecClose(s32 isDiscard)
rec.encoder->stop(true);
rec.recording_time_total = rec.encoder->get_timestamp_ms(rec.encoder->last_video_pts());
if (rec.video_sink)
if (rec.ringbuffer_sink)
{
rec.video_sink->stop(true);
rec.ringbuffer_sink->stop(true);
}
const s64 start_pts = rec.encoder->get_pts(rec.param.scene_metadata.start_time);
@ -1151,7 +1204,7 @@ error_code cellRecClose(s32 isDiscard)
utils::video_provider& video_provider = g_fxo->get<utils::video_provider>();
// Release the video sink if it was used
if (rec.param.video_input == CELL_REC_PARAM_VIDEO_INPUT_DISABLE)
if (rec.param.use_internal_video() || rec.param.use_internal_audio())
{
const recording_mode old_mode = g_recording_mode.exchange(recording_mode::stopped);
@ -1168,7 +1221,7 @@ error_code cellRecClose(s32 isDiscard)
rec.param = {};
rec.encoder.reset();
rec.video_sink.reset();
rec.ringbuffer_sink.reset();
rec.audio_ringbuffer.clear();
rec.video_ringbuffer.clear();
rec.state = rec_state::closed;
@ -1201,7 +1254,7 @@ error_code cellRecStop()
sysutil_register_cb([&rec](ppu_thread& ppu) -> s32
{
// Disable video sink if it was used
if (rec.param.video_input == CELL_REC_PARAM_VIDEO_INPUT_DISABLE)
if (rec.param.use_internal_video() || rec.param.use_internal_audio())
{
const recording_mode old_mode = g_recording_mode.exchange(recording_mode::stopped);
@ -1248,12 +1301,12 @@ error_code cellRecStart()
utils::video_provider& video_provider = g_fxo->get<utils::video_provider>();
// Setup an video sink if it is needed
if (rec.param.video_input == CELL_REC_PARAM_VIDEO_INPUT_DISABLE)
if (rec.param.use_internal_video() || rec.param.use_internal_audio())
{
if (rec.param.ring_sec <= 0)
if (rec.ringbuffer_sink)
{
// Regular recording
if (!video_provider.set_video_sink(rec.encoder, recording_mode::cell))
// Ringbuffer recording. Use cellRec's custom video sink. The encoder will stay idle until we flush the ringbuffer.
if (!video_provider.set_video_sink(rec.ringbuffer_sink, recording_mode::cell))
{
cellRec.error("Failed to set video sink");
rec.cb(ppu, CELL_REC_STATUS_ERR, CELL_REC_ERROR_FATAL, rec.cbUserData);
@ -1262,8 +1315,8 @@ error_code cellRecStart()
}
else
{
// Ringbuffer recording
if (!video_provider.set_video_sink(rec.video_sink, recording_mode::cell))
// Regular recording. The encoder also acts as sink, so it will encode frames and samples as soon as we add them to the sink.
if (!video_provider.set_video_sink(rec.encoder, recording_mode::cell))
{
cellRec.error("Failed to set video sink");
rec.cb(ppu, CELL_REC_STATUS_ERR, CELL_REC_ERROR_FATAL, rec.cbUserData);

2
rpcs3/rpcs3qt/gs_frame.cpp
View File

@ -495,6 +495,8 @@ void gs_frame::toggle_recording()
output_format.height = g_cfg_recording.video.height;
output_format.pitch = g_cfg_recording.video.width * 4;
m_video_encoder->use_internal_audio = true;
m_video_encoder->use_internal_video = true;
m_video_encoder->set_path(video_path);
m_video_encoder->set_framerate(g_cfg_recording.video.framerate);
m_video_encoder->set_video_bitrate(g_cfg_recording.video.video_bps);

1
rpcs3/rpcs3qt/gs_frame.h
View File

@ -5,7 +5,6 @@
#include "util/types.hpp"
#include "util/atomic.hpp"
#include "util/media_utils.h"
#include "util/video_provider.h"
#include "Emu/RSX/GSFrameBase.h"
#include <QWindow>

10
rpcs3/util/video_provider.cpp
View File

@ -84,7 +84,7 @@ namespace utils
{
std::lock_guard lock(m_mutex);
if (!m_video_sink)
if (!m_video_sink || !m_video_sink->use_internal_video)
return false;
const usz timestamp_ms = std::chrono::duration_cast<std::chrono::milliseconds>(steady_clock::now() - m_encoder_start).count() - m_pause_time_ms;
@ -92,7 +92,7 @@ namespace utils
return pts > m_last_video_pts_incoming;
}
recording_mode video_provider::check_state()
recording_mode video_provider::check_mode()
{
if (!m_video_sink || m_video_sink->has_error)
{
@ -125,7 +125,7 @@ namespace utils
{
std::lock_guard lock(m_mutex);
if (check_state() == recording_mode::stopped)
if (check_mode() == recording_mode::stopped)
{
return;
}
@ -149,7 +149,7 @@ namespace utils
{
std::lock_guard lock(m_mutex);
if (!m_video_sink)
if (!m_video_sink || !m_video_sink->use_internal_audio)
return false;
const usz timestamp_us = std::chrono::duration_cast<std::chrono::microseconds>(steady_clock::now() - m_encoder_start).count() - (m_pause_time_ms * 1000ull);
@ -166,7 +166,7 @@ namespace utils
std::lock_guard lock(m_mutex);
if (check_state() == recording_mode::stopped)
if (check_mode() == recording_mode::stopped)
{
return;
}

2
rpcs3/util/video_provider.h
View File

@ -27,7 +27,7 @@ namespace utils
void present_samples(u8* buf, u32 sample_count, u16 channels);
private:
recording_mode check_state();
recording_mode check_mode();
recording_mode m_type = recording_mode::stopped;
std::shared_ptr<video_sink> m_video_sink;

4
rpcs3/util/video_sink.h
View File

@ -91,6 +91,10 @@ namespace utils
std::vector<u8> data;
};
// These two variables should only be set once before we start encoding, so we don't need mutexes or atomics.
bool use_internal_audio = false; // True if we want to fetch samples from cellAudio
bool use_internal_video = false; // True if we want to fetch frames from rsx
protected:
shared_mutex m_mtx;
std::deque<encoder_frame> m_frames_to_encode;