Decouple audio processing and run at variable rate

Currently, processing of audio samples is called from AudioRenderer's Update method, using a fixed 4 buffers to process the given samples. Games call Update at variable rates, depending on framerate and/or sample count, which causes inconsistency in audio processing. From what I've seen, 60 FPS games update every ~0.004s, but 30 FPS/160 sample games update somewhere between 0.02 and 0.04, 5-10x slower. Not enough samples get fed to the backend, leading to a lot of audio skipping. This PR seeks to address this by de-coupling the audio consumption and the audio update. Update remains the same without calling for buffer queuing, and the consume now schedules itself to run based on the sample rate and count.
author: Kelebek1 <eeeedddccc@hotmail.co.uk> 2021-06-20 15:23:16 +0100
committer: Kelebek1 <eeeedddccc@hotmail.co.uk> 2021-06-27 15:58:07 +0100
commit: 0857d6a3dbcaeb30b51326419fb56d9b543601f1 (patch)
tree: 26221ce096533b188aa1fe8d7e82290780edee1d
parent: 432fab7c4f7c8664b2f9319fd6ac6cb8960407e4 (diff)
3 files changed, 115 insertions, 79 deletions
diff --git a/src/audio_core/audio_renderer.cpp b/src/audio_core/audio_renderer.cpp
index 80ffddb104..0757cd8040 100644
--- a/src/audio_core/audio_renderer.cpp
+++ b/src/audio_core/audio_renderer.cpp
@@ -12,6 +12,7 @@
 #include "audio_core/voice_context.h"
 #include "common/logging/log.h"
 #include "common/settings.h"
+#include "core/core_timing.h"
 #include "core/memory.h"
 
 namespace {
@@ -68,7 +69,9 @@ namespace {
 } // namespace
 
 namespace AudioCore {
-AudioRenderer::AudioRenderer(Core::Timing::CoreTiming& core_timing, Core::Memory::Memory& memory_,
+constexpr s32 NUM_BUFFERS = 2;
+
+AudioRenderer::AudioRenderer(Core::Timing::CoreTiming& core_timing_, Core::Memory::Memory& memory_,
                              AudioCommon::AudioRendererParameter params,
                              Stream::ReleaseCallback&& release_callback,
                              std::size_t instance_number)
@@ -77,7 +80,8 @@ AudioRenderer::AudioRenderer(Core::Timing::CoreTiming& core_timing, Core::Memory
       sink_context(params.sink_count), splitter_context(),
       voices(params.voice_count), memory{memory_},
       command_generator(worker_params, voice_context, mix_context, splitter_context, effect_context,
-                        memory) {
+                        memory),
+      core_timing{core_timing_} {
     behavior_info.SetUserRevision(params.revision);
     splitter_context.Initialize(behavior_info, params.splitter_count,
                                 params.num_splitter_send_channels);
@@ -86,16 +90,27 @@ AudioRenderer::AudioRenderer(Core::Timing::CoreTiming& core_timing, Core::Memory
     stream = audio_out->OpenStream(
         core_timing, params.sample_rate, AudioCommon::STREAM_NUM_CHANNELS,
         fmt::format("AudioRenderer-Instance{}", instance_number), std::move(release_callback));
-    audio_out->StartStream(stream);
-
-    QueueMixedBuffer(0);
-    QueueMixedBuffer(1);
-    QueueMixedBuffer(2);
-    QueueMixedBuffer(3);
+    process_event = Core::Timing::CreateEvent(
+        fmt::format("AudioRenderer-Instance{}-Process", instance_number),
+        [this](std::uintptr_t, std::chrono::nanoseconds) { ReleaseAndQueueBuffers(); });
+    for (s32 i = 0; i < NUM_BUFFERS; ++i) {
+        QueueMixedBuffer(i);
+    }
 }
 
 AudioRenderer::~AudioRenderer() = default;
 
+ResultCode AudioRenderer::Start() {
+    audio_out->StartStream(stream);
+    ReleaseAndQueueBuffers();
+    return ResultSuccess;
+}
+
+ResultCode AudioRenderer::Stop() {
+    audio_out->StopStream(stream);
+    return ResultSuccess;
+}
+
 u32 AudioRenderer::GetSampleRate() const {
     return worker_params.sample_rate;
 }
@@ -114,89 +129,88 @@ Stream::State AudioRenderer::GetStreamState() const {
 
 ResultCode AudioRenderer::UpdateAudioRenderer(const std::vector<u8>& input_params,
                                               std::vector<u8>& output_params) {
+    {
+        std::scoped_lock lock{mutex};
+        InfoUpdater info_updater{input_params, output_params, behavior_info};
 
-    InfoUpdater info_updater{input_params, output_params, behavior_info};
-
-    if (!info_updater.UpdateBehaviorInfo(behavior_info)) {
-        LOG_ERROR(Audio, "Failed to update behavior info input parameters");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
-
-    if (!info_updater.UpdateMemoryPools(memory_pool_info)) {
-        LOG_ERROR(Audio, "Failed to update memory pool parameters");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
-
-    if (!info_updater.UpdateVoiceChannelResources(voice_context)) {
-        LOG_ERROR(Audio, "Failed to update voice channel resource parameters");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
+        if (!info_updater.UpdateBehaviorInfo(behavior_info)) {
+            LOG_ERROR(Audio, "Failed to update behavior info input parameters");
+            return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+        }
 
-    if (!info_updater.UpdateVoices(voice_context, memory_pool_info, 0)) {
-        LOG_ERROR(Audio, "Failed to update voice parameters");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
+        if (!info_updater.UpdateMemoryPools(memory_pool_info)) {
+            LOG_ERROR(Audio, "Failed to update memory pool parameters");
+            return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+        }
 
-    // TODO(ogniK): Deal with stopped audio renderer but updates still taking place
-    if (!info_updater.UpdateEffects(effect_context, true)) {
-        LOG_ERROR(Audio, "Failed to update effect parameters");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
+        if (!info_updater.UpdateVoiceChannelResources(voice_context)) {
+            LOG_ERROR(Audio, "Failed to update voice channel resource parameters");
+            return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+        }
 
-    if (behavior_info.IsSplitterSupported()) {
-        if (!info_updater.UpdateSplitterInfo(splitter_context)) {
-            LOG_ERROR(Audio, "Failed to update splitter parameters");
+        if (!info_updater.UpdateVoices(voice_context, memory_pool_info, 0)) {
+            LOG_ERROR(Audio, "Failed to update voice parameters");
             return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
         }
-    }
 
-    const auto mix_result = info_updater.UpdateMixes(mix_context, worker_params.mix_buffer_count,
-                                                     splitter_context, effect_context);
+        // TODO(ogniK): Deal with stopped audio renderer but updates still taking place
+        if (!info_updater.UpdateEffects(effect_context, true)) {
+            LOG_ERROR(Audio, "Failed to update effect parameters");
+            return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+        }
 
-    if (mix_result.IsError()) {
-        LOG_ERROR(Audio, "Failed to update mix parameters");
-        return mix_result;
-    }
+        if (behavior_info.IsSplitterSupported()) {
+            if (!info_updater.UpdateSplitterInfo(splitter_context)) {
+                LOG_ERROR(Audio, "Failed to update splitter parameters");
+                return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+            }
+        }
 
-    // TODO(ogniK): Sinks
-    if (!info_updater.UpdateSinks(sink_context)) {
-        LOG_ERROR(Audio, "Failed to update sink parameters");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
+        const auto mix_result = info_updater.UpdateMixes(
+            mix_context, worker_params.mix_buffer_count, splitter_context, effect_context);
 
-    // TODO(ogniK): Performance buffer
-    if (!info_updater.UpdatePerformanceBuffer()) {
-        LOG_ERROR(Audio, "Failed to update performance buffer parameters");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
+        if (mix_result.IsError()) {
+            LOG_ERROR(Audio, "Failed to update mix parameters");
+            return mix_result;
+        }
 
-    if (!info_updater.UpdateErrorInfo(behavior_info)) {
-        LOG_ERROR(Audio, "Failed to update error info");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
+        // TODO(ogniK): Sinks
+        if (!info_updater.UpdateSinks(sink_context)) {
+            LOG_ERROR(Audio, "Failed to update sink parameters");
+            return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+        }
 
-    if (behavior_info.IsElapsedFrameCountSupported()) {
-        if (!info_updater.UpdateRendererInfo(elapsed_frame_count)) {
-            LOG_ERROR(Audio, "Failed to update renderer info");
+        // TODO(ogniK): Performance buffer
+        if (!info_updater.UpdatePerformanceBuffer()) {
+            LOG_ERROR(Audio, "Failed to update performance buffer parameters");
             return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
         }
-    }
-    // TODO(ogniK): Statistics
 
-    if (!info_updater.WriteOutputHeader()) {
-        LOG_ERROR(Audio, "Failed to write output header");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
+        if (!info_updater.UpdateErrorInfo(behavior_info)) {
+            LOG_ERROR(Audio, "Failed to update error info");
+            return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+        }
 
-    // TODO(ogniK): Check when all sections are implemented
+        if (behavior_info.IsElapsedFrameCountSupported()) {
+            if (!info_updater.UpdateRendererInfo(elapsed_frame_count)) {
+                LOG_ERROR(Audio, "Failed to update renderer info");
+                return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+            }
+        }
+        // TODO(ogniK): Statistics
 
-    if (!info_updater.CheckConsumedSize()) {
-        LOG_ERROR(Audio, "Audio buffers were not consumed!");
-        return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
-    }
+        if (!info_updater.WriteOutputHeader()) {
+            LOG_ERROR(Audio, "Failed to write output header");
+            return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+        }
 
-    ReleaseAndQueueBuffers();
+        // TODO(ogniK): Check when all sections are implemented
 
+        if (!info_updater.CheckConsumedSize()) {
+            LOG_ERROR(Audio, "Audio buffers were not consumed!");
+            return AudioCommon::Audren::ERR_INVALID_PARAMETERS;
+        }
+    }
     return ResultSuccess;
 }
 
@@ -315,10 +329,24 @@ void AudioRenderer::QueueMixedBuffer(Buffer::Tag tag) {
 }
 
 void AudioRenderer::ReleaseAndQueueBuffers() {
-    const auto released_buffers{audio_out->GetTagsAndReleaseBuffers(stream)};
-    for (const auto& tag : released_buffers) {
-        QueueMixedBuffer(tag);
+    if (!stream->IsPlaying()) {
+        return;
+    }
+
+    {
+        std::scoped_lock lock{mutex};
+        const auto released_buffers{audio_out->GetTagsAndReleaseBuffers(stream)};
+        for (const auto& tag : released_buffers) {
+            QueueMixedBuffer(tag);
+        }
     }
+
+    const f32 sample_rate = static_cast<f32>(GetSampleRate());
+    const f32 sample_count = static_cast<f32>(GetSampleCount());
+    const f32 consume_rate = sample_rate / (sample_count * (sample_count / 240));
+    const s32 ms = (1000 / static_cast<s32>(consume_rate)) - 1;
+    const std::chrono::milliseconds next_event_time(std::max(ms / NUM_BUFFERS, 1));
+    core_timing.ScheduleEvent(next_event_time, process_event, {});
 }
 
 } // namespace AudioCore
diff --git a/src/audio_core/audio_renderer.h b/src/audio_core/audio_renderer.h
index 18567f6188..88fdd13dda 100644
--- a/src/audio_core/audio_renderer.h
+++ b/src/audio_core/audio_renderer.h
@@ -6,6 +6,7 @@
 
 #include <array>
 #include <memory>
+#include <mutex>
 #include <vector>
 
 #include "audio_core/behavior_info.h"
@@ -45,6 +46,8 @@ public:
 
     [[nodiscard]] ResultCode UpdateAudioRenderer(const std::vector<u8>& input_params,
                                                  std::vector<u8>& output_params);
+    [[nodiscard]] ResultCode Start();
+    [[nodiscard]] ResultCode Stop();
     void QueueMixedBuffer(Buffer::Tag tag);
     void ReleaseAndQueueBuffers();
     [[nodiscard]] u32 GetSampleRate() const;
@@ -68,6 +71,9 @@ private:
     Core::Memory::Memory& memory;
     CommandGenerator command_generator;
     std::size_t elapsed_frame_count{};
+    Core::Timing::CoreTiming& core_timing;
+    std::shared_ptr<Core::Timing::EventType> process_event;
+    std::mutex mutex;
 };
 
 } // namespace AudioCore
diff --git a/src/core/hle/service/audio/audren_u.cpp b/src/core/hle/service/audio/audren_u.cpp
index 800feba6e8..feb5150e16 100644
--- a/src/core/hle/service/audio/audren_u.cpp
+++ b/src/core/hle/service/audio/audren_u.cpp
@@ -110,17 +110,19 @@ private:
     void Start(Kernel::HLERequestContext& ctx) {
         LOG_WARNING(Service_Audio, "(STUBBED) called");
 
-        IPC::ResponseBuilder rb{ctx, 2};
+        const auto result = renderer->Start();
 
-        rb.Push(ResultSuccess);
+        IPC::ResponseBuilder rb{ctx, 2};
+        rb.Push(result);
     }
 
     void Stop(Kernel::HLERequestContext& ctx) {
         LOG_WARNING(Service_Audio, "(STUBBED) called");
 
-        IPC::ResponseBuilder rb{ctx, 2};
+        const auto result = renderer->Stop();
 
-        rb.Push(ResultSuccess);
+        IPC::ResponseBuilder rb{ctx, 2};
+        rb.Push(result);
     }
 
     void QuerySystemEvent(Kernel::HLERequestContext& ctx) {
author	Kelebek1 <eeeedddccc@hotmail.co.uk>	2021-06-20 15:23:16 +0100
committer	Kelebek1 <eeeedddccc@hotmail.co.uk>	2021-06-27 15:58:07 +0100
commit	0857d6a3dbcaeb30b51326419fb56d9b543601f1 (patch)
tree	26221ce096533b188aa1fe8d7e82290780edee1d
parent	432fab7c4f7c8664b2f9319fd6ac6cb8960407e4 (diff)