manage decoders with ssrc updates

src/abaddon.cpp

@@ -50,6 +50,16 @@ Abaddon::Abaddon()
     m_discord.signal_thread_update().connect(sigc::mem_fun(*this, &Abaddon::DiscordOnThreadUpdate));
     m_discord.signal_message_sent().connect(sigc::mem_fun(*this, &Abaddon::DiscordOnMessageSent));
     m_discord.signal_disconnected().connect(sigc::mem_fun(*this, &Abaddon::DiscordOnDisconnect));
+
+#ifdef WITH_VOICE
+    m_discord.signal_voice_connected().connect(sigc::mem_fun(*this, &Abaddon::OnVoiceConnected));
+    m_discord.signal_voice_disconnected().connect(sigc::mem_fun(*this, &Abaddon::OnVoiceDisconnected));
+    m_discord.signal_voice_speaking().connect([this](const VoiceSpeakingData &m) {
+        printf("%llu has ssrc %u\n", (uint64_t)m.UserID, m.SSRC);
+        m_audio->AddSSRC(m.SSRC);
+    });
+#endif
+
     m_discord.signal_channel_accessibility_changed().connect([this](Snowflake id, bool accessible) {
         if (!accessible)
             m_channels_requested.erase(id);
@@ -406,6 +416,15 @@ void Abaddon::DiscordOnThreadUpdate(const ThreadUpdateData &data) {
     }
 }
 
+#ifdef WITH_VOICE
+void Abaddon::OnVoiceConnected() {
+}
+
+void Abaddon::OnVoiceDisconnected() {
+    m_audio->RemoveAllSSRCs();
+}
+#endif
+
 SettingsManager::Settings &Abaddon::GetSettings() {
     return m_settings.GetSettings();
 }

src/abaddon.hpp

@@ -89,6 +89,11 @@ public:
     void DiscordOnDisconnect(bool is_reconnecting, GatewayCloseCode close_code);
     void DiscordOnThreadUpdate(const ThreadUpdateData &data);
 
+#ifdef WITH_VOICE
+    void OnVoiceConnected();
+    void OnVoiceDisconnected();
+#endif
+
     SettingsManager::Settings &GetSettings();
 
     Glib::RefPtr<Gtk::CssProvider> GetStyleProvider();

src/audio/manager.cpp

@@ -107,8 +107,28 @@ AudioManager::AudioManager() {
 AudioManager::~AudioManager() {
     ma_device_uninit(&m_device);
     ma_device_uninit(&m_capture_device);
-    for (auto &[ssrc, pair] : m_sources) {
-        opus_decoder_destroy(pair.second);
+    RemoveAllSSRCs();
+}
+
+void AudioManager::AddSSRC(uint32_t ssrc) {
+    int error;
+    auto *decoder = opus_decoder_create(48000, 2, &error);
+    m_sources.insert(std::make_pair(ssrc, std::make_pair(std::deque<int16_t> {}, decoder)));
+}
+
+void AudioManager::RemoveSSRC(uint32_t ssrc) {
+    if (auto it = m_sources.find(ssrc); it != m_sources.end()) {
+        opus_decoder_destroy(it->second.second);
+        m_sources.erase(it);
+    }
+}
+
+void AudioManager::RemoveAllSSRCs() {
+    puts("remove all ssrc");
+    for (auto it = m_sources.begin(); it != m_sources.end();) {
+        opus_decoder_destroy(it->second.second);
+        m_sources.erase(it);
+        it++;
     }
 }
 
@@ -120,18 +140,15 @@ void AudioManager::FeedMeOpus(uint32_t ssrc, const std::vector<uint8_t> &data) {
     size_t payload_size = 0;
     const auto *opus_encoded = StripRTPExtensionHeader(data.data(), static_cast<int>(data.size()), payload_size);
     static std::array<opus_int16, 120 * 48 * 2> pcm;
-    if (m_sources.find(ssrc) == m_sources.end()) {
-        int err;
-        auto *decoder = opus_decoder_create(48000, 2, &err);
-        m_sources.insert(std::make_pair(ssrc, std::make_pair(std::deque<int16_t> {}, decoder)));
-    }
-    int decoded = opus_decode(m_sources.at(ssrc).second, opus_encoded, static_cast<opus_int32>(payload_size), pcm.data(), 120 * 48, 0);
-    if (decoded <= 0) {
-    } else {
-        m_mutex.lock();
-        auto &buf = m_sources.at(ssrc).first;
-        buf.insert(buf.end(), pcm.begin(), pcm.begin() + decoded * 2);
-        m_mutex.unlock();
+    if (auto it = m_sources.find(ssrc); it != m_sources.end()) {
+        int decoded = opus_decode(it->second.second, opus_encoded, static_cast<opus_int32>(payload_size), pcm.data(), 120 * 48, 0);
+        if (decoded <= 0) {
+        } else {
+            m_mutex.lock();
+            auto &buf = it->second.first;
+            buf.insert(buf.end(), pcm.begin(), pcm.begin() + decoded * 2);
+            m_mutex.unlock();
+        }
     }
 }
 

src/audio/manager.hpp

@@ -18,6 +18,10 @@ public:
     AudioManager();
     ~AudioManager();
 
+    void AddSSRC(uint32_t ssrc);
+    void RemoveSSRC(uint32_t ssrc);
+    void RemoveAllSSRCs();
+
     void SetOpusBuffer(uint8_t *ptr);
     void FeedMeOpus(uint32_t ssrc, const std::vector<uint8_t> &data);
 

src/discord/discord.cpp

@@ -23,6 +23,14 @@ DiscordClient::DiscordClient(bool mem_store)
     m_websocket.signal_open().connect(sigc::mem_fun(*this, &DiscordClient::HandleSocketOpen));
     m_websocket.signal_close().connect(sigc::mem_fun(*this, &DiscordClient::HandleSocketClose));
 
+#ifdef WITH_VOICE
+    m_voice.signal_connected().connect(sigc::mem_fun(*this, &DiscordClient::OnVoiceConnected));
+    m_voice.signal_disconnected().connect(sigc::mem_fun(*this, &DiscordClient::OnVoiceDisconnected));
+    m_voice.signal_speaking().connect([this](const VoiceSpeakingData &data) {
+        m_signal_voice_speaking.emit(data);
+    });
+#endif
+
     LoadEventMap();
 }
 
@@ -2135,11 +2143,16 @@ void DiscordClient::HandleGatewayGuildMembersChunk(const GatewayMessage &msg) {
 
 #ifdef WITH_VOICE
 void DiscordClient::HandleGatewayVoiceStateUpdate(const GatewayMessage &msg) {
-    VoiceStateUpdateData data = msg.Data;
+    VoiceState data = msg.Data;
     if (data.UserID == m_user_data.ID) {
         printf("voice session id: %s\n", data.SessionID.c_str());
         m_voice.SetSessionID(data.SessionID);
     }
+    if (data.ChannelID.has_value()) {
+        SetVoiceState(data.UserID, *data.ChannelID);
+    } else {
+        ClearVoiceState(data.UserID);
+    }
 }
 
 void DiscordClient::HandleGatewayVoiceServerUpdate(const GatewayMessage &msg) {
@@ -2170,6 +2183,15 @@ void DiscordClient::HandleGatewayReadySupplemental(const GatewayMessage &msg) {
             m_user_to_status[p.UserID] = PresenceStatus::DND;
         m_signal_presence_update.emit(*user, m_user_to_status.at(p.UserID));
     }
+#ifdef WITH_VOICE
+    for (const auto &g : data.Guilds) {
+        for (const auto &s : g.VoiceStates) {
+            if (s.ChannelID.has_value()) {
+                SetVoiceState(s.UserID, *s.ChannelID);
+            }
+        }
+    }
+#endif
 }
 
 void DiscordClient::HandleGatewayReconnect(const GatewayMessage &msg) {
@@ -2602,6 +2624,29 @@ void DiscordClient::HandleReadyGuildSettings(const ReadyEventData &data) {
     }
 }
 
+#ifdef WITH_VOICE
+void DiscordClient::SetVoiceState(Snowflake user_id, Snowflake channel_id) {
+    m_voice_state_user_channel[user_id] = channel_id;
+    m_voice_state_channel_users[channel_id].insert(user_id);
+}
+
+void DiscordClient::ClearVoiceState(Snowflake user_id) {
+    if (const auto it = m_voice_state_user_channel.find(user_id); it != m_voice_state_user_channel.end()) {
+        m_voice_state_channel_users[it->second].erase(user_id);
+        // invalidated
+        m_voice_state_user_channel.erase(user_id);
+    }
+}
+
+void DiscordClient::OnVoiceConnected() {
+    m_signal_voice_connected.emit();
+}
+
+void DiscordClient::OnVoiceDisconnected() {
+    m_signal_voice_disconnected.emit();
+}
+#endif
+
 void DiscordClient::LoadEventMap() {
     m_event_map["READY"] = GatewayEvent::READY;
     m_event_map["MESSAGE_CREATE"] = GatewayEvent::MESSAGE_CREATE;
@@ -2858,3 +2903,15 @@ DiscordClient::type_signal_channel_accessibility_changed DiscordClient::signal_c
 DiscordClient::type_signal_message_send_fail DiscordClient::signal_message_send_fail() {
     return m_signal_message_send_fail;
 }
+
+DiscordClient::type_signal_voice_connected DiscordClient::signal_voice_connected() {
+    return m_signal_voice_connected;
+}
+
+DiscordClient::type_signal_voice_disconnected DiscordClient::signal_voice_disconnected() {
+    return m_signal_voice_disconnected;
+}
+
+DiscordClient::type_signal_voice_speaking DiscordClient::signal_voice_speaking() {
+    return m_signal_voice_speaking;
+}

src/discord/discord.hpp

@@ -339,6 +339,15 @@ private:
     DiscordVoiceClient m_voice;
 
     Snowflake m_voice_channel_id;
+    // todo sql i guess
+    std::unordered_map<Snowflake, Snowflake> m_voice_state_user_channel;
+    std::unordered_map<Snowflake, std::unordered_set<Snowflake>> m_voice_state_channel_users;
+
+    void SetVoiceState(Snowflake user_id, Snowflake channel_id);
+    void ClearVoiceState(Snowflake user_id);
+
+    void OnVoiceConnected();
+    void OnVoiceDisconnected();
 #endif
 
     mutable std::mutex m_msg_mutex;
@@ -413,6 +422,10 @@ public:
     typedef sigc::signal<void> type_signal_connected;
     typedef sigc::signal<void, std::string, float> type_signal_message_progress;
 
+    using type_signal_voice_connected = sigc::signal<void()>;
+    using type_signal_voice_disconnected = sigc::signal<void()>;
+    using type_signal_voice_speaking = sigc::signal<void(VoiceSpeakingData)>;
+
     type_signal_gateway_ready signal_gateway_ready();
     type_signal_message_create signal_message_create();
     type_signal_message_delete signal_message_delete();
@@ -467,6 +480,10 @@ public:
     type_signal_connected signal_connected();
     type_signal_message_progress signal_message_progress();
 
+    type_signal_voice_connected signal_voice_connected();
+    type_signal_voice_disconnected signal_voice_disconnected();
+    type_signal_voice_speaking signal_voice_speaking();
+
 protected:
     type_signal_gateway_ready m_signal_gateway_ready;
     type_signal_message_create m_signal_message_create;
@@ -521,4 +538,8 @@ protected:
     type_signal_disconnected m_signal_disconnected;
     type_signal_connected m_signal_connected;
     type_signal_message_progress m_signal_message_progress;
+
+    type_signal_voice_connected m_signal_voice_connected;
+    type_signal_voice_disconnected m_signal_voice_disconnected;
+    type_signal_voice_speaking m_signal_voice_speaking;
 };

src/discord/objects.cpp

@@ -233,6 +233,11 @@ void from_json(const nlohmann::json &j, SupplementalMergedPresencesData &m) {
     JS_D("friends", m.Friends);
 }
 
+void from_json(const nlohmann::json &j, SupplementalGuildEntry &m) {
+    JS_D("id", m.ID);
+    JS_D("voice_states", m.VoiceStates);
+}
+
 void from_json(const nlohmann::json &j, ReadySupplementalData &m) {
     JS_D("merged_presences", m.MergedPresences);
 }
@@ -658,14 +663,24 @@ void to_json(nlohmann::json &j, const VoiceStateUpdateMessage &m) {
     // j["d"]["preferred_region"] = m.PreferredRegion;
 }
 
-void from_json(const nlohmann::json &j, VoiceStateUpdateData &m) {
-    JS_ON("user_id", m.UserID);
-    JS_ON("session_id", m.SessionID);
-}
-
 void from_json(const nlohmann::json &j, VoiceServerUpdateData &m) {
     JS_D("token", m.Token);
     JS_D("guild_id", m.GuildID);
     JS_D("endpoint", m.Endpoint);
 }
 #endif
+
+void from_json(const nlohmann::json &j, VoiceState &m) {
+    JS_O("guild_id", m.GuildID);
+    JS_N("channel_id", m.ChannelID);
+    JS_D("deaf", m.IsDeafened);
+    JS_D("mute", m.IsMuted);
+    JS_D("self_deaf", m.IsSelfDeafened);
+    JS_D("self_mute", m.IsSelfMuted);
+    JS_D("self_video", m.IsSelfVideo);
+    JS_O("self_stream", m.IsSelfStream);
+    JS_D("suppress", m.IsSuppressed);
+    JS_D("user_id", m.UserID);
+    JS_N("member", m.Member);
+    JS_D("session_id", m.SessionID);
+}

src/discord/objects.hpp

@@ -354,8 +354,18 @@ struct SupplementalMergedPresencesData {
     friend void from_json(const nlohmann::json &j, SupplementalMergedPresencesData &m);
 };
 
+struct VoiceState;
+struct SupplementalGuildEntry {
+    // std::vector<?> EmbeddedActivities;
+    Snowflake ID;
+    std::vector<VoiceState> VoiceStates;
+
+    friend void from_json(const nlohmann::json &j, SupplementalGuildEntry &m);
+};
+
 struct ReadySupplementalData {
     SupplementalMergedPresencesData MergedPresences;
+    std::vector<SupplementalGuildEntry> Guilds;
 
     friend void from_json(const nlohmann::json &j, ReadySupplementalData &m);
 };
@@ -879,13 +889,6 @@ struct VoiceStateUpdateMessage {
     friend void to_json(nlohmann::json &j, const VoiceStateUpdateMessage &m);
 };
 
-struct VoiceStateUpdateData {
-    Snowflake UserID;
-    std::string SessionID;
-
-    friend void from_json(const nlohmann::json &j, VoiceStateUpdateData &m);
-};
-
 struct VoiceServerUpdateData {
     std::string Token;
     Snowflake GuildID;
@@ -894,3 +897,20 @@ struct VoiceServerUpdateData {
     friend void from_json(const nlohmann::json &j, VoiceServerUpdateData &m);
 };
 #endif
+
+struct VoiceState {
+    std::optional<Snowflake> ChannelID;
+    bool IsDeafened;
+    bool IsMuted;
+    std::optional<Snowflake> GuildID;
+    std::optional<GuildMember> Member;
+    bool IsSelfDeafened;
+    bool IsSelfMuted;
+    bool IsSelfVideo;
+    bool IsSelfStream = false;
+    std::string SessionID;
+    bool IsSuppressed;
+    Snowflake UserID;
+
+    friend void from_json(const nlohmann::json &j, VoiceState &m);
+};

src/discord/voiceclient.cpp

@@ -1,5 +1,5 @@
 #ifdef WITH_VOICE
-    // clang-format off
+// clang-format off
 #include "voiceclient.hpp"
 #include "json.hpp"
 #include <sodium.h>
@@ -186,6 +186,7 @@ DiscordVoiceClient::~DiscordVoiceClient() {
 
 void DiscordVoiceClient::Start() {
     m_ws.StartConnection("wss://" + m_endpoint + "/?v=7");
+    m_heartbeat_waiter.revive();
 }
 
 void DiscordVoiceClient::Stop() {
@@ -195,6 +196,7 @@ void DiscordVoiceClient::Stop() {
         m_heartbeat_waiter.kill();
         if (m_heartbeat_thread.joinable()) m_heartbeat_thread.join();
         m_connected = false;
+        m_signal_disconnected.emit();
     }
 }
 
@@ -235,6 +237,9 @@ void DiscordVoiceClient::OnGatewayMessage(const std::string &str) {
         case VoiceGatewayOp::SessionDescription: {
             HandleGatewaySessionDescription(msg);
         } break;
+        case VoiceGatewayOp::Speaking: {
+            HandleGatewaySpeaking(msg);
+        } break;
         default: break;
     }
 }
@@ -273,7 +278,7 @@ void DiscordVoiceClient::HandleGatewaySessionDescription(const VoiceGatewayMessa
     VoiceSpeakingMessage msg;
     msg.Delay = 0;
     msg.SSRC = m_ssrc;
-    msg.Speaking = VoiceSpeakingMessage::Microphone;
+    msg.Speaking = VoiceSpeakingType::Microphone;
     m_ws.Send(msg);
 
     m_secret_key = d.SecretKey;
@@ -286,6 +291,12 @@ void DiscordVoiceClient::HandleGatewaySessionDescription(const VoiceGatewayMessa
     m_udp.SendEncrypted({ 0xF8, 0xFF, 0xFE });
     m_udp.Run();
     m_connected = true;
+    m_signal_connected.emit();
+}
+
+void DiscordVoiceClient::HandleGatewaySpeaking(const VoiceGatewayMessage &m) {
+    VoiceSpeakingData data = m.Data;
+    m_signal_speaking.emit(data);
 }
 
 void DiscordVoiceClient::Identify() {
@@ -365,6 +376,18 @@ void DiscordVoiceClient::HeartbeatThread() {
     }
 }
 
+DiscordVoiceClient::type_signal_disconnected DiscordVoiceClient::signal_connected() {
+    return m_signal_connected;
+}
+
+DiscordVoiceClient::type_signal_disconnected DiscordVoiceClient::signal_disconnected() {
+    return m_signal_disconnected;
+}
+
+DiscordVoiceClient::type_signal_speaking DiscordVoiceClient::signal_speaking() {
+    return m_signal_speaking;
+}
+
 void from_json(const nlohmann::json &j, VoiceGatewayMessage &m) {
     JS_D("op", m.Opcode);
     m.Data = j.at("d");
@@ -431,4 +454,10 @@ void to_json(nlohmann::json &j, const VoiceSpeakingMessage &m) {
     j["d"]["delay"] = m.Delay;
     j["d"]["ssrc"] = m.SSRC;
 }
+
+void from_json(const nlohmann::json &j, VoiceSpeakingData &m) {
+    JS_D("user_id", m.UserID);
+    JS_D("ssrc", m.SSRC);
+    JS_D("speaking", m.Speaking);
+}
 #endif

src/discord/voiceclient.hpp

@@ -1,6 +1,6 @@
 #pragma once
 #ifdef WITH_VOICE
-// clang-format off
+    // clang-format off
 #include "snowflake.hpp"
 #include "waiter.hpp"
 #include "websocket.hpp"
@@ -8,6 +8,7 @@
 #include <queue>
 #include <string>
 #include <glibmm/dispatcher.h>
+#include <sigc++/sigc++.h>
 // clang-format on
 
 enum class VoiceGatewayCloseCode : uint16_t {
@@ -110,20 +111,28 @@ struct VoiceSessionDescriptionData {
     friend void from_json(const nlohmann::json &j, VoiceSessionDescriptionData &m);
 };
 
-struct VoiceSpeakingMessage {
-    enum {
-        Microphone = 1 << 0,
-        Soundshare = 1 << 1,
-        Priority = 1 << 2,
-    };
+enum class VoiceSpeakingType {
+    Microphone = 1 << 0,
+    Soundshare = 1 << 1,
+    Priority = 1 << 2,
+};
 
-    int Speaking;
+struct VoiceSpeakingMessage {
+    VoiceSpeakingType Speaking;
     int Delay;
     uint32_t SSRC;
 
     friend void to_json(nlohmann::json &j, const VoiceSpeakingMessage &m);
 };
 
+struct VoiceSpeakingData {
+    Snowflake UserID;
+    uint32_t SSRC;
+    VoiceSpeakingType Speaking;
+
+    friend void from_json(const nlohmann::json &j, VoiceSpeakingData &m);
+};
+
 class UDPSocket {
 public:
     UDPSocket();
@@ -188,6 +197,7 @@ private:
     void HandleGatewayHello(const VoiceGatewayMessage &m);
     void HandleGatewayReady(const VoiceGatewayMessage &m);
     void HandleGatewaySessionDescription(const VoiceGatewayMessage &m);
+    void HandleGatewaySpeaking(const VoiceGatewayMessage &m);
 
     void Identify();
     void Discovery();
@@ -228,5 +238,17 @@ private:
     std::array<uint8_t, 1275> m_opus_buffer;
 
     std::atomic<bool> m_connected = false;
+
+    using type_signal_connected = sigc::signal<void()>;
+    using type_signal_disconnected = sigc::signal<void()>;
+    using type_signal_speaking = sigc::signal<void(VoiceSpeakingData)>;
+    type_signal_connected m_signal_connected;
+    type_signal_disconnected m_signal_disconnected;
+    type_signal_speaking m_signal_speaking;
+
+public:
+    type_signal_connected signal_connected();
+    type_signal_disconnected signal_disconnected();
+    type_signal_speaking signal_speaking();
 };
 #endif