diff --git a/src/client/voice/VoiceConnection.js b/src/client/voice/VoiceConnection.js
index de5f51bfd204..bdc21d51e556 100644
--- a/src/client/voice/VoiceConnection.js
+++ b/src/client/voice/VoiceConnection.js
@@ -473,7 +473,11 @@ class VoiceConnection extends EventEmitter {
   }
 
   onStartSpeaking({ user_id, ssrc, speaking }) {
-    this.ssrcMap.set(+ssrc, { userID: user_id, speaking: speaking });
+    this.ssrcMap.set(+ssrc, {
+      ...(this.ssrcMap.get(+ssrc) || {}),
+      userID: user_id,
+      speaking: speaking,
+    });
   }
 
   /**
diff --git a/src/client/voice/networking/VoiceWebSocket.js b/src/client/voice/networking/VoiceWebSocket.js
index fefd241b58a8..db344010ad01 100644
--- a/src/client/voice/networking/VoiceWebSocket.js
+++ b/src/client/voice/networking/VoiceWebSocket.js
@@ -189,7 +189,11 @@ class VoiceWebSocket extends EventEmitter {
         this.emit('sessionDescription', packet.d);
         break;
       case VoiceOPCodes.CLIENT_CONNECT:
-        this.connection.ssrcMap.set(+packet.d.audio_ssrc, { userID: packet.d.user_id, speaking: 0 });
+        this.connection.ssrcMap.set(+packet.d.audio_ssrc, {
+          userID: packet.d.user_id,
+          speaking: 0,
+          hasVideo: Boolean(packet.d.video_ssrc),
+        });
         break;
       case VoiceOPCodes.CLIENT_DISCONNECT:
         const streamInfo = this.connection.receiver && this.connection.receiver.packets.streams.get(packet.d.user_id);
diff --git a/src/client/voice/receiver/PacketHandler.js b/src/client/voice/receiver/PacketHandler.js
index ebcd6732e963..bf1a220a2fd8 100644
--- a/src/client/voice/receiver/PacketHandler.js
+++ b/src/client/voice/receiver/PacketHandler.js
@@ -1,7 +1,9 @@
 'use strict';
 
 const EventEmitter = require('events');
+const Speaking = require('../../../util/Speaking');
 const secretbox = require('../util/Secretbox');
+const { SILENCE_FRAME } = require('../util/Silence');
 
 // The delay between packets when a user is considered to have stopped speaking
 // https://github.com/discordjs/discord.js/issues/3524#issuecomment-540373200
@@ -84,12 +86,30 @@ class PacketHandler extends EventEmitter {
     const userStat = this.connection.ssrcMap.get(ssrc);
     if (!userStat) return;
 
+    let opusPacket;
+    const streamInfo = this.streams.get(userStat.userID);
+    // If the user is in video, we need to check if the packet is just silence
+    if (userStat.hasVideo) {
+      opusPacket = this.parseBuffer(buffer);
+      if (opusPacket instanceof Error) {
+        // Only emit an error if we were actively receiving packets from this user
+        if (streamInfo) {
+          this.emit('error', opusPacket);
+          return;
+        }
+      }
+      if (SILENCE_FRAME.equals(opusPacket)) {
+        // If this is a silence frame, pretend we never received it
+        return;
+      }
+    }
+
     let speakingTimeout = this.speakingTimeouts.get(ssrc);
     if (typeof speakingTimeout === 'undefined') {
       // Ensure at least the speaking bit is set.
       // As the object is by reference, it's only needed once per client re-connect.
       if (userStat.speaking === 0) {
-        userStat.speaking = 1;
+        userStat.speaking = Speaking.FLAGS.SPEAKING;
       }
       this.connection.onSpeaking({ user_id: userStat.userID, ssrc: ssrc, speaking: userStat.speaking });
       speakingTimeout = this.receiver.connection.client.setTimeout(() => {
@@ -106,15 +126,17 @@ class PacketHandler extends EventEmitter {
       speakingTimeout.refresh();
     }
 
-    let stream = this.streams.get(userStat.userID);
-    if (!stream) return;
-    stream = stream.stream;
-    const opusPacket = this.parseBuffer(buffer);
-    if (opusPacket instanceof Error) {
-      this.emit('error', opusPacket);
-      return;
+    if (streamInfo) {
+      const { stream } = streamInfo;
+      if (!opusPacket) {
+        opusPacket = this.parseBuffer(buffer);
+        if (opusPacket instanceof Error) {
+          this.emit('error', opusPacket);
+          return;
+        }
+      }
+      stream.push(opusPacket);
     }
-    stream.push(opusPacket);
   }
 }
 
diff --git a/src/client/voice/util/Silence.js b/src/client/voice/util/Silence.js
index 9bea3d00c6b7..7930ae6535fe 100644
--- a/src/client/voice/util/Silence.js
+++ b/src/client/voice/util/Silence.js
@@ -10,4 +10,6 @@ class Silence extends Readable {
   }
 }
 
+Silence.SILENCE_FRAME = SILENCE_FRAME;
+
 module.exports = Silence;