diff --git a/src/Responders/GuildMemberJoinedResponder.cs b/src/Responders/GuildMemberJoinedResponder.cs
index fc18d14..6856f25 100644
--- a/src/Responders/GuildMemberJoinedResponder.cs
+++ b/src/Responders/GuildMemberJoinedResponder.cs
@@ -92,7 +92,7 @@ public class GuildMemberJoinedResponder : IResponder<IGuildMemberAdd>
 
         var assignRoles = new List<Snowflake>();
 
-        if (!GuildSettings.RemoveRolesOnMute.Get(cfg))
+        if (!GuildSettings.RemoveRolesOnMute.Get(cfg) || memberData.MutedUntil is null)
         {
             assignRoles.AddRange(memberData.Roles.ConvertAll(r => r.ToSnowflake()));
         }