diff --git a/speechbrain/decoders/ctc.py b/speechbrain/decoders/ctc.py
index d25dc1f5758de3bd38059aae286365450e465ba7..223ddfeb57f1711f1282f27430cd2d7a112f0a6a 100644
--- a/speechbrain/decoders/ctc.py
+++ b/speechbrain/decoders/ctc.py
@@ -232,8 +232,9 @@ class CTCPrefixScore:
                 self.last_frame_index[i // beam_size], i
             ]
 
-        # Exclude blank probs for joint scoring
-        psi[:, self.blank_index] = self.minus_inf
+        if self.eos_index != self.blank_index:
+            # Exclude blank probs for joint scoring
+            psi[:, self.blank_index] = self.minus_inf
 
         return psi - psi_prev, (r, psi, scoring_table)