Unverified Commit 2391ae26 authored by NingMa's avatar NingMa Committed by GitHub

Update cross_attention.py

parent 4afe3dcb
......@@ -36,7 +36,7 @@ class CrossAttention(nn.Module):
attention_probs_dropout_prob = 0.2
self.attn_dropout = nn.Dropout(attention_probs_dropout_prob)
# 做完self-attention 做一个前馈全连接 LayerNorm 输出
#
self.dense = nn.Linear(hidden_size, hidden_size)
self.LayerNorm = LayerNorm(hidden_size, eps=1e-12)
self.out_dropout = nn.Dropout(hidden_dropout_prob)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment