add residual handling

landoskape · May 10, 2024 · 63ccfc7 · 63ccfc7
1 parent 3836c06
commit 63ccfc7
Showing 1 changed file with 1 addition and 3 deletions.
diff --git a/dominoes/networks/transformer_modules.py b/dominoes/networks/transformer_modules.py
@@ -23,7 +23,6 @@ def get_transformer_layer(
     num_multimodal=0,
     kqv_bias=False,
     mlp_bias=True,
-    residual=False,
 ):
     """
     create transformer layer with requested arguments
@@ -35,7 +34,6 @@ def get_transformer_layer(
         kqnorm=kqnorm,
         kqv_bias=kqv_bias,
         mlp_bias=mlp_bias,
-        residual=residual,
     )
     if multimodal:
         transformer_kwargs["num_multimodal"] = num_multimodal
@@ -79,7 +77,7 @@ def __init__(self, embedding_dim, contextual, multimodal, num_heads=8, expansion
             multimodal=multimodal,
             num_multimodal=num_multimodal,
             kqv_bias=kqv_bias,
-            residual=False,
+            residual=False,  # residual is handled in the transformer layer's forward pass
         )
 
         # make the mlp layers