Skip to content

Commit bb1fd90

Browse files
committed
chore: added docstrings to Llama3Initializer
1 parent c3cf4f5 commit bb1fd90

1 file changed

Lines changed: 9 additions & 0 deletions

File tree

src/modalities/models/gpt2/llama3_like_initialization.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,15 @@ class Llama3Initializer(ModelInitializationIF):
2424
"""
2525

2626
def __init__(self, num_layers: int, n_embd: int, depth_init: bool) -> None:
27+
"""
28+
Initializes the Llama3Initializer.
29+
Args:
30+
num_layers: The number of transformer layers in the model. Used to calculate std for certain parameters.
31+
n_embd: The embedding dimension of the model. Used to calculate std and truncation for certain parameters.
32+
depth_init: Whether to use depth-aware initialization for certain parameters, where the std
33+
is scaled based on the layer's depth in the model. If False, a constant std is
34+
used for all layers baed on num_layers.
35+
"""
2736
super().__init__()
2837
self.depth_init = depth_init
2938

0 commit comments

Comments
 (0)