-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathfinal_params.txt
More file actions
81 lines (81 loc) · 13.9 KB
/
final_params.txt
File metadata and controls
81 lines (81 loc) · 13.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
Model | Non-Emb Params | Total | d_ff | Norm | Pos | Act | Bias
------------------------------------------------------------------------------------------
neon001 | 2,108,928 | 2,371,072 | 512 | Unknown | RoPE | GELU | True
neon002 | 2,099,968 | 2,362,112 | 512 | RMS | RoPE | GELU | False
neon003 | 1,706,752 | 1,968,896 | 512 | RMS | RoPE | GELU | False
neon004 | 1,706,752 | 1,968,896 | 512 | RMS | RoPE | Unknown | False
neon005 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon006 | 2,493,184 | 2,755,328 | 512 | RMS | RoPE | SwiGLU | False
neon007 | 2,627,840 | 2,889,984 | 512 | RMS | RoPE | SwiGLU | False
neon008 | 2,626,560 | 2,888,704 | 512 | LN | RoPE | SwiGLU | False
neon009 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon010 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon011 | 11,837,312 | 12,230,528 | 768 | RMS | RoPE | SwiGLU | False
neon012 | 15,761,024 | 16,285,312 | 1024 | RMS | RoPE | SwiGLU | False
neon013 | 8,211,200 | 8,538,880 | 640 | RMS | RoPE | SwiGLU | False
neon014 | 14,186,496 | 14,579,712 | 1536 | RMS | RoPE | SwiGLU | False
neon015 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon016 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon017 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon018 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon019 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon020 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon021 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon022 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon023 | 5,772,544 | 6,034,688 | 512 | RMS | RoPE | SwiGLU | False
neon024 | 5,772,544 | 6,034,688 | 512 | RMS | RoPE | SwiGLU | False
neon025 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon026 | 2,888,448 | 3,150,592 | 598 | RMS | RoPE | SwiGLU | False
neon027 | 2,886,656 | 3,148,800 | 592 | RMS | RoPE | SwiGLU | False
neon028 | 2,886,400 | 3,148,544 | 640 | RMS | RoPE | SwiGLU | False
neon029 | 2,886,636 | 3,148,780 | 891 | Unknown | RoPE | GELU | True
neon030 | 2,886,400 | 3,148,544 | 896 | RMS | RoPE | GELU | False
neon031 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon032 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon033 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon034 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon035 | 2,624,768 | 2,886,912 | 512 | RMS | RoPE | SwiGLU | False
neon036 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon037 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon038 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon039 | 2,624,256 | 2,886,400 | 512 | RMS | RoPE | SwiGLU | False
neon040 | 2,624,512 | 2,886,656 | 512 | RMS | RoPE | SwiGLU | False
neon041 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon042 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon043 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon044 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon045 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon046 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon047 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon048 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon049 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon050 | 2,641,152 | 2,903,296 | 512 | RMS | RoPE | SwiGLU | False
neon051 | 2,625,280 | 2,887,424 | 512 | RMS | RoPE | SwiGLU | False
neon052 | 2,673,664 | 2,935,808 | 512 | RMS | RoPE | SwiGLU | False
neon053 | 2,886,400 | 3,148,544 | 512 | RMS | RoPE | SwiGLU | False
neon054 | 2,640,896 | 2,903,040 | 512 | RMS | RoPE | SwiGLU | False
neon055 | 2,886,656 | 3,148,800 | 592 | RMS | RoPE | SwiGLU | False
neon056 | 2,903,296 | 3,165,440 | 592 | RMS | RoPE | SwiGLU | False
neon057 | 2,886,656 | 3,148,800 | 592 | RMS | RoPE | SwiGLU | False
neon058 | 2,640,640 | 2,902,784 | 512 | RMS | RoPE | SwiGLU | False
neon059 | 2,887,424 | 3,149,568 | 592 | RMS | RoPE | SwiGLU | False
neon060 | 2,886,656 | 3,148,800 | 592 | RMS | RoPE | SwiGLU | False
neon061 | 9,718,528 | 9,980,672 | 2736 | RMS | RoPE | SwiGLU | False
neon062 | 2,624,768 | 2,886,912 | 512 | RMS | RoPE | Unknown | False
neon063 | 3,937,024 | 4,199,168 | 512 | RMS | RoPE | Unknown | False
neon064 | 2,755,072 | 3,017,216 | 512 | RMS | RoPE | SwiGLU | False
neon065 | 4,200,704 | 4,462,848 | 512 | RMS | RoPE | SwiGLU | False
neon066 | 2,887,936 | 3,150,080 | 512 | RMS | RoPE | SwiGLU | False
neon067 | 2,886,912 | 3,149,056 | 512 | RMS | RoPE | SwiGLU | False
neon068 | 2,886,144 | 3,148,288 | 512 | RMS | RoPE | SwiGLU | False
neon069 | 2,886,016 | 3,148,160 | 512 | RMS | RoPE | SwiGLU | False
neon070 | 2,837,248 | 3,099,392 | 576 | RMS | RoPE | Unknown | False
neon071 | 2,984,704 | 3,246,848 | 640 | RMS | RoPE | Unknown | False
neon072 | 3,214,080 | 3,476,224 | 512 | RMS | RoPE | Unknown | False
neon073 | 2,837,248 | 3,099,392 | 576 | RMS | RoPE | Unknown | False
neon074 | 2,837,248 | 3,099,392 | 576 | RMS | RoPE | Unknown | False
neon075 | 2,837,248 | 3,099,392 | 576 | RMS | RoPE | Unknown | False
neon076 | 2,825,040 | 3,070,800 | 480 | RMS | RoPE | Unknown | False
neon077 | 2,823,936 | 3,086,080 | 368 | RMS | RoPE | Unknown | False
neon078 | 2,862,208 | 3,124,352 | 500 | RMS | RoPE | SwiGLU | False
neon079 | 2,867,096 | 3,129,240 | 480 | RMS | RoPE | SwiGLU | False