feat: add Qwen3_8B

Author: JYMiracle305

CMakeLists.txt

@@ -190,6 +190,14 @@ add_executable(llama3
 )
 link_infini_train_exe(llama3)
 
+add_executable(qwen3
+  example/qwen3/main.cc
+  example/common/tiny_shakespeare_dataset.cc
+  example/common/utils.cc
+  example/qwen3/checkpoint_loader.cc
+  example/common/tokenizer.cc
+)
+link_infini_train_exe(qwen3)
 # Tools
 add_subdirectory(tools/infini_run)
 set_target_properties(infini_run PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

example/qwen3/checkpoint_loader.cc

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+namespace infini_train::nn {
+class TransformerModel;
+} // namespace infini_train::nn
+
+namespace qwen3 {
+std::shared_ptr<infini_train::nn::TransformerModel> LoadFromLLMC(const std::string &filepath);
+} // namespace qwen3

example/qwen3/checkpoint_loader.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "infini_train/include/nn/modules/transformer/transformer_config.h"
+
+namespace nn = infini_train::nn;
+namespace qwen3 {
+inline nn::TransformerConfig Qwen3Config() {
+    return {.block_size = 40960,
+            .vocab_size = 151936,
+            .original_vocab_size = 151936,
+            .n_layer = 36,
+            .n_head = 32,
+            .n_kv_head = 8,
+            .n_embd = 4096,
+            .attention_type = nn::AttentionType::kRoPE,
+            .activation_type = nn::MLPType::kSwiGLU,
+            .norm_type = nn::NormType::kRMSNorm,
+            .add_bias_linear = false,
+            .add_bias_lm_head = false,
+            .tie_weights = false,
+            .ffn_expansion_ratio = 4.5f, // 4096*4.5*2/3 = 12288
+            .ffn_dim_multiplier = std::nullopt,
+            .multiple_of = 1,
+            .rope_theta = 1000000.0f,
+            .use_scaled_rope = false,
+            .norm_eps = 1e-6f};
+}
+} // namespace qwen3