diff --git a/conversion/__init__.py b/conversion/__init__.py index fe3dd09331b9..c46aab837a30 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -19,6 +19,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "AfmoeForCausalLM": "afmoe", "LagunaForCausalLM": "laguna", + "FlintForCausalLM": "flint", "ApertusForCausalLM": "llama", "ArceeForCausalLM": "llama", "ArcticForCausalLM": "arctic", diff --git a/conversion/base.py b/conversion/base.py index adb616c379b1..19bd1b2ec3bf 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1529,7 +1529,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_expert_group_used_count(n_group_used) logger.info(f"gguf: expert groups used count = {n_group_used}") - if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func"], optional=True)) is not None: + if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func", "router_score_function"], optional=True)) is not None: if score_func == "sigmoid": self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) elif score_func == "softmax": @@ -1695,6 +1695,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "1444df51289cfa8063b96f0e62b1125440111bc79a52003ea14b6eac7016fd5f": # ref: https://huggingface.co/openbmb/MiniCPM-V-4_6 res = "qwen35" + if chkhsh == "9ff8c72e7519833cf13eb20da2d87641873339e004dddddde384920fba6431b5": + # ref: https://huggingface.co/fastino/GLiNER2.5-Decide + res = "default" if chkhsh == "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273": # ref: https://huggingface.co/alvarobartt/grok-2-tokenizer res = "grok-2" @@ -1737,11 +1740,6 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "8e62295832751ca1e8f92f2226f403dea30dc5165e448b5bfa05af5340c64ec7": # ref: https://huggingface.co/BAAI/bge-large-zh-v1.5 res = "bert-bge-large" - if chkhsh == "9ff8c72e7519833cf13eb20da2d87641873339e004dddddde384920fba6431b5": - # ref: DeBERTa-v3 / GLiNER2.x -- Metaspace pre-tokenizer with - # prepend_scheme "always" is llama.cpp's "default" SPM behaviour. - # Unigram model, so the writer also needs token scores. - res = "default" if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166": # ref: https://huggingface.co/mosaicml/mpt-7b res = "mpt" @@ -1766,6 +1764,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e": # ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0 res = "cohere2moe" + if chkhsh == "d5420e601e979078c670bc503d9ac411c311fa76b76a76267c31e0d6ba816637": + # ref: https://huggingface.co/RecursiveSI/flint-preview + res = "flint" if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea": # ref: https://huggingface.co/Qwen/Qwen1.5-7B res = "qwen2" diff --git a/conversion/flint.py b/conversion/flint.py new file mode 100644 index 000000000000..1fd4776f294e --- /dev/null +++ b/conversion/flint.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from .base import ModelBase, gguf +from .qwen import Qwen2MoeModel + + +@ModelBase.register("FlintForCausalLM") +@ModelBase.example("RecursiveSI/flint-preview") +class FlintModel(Qwen2MoeModel): + model_arch = gguf.MODEL_ARCH.FLINT + + def set_gguf_parameters(self): + super().set_gguf_parameters() + dense = self.hparams["mlp_only_layers"] + if dense != list(range(len(dense))): + raise ValueError("Flint dense layers must precede the MoE layers") + self.gguf_writer.add_leading_dense_block_count(len(dense)) + self.gguf_writer.add_expert_shared_count( + self.hparams["shared_expert_intermediate_size"] // self.hparams["moe_intermediate_size"] + ) + self.gguf_writer.add_expert_weights_norm(True) + self.gguf_writer.add_expert_weights_scale(1.0) + self.gguf_writer.add_sliding_window(self.hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([ + kind == "sliding_attention" for kind in self.hparams["layer_types"] + ]) + self.gguf_writer.add_rope_pattern([ + i not in self.hparams["no_rope_layer_indices"] for i in range(self.block_count) + ]) + + def modify_tensors(self, data_torch, name, bid): + if name.endswith(".softmax_offset"): + name += ".weight" + if name.endswith(".gate.expert_bias"): + if not self.hparams["router_expert_bias"]: + return + name += ".bias" + yield from super().modify_tensors(data_torch, name, bid) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 0afbcc7e93a3..7da0bfadb0d9 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -101,6 +101,7 @@ class TOKENIZER_TYPE(IntEnum): {"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", }, {"name": "tiny_aya", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", }, {"name": "cohere2moe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0", }, + {"name": "flint", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/RecursiveSI/flint-preview", }, {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", }, {"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", }, {"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", }, @@ -188,6 +189,10 @@ class TOKENIZER_TYPE(IntEnum): {"name": "kimi-k2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/moonshotai/Kimi-K2-Base", "chkhsh": "81212dc7cdb7e0c1074ca62c5aeab0d43c9f52b8a737be7b12a777c953027890"}, {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B", "chkhsh": "d4540891389ea895b53b399da6ac824becc30f2fba0e9ddbb98f92e55ca0e97c"}, {"name": "qwen35", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openbmb/MiniCPM-V-4_6", "chkhsh": "1444df51289cfa8063b96f0e62b1125440111bc79a52003ea14b6eac7016fd5f"}, + # DeBERTa-v3 / GLiNER2.5 is a Unigram tokenizer whose Metaspace pre-tokenizer (prepend_scheme "always") + # matches llama.cpp's "default". Listed as BPE only because the generator drops SPM/UGM entries from the + # hash table; with a pre-computed chkhsh the type is never used to load anything. + {"name": "default", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/fastino/GLiNER2.5-Decide", "chkhsh": "9ff8c72e7519833cf13eb20da2d87641873339e004dddddde384920fba6431b5"}, {"name": "grok-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"}, # jina-v2-de variants {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"}, diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 8ed5f4ebbc2d..9e56e46b3189 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -774,6 +774,13 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( return src_ss[0]; }; + auto handle_pool_1d = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { + if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_0) { + return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; + } + return src_ss[0]; + }; + auto handle_flash_attn_ext = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { GGML_ASSERT(tensor->src[3] == nullptr || src_ss[3].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); @@ -990,12 +997,14 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( case GGML_OP_CONV_3D: case GGML_OP_CONV_2D_DW: case GGML_OP_CONV_TRANSPOSE_2D: - case GGML_OP_POOL_1D: case GGML_OP_POOL_2D: case GGML_OP_POOL_2D_BACK: case GGML_OP_UPSCALE: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; + case GGML_OP_POOL_1D: { + split_state = handle_pool_1d(src_ss); + } break; case GGML_OP_PAD: { split_state = handle_pad(src_ss); } break; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index a09885f06c7e..ccee63ba48e3 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -543,6 +543,7 @@ class MODEL_ARCH(IntEnum): QWEN2VL = auto() QWEN3 = auto() QWEN3MOE = auto() + FLINT = auto() QWEN3NEXT = auto() HRM_TEXT = auto() QWEN3VL = auto() @@ -1328,6 +1329,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.QWEN2MOE: "qwen2moe", MODEL_ARCH.QWEN2VL: "qwen2vl", MODEL_ARCH.QWEN3: "qwen3", + MODEL_ARCH.FLINT: "flint", MODEL_ARCH.QWEN3MOE: "qwen3moe", MODEL_ARCH.QWEN3NEXT: "qwen3next", MODEL_ARCH.HRM_TEXT: "hrm_text", @@ -2853,6 +2855,33 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, ], + + MODEL_ARCH.FLINT: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + ], MODEL_ARCH.QWEN3NEXT: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index d8ed952012c6..53adf418e9c3 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -394,6 +394,7 @@ class TensorNameMap: ), MODEL_TENSOR.ATTN_SINKS: ( + "model.layers.{bid}.self_attn.softmax_offset", # flint "model.layers.{bid}.self_attn.sinks", # openai-moe "model.layers.{bid}.self_attn.attention_sink_bias", # mimov2 "model.layers.{bid}.self_attn.learnable_sink_param", # hy-v4 diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 162cbf08817e..d2973f40e382 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -36,6 +36,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_QWEN2MOE, "qwen2moe" }, { LLM_ARCH_QWEN2VL, "qwen2vl" }, { LLM_ARCH_QWEN3, "qwen3" }, + { LLM_ARCH_FLINT, "flint" }, { LLM_ARCH_QWEN3MOE, "qwen3moe" }, { LLM_ARCH_QWEN3NEXT, "qwen3next" }, { LLM_ARCH_QWEN3VL, "qwen3vl" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 4e9dbff7c689..957b0824124f 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -42,6 +42,7 @@ enum llm_arch { LLM_ARCH_QWEN2VL, LLM_ARCH_QWEN3, LLM_ARCH_QWEN3MOE, + LLM_ARCH_FLINT, LLM_ARCH_QWEN3NEXT, LLM_ARCH_QWEN3VL, LLM_ARCH_QWEN3VLMOE, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index e56f3ba14e1d..7b9b941d007c 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1585,6 +1585,43 @@ ggml_tensor * llm_graph_context::build_lora_mm( return res; } +llm_graph_amax_src1 llm_graph_context::build_amax_src1_f16( + ggml_tensor * cur) const { + llm_graph_amax_src1 result = { + /*.res =*/ cur, + /*.scale_s =*/ nullptr, + }; + + const int64_t n_embd = cur->ne[0]; + + // amax = max|cur| along the rows for each [expert, token] -> [1, n_expert, n_tokens] + ggml_tensor * ab = ggml_abs(ctx0, cur); + ggml_tensor * amax = ggml_pool_1d(ctx0, ab, GGML_OP_POOL_MAX, n_embd, n_embd, 0); + + // ref: https://github.com/ggml-org/llama.cpp/pull/26223 + constexpr float k_log2_e = 1.4426950408889634f; // 1 / ln(2) + constexpr float k_ln_2 = 0.6931471805599453f; + + ggml_tensor * e; + // note: this can technically be: + // e = ggml_scale(ctx0, amax, 1.0f/32768.0f); + // but we leave x32 headroom for the F16 sums of 32 values in Q8 activation blocks + e = ggml_scale(ctx0, amax, 1.0f/1024.0f); + e = ggml_log (ctx0, e); + e = ggml_scale(ctx0, e, k_log2_e); + e = ggml_ceil (ctx0, e); + e = ggml_clamp(ctx0, e, 0.0f, 127.0f); + e = ggml_scale(ctx0, e, k_ln_2); + e = ggml_exp (ctx0, e); + + result = { + /*.res =*/ ggml_div(ctx0, cur, e), + /*.scale_s =*/ e, + }; + + return result; +} + ggml_tensor * llm_graph_context::build_lora_mm_id( ggml_tensor * w, // ggml_tensor * as ggml_tensor * cur, // ggml_tensor * b @@ -2359,12 +2396,25 @@ ggml_tensor * llm_graph_context::build_moe_ffn( GGML_ABORT("fatal error"); } - experts = build_lora_mm_id(down_exps, cur, selected_experts, down_exps_s, slots); // [n_embd, n_expert_used, n_tokens] - if (arch == LLM_ARCH_MISTRAL4) { - // src1 can exceed F16 range - ggml_prec_set_src(experts, GGML_PREC_F32, 1); + { + // TODO: this graph-level rescaling should only be applied when the corresponding backend does not support + // ggml_prec_set_src(experts, GGML_PREC_BF16, 1); + // ref: https://github.com/ggml-org/llama.cpp/pull/30656#issuecomment-6269597922 + llm_graph_amax_src1 amax_src1 = { nullptr, nullptr }; + + if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT) { + amax_src1 = build_amax_src1_f16(cur); + cur = amax_src1.res; + } + + experts = build_lora_mm_id(down_exps, cur, selected_experts, down_exps_s, slots); // [n_embd, n_expert_used, n_tokens] + //ggml_prec_set_src(experts, GGML_PREC_BF16, 1); // (see TODO above) + cb(experts, "ffn_moe_down", il); + + if (amax_src1.scale_s) { + experts = ggml_mul(ctx0, experts, amax_src1.scale_s); + } } - cb(experts, "ffn_moe_down", il); if (down_exps_s) { cb(experts, "ffn_moe_down_scaled", il); diff --git a/src/llama-graph.h b/src/llama-graph.h index 02ac03429f62..8efc4449357c 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -1013,6 +1013,11 @@ struct llm_graph_qkv { ggml_tensor * v; // [n_embd_head, n_head_kv, n_tokens] }; +struct llm_graph_amax_src1 { + ggml_tensor * res; // cur rescaled + ggml_tensor * scale_s; // scale to undo on the matmul result, or nullptr +}; + struct llm_graph_context { const llm_arch arch; @@ -1109,6 +1114,11 @@ struct llm_graph_context { ggml_tensor * w_s = nullptr, ggml_tensor * slots = nullptr) const; + // rescale `cur` (the src1 of a matmul) by a power of two, computed from its maximum absolute + // keep both individual values and Q8 activation block sums within F16 range + llm_graph_amax_src1 build_amax_src1_f16( + ggml_tensor * cur) const; + ggml_tensor * build_norm( ggml_tensor * cur, ggml_tensor * mw, diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 463510caa768..57aaf0f6948e 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -880,7 +880,10 @@ bool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_co LLAMA_LOG_DEBUG("%s: applying K-shift\n", __func__); // apply K-shift if needed - if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE) { + if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE && + std::any_of(layers.begin(), layers.end(), [&](const auto & layer) { + return hparams.has_rope(layer.il); + })) { ggml_backend_sched_reset(sched); auto * res = lctx->get_gf_res_reserve(); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index f25c6ae24069..88cc66ca3e94 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -113,6 +113,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_qwen2moe(params); case LLM_ARCH_QWEN3: return new llama_model_qwen3(params); + case LLM_ARCH_FLINT: + return new llama_model_flint(params); case LLM_ARCH_QWEN3MOE: return new llama_model_qwen3moe(params); case LLM_ARCH_QWEN3VL: @@ -3170,6 +3172,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_DREAM: case LLM_ARCH_QWEN2MOE: case LLM_ARCH_QWEN3: + case LLM_ARCH_FLINT: case LLM_ARCH_QWEN3MOE: case LLM_ARCH_LLADA_MOE: case LLM_ARCH_RND1: diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index dbadaf100b64..7801ecc6e8d9 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -281,6 +281,13 @@ struct llm_tokenizer_bpe : llm_tokenizer { llm_tokenizer_bpe(const llama_vocab & vocab) { GGML_ASSERT(vocab.get_type() == LLAMA_VOCAB_TYPE_BPE); switch (vocab.get_pre_type()) { + case LLAMA_VOCAB_PRE_TYPE_FLINT: + regex_exprs = { + "\\p{N}{1,3}", + "[一-龥぀-ゟ゠-ヿ]+", + "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?| ?[\\p{P}\\p{S}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + break; case LLAMA_VOCAB_PRE_TYPE_LLAMA3: regex_exprs = { // original regex from tokenizer.json @@ -2278,6 +2285,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { LLAMA_LOG_WARN("%s: ************************************ \n", __func__); LLAMA_LOG_WARN("%s: \n", __func__); pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT; + } else if (tokenizer_pre == "flint") { + pre_type = LLAMA_VOCAB_PRE_TYPE_FLINT; + clean_spaces = false; } else if (tokenizer_pre == "default") { pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT; } else if (tokenizer_pre == "minicpm5") { @@ -2997,7 +3007,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { // this is currently determined based on the token text, which is obviously not ideal // ref: https://github.com/ggml-org/llama.cpp/issues/9606 special_eog_ids.clear(); - if (special_fim_pad_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_pad_id) == 0) { special_eog_ids.insert(special_fim_pad_id); } @@ -3030,6 +3039,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { || t.first == "[EOT]" // Kimi-K2 || t.first == "[EOS]" // Kimi-K2 || t.first == "<|end_of_text|>" + || (pre_type == LLAMA_VOCAB_PRE_TYPE_FLINT && (t.first == "<|user|>" || t.first == "<|observation|>")) || t.first == "" // smoldocling || t.first == "" // gemma4 || t.first == "" // gemma4 diff --git a/src/llama-vocab.h b/src/llama-vocab.h index 2d40c6271017..c483ea70968c 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -70,6 +70,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59, LLAMA_VOCAB_PRE_TYPE_MMBERT = 60, LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61, + LLAMA_VOCAB_PRE_TYPE_FLINT = 62, }; struct LLM_KV; diff --git a/src/models/flint.cpp b/src/models/flint.cpp new file mode 100644 index 000000000000..32e3e9bb4337 --- /dev/null +++ b/src/models/flint.cpp @@ -0,0 +1,196 @@ +#include "models.h" + +void llama_model_flint::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_flint::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_ff_exp = hparams.n_ff_exp(); + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + if (i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } else { + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); + + if (n_expert == 0) { + throw std::runtime_error("n_expert must be > 0"); + } + if (n_expert_used == 0) { + throw std::runtime_error("n_expert_used must be > 0"); + } + + // MoE branch + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); + + // Shared expert branch + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { hparams.n_ff_shexp, n_embd}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0); + } + } +} + +std::unique_ptr llama_model_flint::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_flint::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + GGML_ASSERT(n_embd_head == n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self_attention + { + // compute Q and K and RoPE them + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, n_head_kv, il); + + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); + if (hparams.has_rope(il)) { + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, 1.0f / sqrtf(float(n_embd_head)), il); + } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // MoE branch + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, nullptr, nullptr, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); + cb(moe_out, "ffn_moe_out", il); + + { + ggml_tensor * ffn_shexp = + build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, + model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s, + model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + } + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur, model.output_s); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/models.h b/src/models/models.h index 2c02d34975b6..704e88fda5e0 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -615,6 +615,19 @@ struct llama_model_qwen3moe : public llama_model_base { }; +struct llama_model_flint : public llama_model_base { + llama_model_flint(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_qwen3vl : public llama_model_base { llama_model_qwen3vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index e839e20ce3f5..7f756da10482 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -69,6 +69,9 @@ falcon ,0 ,any ,ADD+ADD , 2 falcon ,0 ,any ,NORM+MUL+ADD , 5 falcon-h1 ,0 ,any ,ADD+ADD , 2 falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9 +flint ,0 ,any ,ADD+ADD , 1 +flint ,0 ,any ,MUL+ADD , 2 +flint ,0 ,any ,RMS_NORM+MUL , 9 gemma ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 78128ebf0377..833de37a661b 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -299,13 +299,16 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || - arch == LLM_ARCH_MAPLE) { + arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) { std::vector pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { pattern.push_back(il % 2); } ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); + if (arch == LLM_ARCH_FLINT) { + ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, pattern); + } } else { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2)); } @@ -437,6 +440,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f); ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1)); + if (arch == LLM_ARCH_FLINT) { + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); + } if (arch == LLM_ARCH_K2_HORIZON) { ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, uint32_t(2)); ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, uint32_t(2)); @@ -653,6 +659,7 @@ static bool check_causal_attn_toggle( static bool moe_mandatory(const llm_arch arch) { switch (arch) { + case LLM_ARCH_FLINT: case LLM_ARCH_LLAMA4: case LLM_ARCH_COHERE2MOE: case LLM_ARCH_GROK: