# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # This file was automatically generated from src/transformers/models/minicpm3/modular_minicpm3.py. # Do NOT edit this file manually as any edits will be overwritten by the generation of # the file from the modular. If any change should be done, please apply the change to the # modular_minicpm3.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 The OpenBMB Team and the HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import math from huggingface_hub.dataclasses import strict from ...configuration_utils import PreTrainedConfig from ...modeling_rope_utils import RopeParameters from ...utils import auto_docstring @auto_docstring(checkpoint="openbmb/MiniCPM3-4B") @strict class MiniCPM3Config(PreTrainedConfig): r""" kv_lora_rank (`int`, *optional*, defaults to 256): Rank of the low-rank KV projection in multi-head latent attention. q_lora_rank (`int`, *optional*, defaults to 768): Rank of the low-rank query projection in multi-head latent attention. If `None`, the query projection is a single dense projection rather than a low-rank one. qk_nope_head_dim (`int`, *optional*, defaults to 64): Dimension of the non-RoPE part of each query/key head. qk_rope_head_dim (`int`, *optional*, defaults to 32): Dimension of the RoPE part of each query/key head. v_head_dim (`int`, *optional*): Dimension of each value head. If `None`, defaults to `hidden_size // num_attention_heads`. scale_emb (`int` or `float`, *optional*, defaults to 12): Multiplier applied to input embeddings. scale_depth (`int` or `float`, *optional*, defaults to 1.4): Multiplier for residual connections; the effective scaling is `scale_depth / sqrt(num_hidden_layers)`. If `None`, defaults to `sqrt(num_hidden_layers)` (no-op scaling). dim_model_base (`int`, *optional*, defaults to 256): Base model dimension used to scale logits before the language model head. If `None`, defaults to `hidden_size` (no-op scaling). Example: ```python >>> from transformers import MiniCPM3Model, MiniCPM3Config >>> # Initializing a MiniCPM3 style configuration >>> configuration = MiniCPM3Config() >>> # Initializing a model from the configuration >>> model = MiniCPM3Model(configuration) >>> # Accessing the model configuration >>> configuration = model.config ``` """ model_type = "minicpm3" keys_to_ignore_at_inference = ["past_key_values"] base_model_tp_plan = { "layers.*.self_attn.q_proj": "colwise", "layers.*.self_attn.q_b_proj": "colwise", "layers.*.self_attn.kv_a_proj_with_mqa": "mla_kv_a_proj", "layers.*.self_attn.kv_b_proj": "colwise", "layers.*.self_attn.o_proj": "rowwise", "layers.*.mlp.gate_proj": "colwise", "layers.*.mlp.up_proj": "colwise", "layers.*.mlp.down_proj": "rowwise", } base_model_pp_plan = { "embed_tokens": (["input_ids"], ["inputs_embeds"]), "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } # Only fields whose defaults differ from `LlamaConfig` are redeclared here; the rest are inherited. # Defaults match the `openbmb/MiniCPM3-4B` checkpoint. vocab_size: int = 73448 hidden_size: int = 2560 intermediate_size: int = 6400 num_hidden_layers: int = 62 num_attention_heads: int = 40 num_key_value_heads: int | None = 40 hidden_act: str = "silu" max_position_embeddings: int = 32768 initializer_range: float = 0.1 rms_norm_eps: float = 1e-5 use_cache: bool = True pad_token_id: int | None = None bos_token_id: int | None = 1 eos_token_id: int | list[int] | None = 2 pretraining_tp: int | None = 1 tie_word_embeddings: bool = True rope_parameters: RopeParameters | dict | None = None attention_bias: bool = False attention_dropout: int | float | None = 0.0 mlp_bias: bool = False head_dim: int | None = None kv_lora_rank: int = 256 q_lora_rank: int | None = 768 qk_nope_head_dim: int = 64 qk_rope_head_dim: int = 32 v_head_dim: int | None = None scale_emb: int | float = 12 scale_depth: int | float | None = 1.4 dim_model_base: int | None = 256 def __post_init__(self, **kwargs): # In MLA the per-head dim used by RoPE is the rotary part, not `hidden_size / num_attention_heads`. self.head_dim = self.qk_rope_head_dim # When explicitly set to `None`, these collapse to no-op scalings (e.g. for a randomly # initialised tiny model); the class defaults otherwise match `openbmb/MiniCPM3-4B`. if self.v_head_dim is None: self.v_head_dim = self.hidden_size // self.num_attention_heads if self.scale_depth is None: self.scale_depth = math.sqrt(self.num_hidden_layers) if self.dim_model_base is None: self.dim_model_base = self.hidden_size if self.head_dim is None: self.head_dim = self.hidden_size // self.num_attention_heads if self.num_key_value_heads is None: self.num_key_value_heads = self.num_attention_heads super().__post_init__(**kwargs) def validate_architecture(self): """Part of `@strict`-powered validation. Validates the architecture of the config.""" if self.hidden_size % self.num_attention_heads != 0: raise ValueError( f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " f"heads ({self.num_attention_heads})." ) @property def logits_scaling(self) -> float: # Hidden states are divided by this factor before the LM head (`1` when `dim_model_base == hidden_size`). return self.hidden_size / self.dim_model_base __all__ = ["MiniCPM3Config"]