# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # This file was automatically generated from src/transformers/models/inkling/modular_inkling.py. # Do NOT edit this file manually as any edits will be overwritten by the generation of # the file from the modular. If any change should be done, please apply the change to the # modular_inkling.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 the HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # coding=utf-8 from huggingface_hub.dataclasses import strict from ...configuration_utils import PreTrainedConfig @strict class InklingTextConfig(PreTrainedConfig): model_type = "inkling_text" base_config_key = "text_config" base_model_tp_plan = { "embed_tokens": "embedding_rowwise", "layers.*.mlp.experts.gate_up_proj": "colwise", "layers.*.mlp.experts.down_proj": "rowwise", "layers.*.mlp.experts": "moe_tp_experts", "layers.*.mlp.shared_experts.gate_proj": "colwise", "layers.*.mlp.shared_experts.up_proj": "colwise", "layers.*.mlp.shared_experts.down_proj": "rowwise", "layers.*.mlp.shared_experts": "all_reduce", "layers.*.mlp.gate_proj": "colwise", "layers.*.mlp.up_proj": "colwise", "layers.*.mlp.down_proj": "rowwise", } base_model_pp_plan = { "embed_tokens": (["input_ids"], ["inputs_embeds"]), "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } base_model_ep_plan = { "layers.*.mlp.gate": "ep_router", "layers.*.mlp.experts.gate_up_proj": "grouped_gemm", "layers.*.mlp.experts.down_proj": "grouped_gemm", "layers.*.mlp.experts": "moe_tp_experts", } attribute_map = { "embedding_multiplier": "logits_mup_width_multiplier", "sliding_window": "sliding_window_size", "num_local_experts": "n_routed_experts", # checkpoints store `sconv_kernel_size`; without the mapping a fresh config has no such # attribute (InklingAttention reads it) and a checkpoint value would bypass `conv_kernel_size` "sconv_kernel_size": "conv_kernel_size", # checkpoints advertise the context length as `model_max_length` "model_max_length": "max_position_embeddings", } vocab_size: int = 201024 # `unembed` row count when the checkpoint head is not padded to `vocab_size` (big model: 200058) unpadded_vocab_size: int | None = None hidden_size: int = 6144 num_hidden_layers: int = 66 num_attention_heads: int = 64 num_key_value_heads: int = 8 head_dim: int = 128 swa_num_attention_heads: int = 64 swa_num_key_value_heads: int = 16 swa_head_dim: int = 128 sliding_window_size: int = 512 d_rel: int = 16 rel_extent: int = 1024 log_scaling_n_floor: int | None = None log_scaling_alpha: float = 0.1 local_layer_ids: list[int] | None = None layer_types: list[str] | None = None max_position_embeddings: int = 131072 rms_norm_eps: float = 1e-6 conv_kernel_size: int = 4 mlp_layer_types: list[str] | None = None intermediate_size: int = 24576 hidden_act: str = "silu" # MoE moe_intermediate_size: int = 3072 n_routed_experts: int = 256 num_experts_per_tok: int = 6 n_shared_experts: int = 2 shared_expert_sink: bool = True route_scale: float = 8.0 logits_mup_width_multiplier: float = 24.0 rms_norm_eps_moe_gate: float = 1e-6 attention_dropout: float = 0.0 initializer_range: float = 0.02 pad_token_id: int | None = None bos_token_id: int | None = 1 eos_token_id: int | None = 2 # MTP related fields num_mtp_layers: int | None = None chain_hidden_post_norm: bool = False mtp_hidden_states_first: bool = True mtp_local_layer_ids: list[int] | None = None def __post_init__(self, **kwargs): if self.layer_types is None: if self.local_layer_ids is not None: local_layer_ids = set(self.local_layer_ids) else: local_layer_ids = {i for i in range(self.num_hidden_layers) if (i + 1) % 6} self.layer_types = [ "hybrid_sliding" if i in local_layer_ids else "hybrid" for i in range(self.num_hidden_layers) ] if self.mlp_layer_types is None: dense_mlp_idx = kwargs.pop("dense_mlp_idx", 0) self.mlp_layer_types = ["dense" if i < dense_mlp_idx else "sparse" for i in range(self.num_hidden_layers)] if kwargs.get("dense_intermediate_size") is not None: self.intermediate_size = kwargs.pop("dense_intermediate_size") # The architecture contains 4 conv modules per layer, each needing a different conv cache self.number_of_conv_states = 4 super().__post_init__(**kwargs) @property def mtp_layer_types(self): if self.num_mtp_layers is not None: if self.mtp_local_layer_ids is None: return ["hybrid"] * self.num_mtp_layers else: return [ "hybrid_sliding" if i in self.mtp_local_layer_ids else "hybrid" for i in range(self.num_mtp_layers) ] return None # MTP layers are always dense MLP @property def mtp_mlp_layer_types(self): if self.num_mtp_layers is not None: return ["dense"] * self.num_mtp_layers return None @strict class InklingAudioConfig(PreTrainedConfig): model_type = "inkling_audio" base_config_key = "audio_config" attribute_map = { "num_codebooks": "n_mel_bins", "codebook_size": "mel_vocab_size", "hidden_size": "text_hidden_size", } n_mel_bins: int = 80 mel_vocab_size: int = 256 text_hidden_size: int = 6144 rms_norm_eps: float = 1e-6 initializer_range: float = 0.02 @strict class InklingVisionConfig(PreTrainedConfig): model_type = "inkling_vision" base_config_key = "vision_config" attribute_map = {"num_hidden_layers": "n_layers"} text_hidden_size: int = 6144 patch_size: int = 40 temporal_patch_size: int = 2 num_channels: int = 3 hidden_size: int = 1024 num_hidden_layers: int = 24 num_attention_heads: int = 16 rms_norm_eps: float = 1e-6 initializer_range: float = 0.02 @strict class InklingConfig(PreTrainedConfig): """Top-level multimodal config (`InklingMMConfig` in the SGLang source).""" model_type = "inkling_mm_model" sub_configs = { "text_config": InklingTextConfig, "audio_config": InklingAudioConfig, "vision_config": InklingVisionConfig, } text_config: InklingTextConfig | dict | None = None audio_config: InklingAudioConfig | dict | None = None vision_config: InklingVisionConfig | dict | None = None image_token_id: int = 200054 audio_token_id: int = 200053 image_bos_token_id: int = 200005 audio_bos_token_id: int = 200020 def __post_init__(self, **kwargs): # checkpoints carry the MTP fields in a top-level `mtp_config` block mtp_config = kwargs.get("mtp_config") or {} if isinstance(self.text_config, dict): self.text_config.setdefault("num_mtp_layers", mtp_config.get("num_nextn_predict_layers")) self.text_config.setdefault("chain_hidden_post_norm", mtp_config.get("chain_hidden_post_norm", False)) self.text_config.setdefault("mtp_local_layer_ids", mtp_config.get("local_layer_ids")) if isinstance(self.audio_config, dict): self.audio_config = self.sub_configs["audio_config"](**self.audio_config) elif self.audio_config is None: self.audio_config = self.sub_configs["audio_config"]() if isinstance(self.vision_config, dict): self.vision_config = self.sub_configs["vision_config"](**self.vision_config) elif self.vision_config is None: self.vision_config = self.sub_configs["vision_config"]() if isinstance(self.text_config, dict): self.text_config = self.sub_configs["text_config"](**self.text_config) elif self.text_config is None: self.text_config = self.sub_configs["text_config"]() self.vision_config.text_hidden_size = self.text_config.hidden_size self.audio_config.text_hidden_size = self.text_config.hidden_size super().__post_init__(**kwargs) __all__ = ["InklingConfig", "InklingTextConfig", "InklingAudioConfig", "InklingVisionConfig"]