MAGI-7B-MoE — initial architecture checkpoint

PROBE_NOT_BASE.json

@@ -0,0 +1,10 @@
+{
+  "claim_class": "RUNTIME_PROBE_NOT_BASE",
+  "final_loss": 3.2434604167938232,
+  "model": "MAGI-7B-MoE",
+  "production_checkpoint": false,
+  "status": "OK",
+  "steps": 50,
+  "tokenizer": "magi_bringup_8k_v0.1",
+  "tokenizer_note": "bringup_8k \u2014 forbidden for BASE / MAGI_TOKENIZER_V1 path"
+}

README.md

@@ -0,0 +1,110 @@
+---
+license: other
+tags:
+  - magi
+  - mixture-of-experts
+  - moe
+  - language-model
+  - transformer
+  - nullxes
+---
+
+# MAGI-7B-MoE
+
+**MAGI-7B-MoE** is an experimental sparse Mixture-of-Experts language model developed by **NULLXES** as part of the MAGI model family.
+
+MAGI is a research program focused on sparse model architectures, expert routing, scalable training systems, and synthetic intelligence.
+
+## Architecture
+
+MAGI-7B-MoE uses a decoder-only sparse Mixture-of-Experts architecture.
+
+Current architecture configuration:
+
+- Decoder-only Transformer
+- Sparse Mixture-of-Experts
+- 64 routed experts
+- 1 shared expert
+- Top-4 expert routing
+- 3 dense layers
+- 29 MoE layers
+
+The complete architecture configuration included with this release is available in:
+
+`magi_7b_moe_v0.1.yaml`
+
+## Release status
+
+This repository contains an **early MAGI architecture checkpoint**.
+
+The checkpoint was produced to validate:
+
+- MAGI model construction
+- CUDA/H200 execution
+- forward and backward passes
+- sparse expert routing
+- router behavior
+- optimization
+- gradient flow
+- checkpoint serialization
+- end-to-end MAGI training infrastructure
+
+This checkpoint is **not a production-ready base model**.
+
+It uses the temporary MAGI bring-up tokenizer:
+
+`magi_bringup_8k_v0.1`
+
+It must not be represented as a checkpoint trained with the final `MAGI_TOKENIZER_V1`.
+
+The internal artifact metadata is retained in `PROBE_NOT_BASE.json` for reproducibility and release provenance.
+
+## Initial training validation
+
+Hardware:
+
+- NVIDIA H200
+- CUDA
+- BF16 supported
+
+Training configuration:
+
+- Steps: 50
+- Sequence length: 1,024
+- Batch size: 1
+- Tokens processed: 51,200
+- Learning rate: 3e-4
+
+Observed training result:
+
+- Initial loss: 9.4176
+- Final loss: 3.2435
+- Loss delta: 6.1741
+- Final dead experts: 0
+- Final router entropy: 4.0884
+- Final expert imbalance ratio: 2.9494
+- Mean throughput: ~1,392.7 tokens/s
+
+Training status:
+
+`OK`
+
+These values describe an architecture bring-up run and should not be interpreted as downstream model-quality benchmarks.
+
+## Checkpoint
+
+Weights are distributed as:
+
+`model.safetensors`
+
+Optimizer state is intentionally not included in this public package.
+
+## MAGI
+
+MAGI is developed by NULLXES.
+
+The broader MAGI program explores scalable sparse neural architectures and training infrastructure for synthetic intelligence systems.
+
+---
+
+© 2026 NULLXES

magi_7b_moe_v0.1.yaml

@@ -0,0 +1,111 @@
+# MAGI-7B-MoE v0.1 — SCALE_0
+# First real member of the MAGI Sparse-MoE family (recipe prover before 35B).
+# NOT a disposable bring-up toy. BASE training waits for MAGI_TOKENIZER_V1.
+# Dense MAGI-7B is A/B reference only — not the scaling path.
+
+meta:
+  name: MAGI-7B-MoE
+  version: v0.1
+  program_parent: MAGI-5.5GTBS
+  status: SCALE_0
+  role: magi_family_scale_0_recipe_prover
+  from_zero: true
+  model_class: moe_decoder
+  primary_role: sparse_moe_scale_0
+  parent_family: magi_35b_moe
+  topology_source: configs/magi_35b_moe_v0.1.yaml
+  claim_class_topology: CALCULATED
+  hf_architecture: MagiForCausalLM
+  canonical_training_target: true
+  scaling_path: true
+  production_pretraining_allowed: false
+  blocked_until: MAGI_TOKENIZER_V1
+  next_phase_ref: data/program/MAGI_NEXT_PHASE_v0.1.yaml
+  notes: "Train BASE only after tokenizer freeze. bringup_8k is runtime probe only."
+
+architecture:
+  type: decoder_only_prenorm_sparse_moe
+  d_model: 2048
+  n_layers: 32
+  n_dense_layers: 3
+  n_moe_layers: 29
+  n_heads: 16
+  n_kv_heads: 8
+  d_head: 128
+  d_ff_dense: 5632
+  d_ff_expert: 512
+  n_routed_experts: 64
+  n_shared_experts: 1
+  top_k: 4
+  vocab_size: 8192
+  tied_embeddings: true
+  norm: rmsnorm
+  rmsnorm_eps: 1.0e-6
+  activation: swiglu
+  positional: rope
+  rope_theta: 1000000.0
+  bias: false
+  attention: gqa
+  train_context: 4096
+  infer_context: 8192
+  vocab_note: "8192 is temporary architecture default for meta/runtime probes; BASE retargets to MAGI_TOKENIZER_V1 vocab after freeze"
+
+moe:
+  gate: sigmoid_normalize_topk
+  gate_dtype: fp32
+  capacity_factor_train_min: 1.0
+  capacity_factor_train_max: 1.25
+  load_balance: aux_loss_free_bias
+  router_z_loss_coeff: 1.0e-5
+  bias_update_rate: 1.0e-3
+  hierarchical_routing_hooks: true
+
+init:
+  emb_std: 0.02
+  residual_std_formula: "0.02 / sqrt(2 * n_layers)"
+  claim_class: HYPOTHESIS
+
+precision:
+  proxy: bf16
+  production: bf16
+  master_weights: fp32
+  router_loss_softmax: fp32
+  loss: fp32
+
+hardware:
+  target: H200_SXM
+  proxy_gpu: H200
+  gpu_vram_gb: 141
+  dtype: bfloat16
+  recommended_batch: 1
+  recommended_seq: 2048
+  notes: "Sparse MoE; Adam stores ALL experts (~96GB). Active/token ~1B."
+
+tokenizer:
+  production_id: MAGI_TOKENIZER_V1
+  production_artifact: tokenizer/artifacts/magi_tokenizer_v1.json
+  production_status: MISSING_PENDING_SWEEP
+  bringup_probe_id: magi_bringup_8k_v0.1
+  bringup_probe_config: configs/tokenizer_bringup_8k_v0.1.yaml
+  bringup_probe_artifact: tokenizer/artifacts/magi_bringup_8k_v0.1.json
+  bringup_pretraining_allowed: false
+  bringup_checkpoint_allowed: false
+
+training_plan:
+  r0_tokens: "1e9-3e9"
+  r1_tokens: "1e10-3e10"
+  flagship_after_r1: configs/magi_35b_moe_v0.1.yaml
+
+hf:
+  model_type: magi
+  architecture: MagiForCausalLM
+  use_cache: true
+  return_dict: true
+  tie_word_embeddings: true
+
+expected_param_count:
+  total: 6456870912
+  active_per_token: 966526976
+  cognitive_cycle_1pass: 966526976
+  cognitive_cycle_2pass: 1933053952
+  claim_class: CALCULATED

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33d85ed2191bf4431016792e8e242e781e5aa6f9bb89ba6d5363cbe9cc93fdbf
+size 25828190096

train_meta.json

@@ -0,0 +1,107 @@
+{
+  "checkpoint": {
+    "config_path": "/workspace/NULLXES-MAGI-5.5GTBS/configs/magi_7b_moe_v0.1.yaml",
+    "consumed_tokens": 51200,
+    "global_step": 50,
+    "loss": 3.2434604167938232,
+    "model_name": "MAGI-7B-MoE",
+    "step": 50,
+    "tokenizer_id": "magi_bringup_8k_v0.1"
+  },
+  "manifest": {
+    "active_parameters_per_token": null,
+    "artifacts": {
+      "model": {
+        "bytes": 25828190096,
+        "kind": "model",
+        "path": "model.safetensors",
+        "sha256": "33d85ed2191bf4431016792e8e242e781e5aa6f9bb89ba6d5363cbe9cc93fdbf"
+      },
+      "optimizer": {
+        "bytes": 51660189376,
+        "kind": "optimizer",
+        "path": "optimizer.pt",
+        "sha256": "19b47e3c092027e028720be82556d8a05d41382e4e6defb9d46febf3a05c8e83"
+      },
+      "rng": {
+        "bytes": 10240,
+        "kind": "rng",
+        "path": "rng.pt",
+        "sha256": "bcf2ed3771d2fcc72e66ebc1e516684f616af72ae6deabc1300e9d38ac461e89"
+      }
+    },
+    "checkpoint_format": "magi_single_gpu_v0.3",
+    "checkpoint_schema_version": "1",
+    "compute_dtype": "bf16",
+    "config_path": "/workspace/NULLXES-MAGI-5.5GTBS/configs/magi_7b_moe_v0.1.yaml",
+    "config_sha256": "b1ce9fe752e5b7ffa7771218cf636fd079d89ee83c2aebf5c2c04ebdf0cdca8a",
+    "consumed_samples": 50,
+    "consumed_tokens": 51200,
+    "dataset_manifest_id": "MAGI_DATA_MANIFEST_v0.1",
+    "dataset_manifest_sha256": "22fbaf1f8e36b823d8e8db50aa4b47e70035eb6f6571440a56285e3313b552fe",
+    "global_step": 50,
+    "manifest_version": "1.0",
+    "mixture_id": "base_mixture_v0_1",
+    "model_architecture": "MagiForCausalLM",
+    "model_name": "MAGI-7B-MoE",
+    "model_revision": "v0.1",
+    "parallelism": {
+      "cp": 1,
+      "dp": 1,
+      "ep": 1,
+      "pp": 1,
+      "tp": 1
+    },
+    "parameter_dtype": "float32",
+    "run_id": "c6a1d557-e620-4704-9102-f11f31637801",
+    "state_sections": [
+      "model",
+      "optimizer",
+      "rng",
+      "config",
+      "tokenizer"
+    ],
+    "tokenizer_id": "magi_bringup_8k_v0.1",
+    "tokenizer_sha256": "ac8cc35bf57b99b6c5b848600d45a1fc1875bc56fd9817b16f041ea36ca2911e",
+    "total_parameters": 6456870912,
+    "train_config_sha256": "5a3be59ddc2e3155205fbec690a345f3253df31dfbca2301d329ddbf60a95cd5",
+    "world_size": 1
+  },
+  "metrics": {
+    "consumed_tokens": 51200,
+    "final_dead_experts": 0,
+    "final_grad_norm": 0.9999998242570162,
+    "final_imbalance_ratio": 2.9493534564971924,
+    "final_router_entropy": 4.088359355926514,
+    "first_loss": 9.417561531066895,
+    "global_step": 50,
+    "last_loss": 3.2434604167938232,
+    "loss_delta": 6.174101114273071,
+    "loss_improved": true,
+    "mean_tok_s": 1392.7007314726904,
+    "status": "OK",
+    "steps": 50
+  },
+  "optimizer": "optimizer.pt",
+  "rng": "rng.pt",
+  "tokenizer_artifact": "/workspace/NULLXES-MAGI-5.5GTBS/tokenizer/artifacts/magi_bringup_8k_v0.1.json",
+  "tokenizer_id": "magi_bringup_8k_v0.1",
+  "tokenizer_path": "/workspace/NULLXES-MAGI-5.5GTBS/tokenizer/artifacts/magi_bringup_8k_v0.1.json",
+  "train_config": {
+    "amp_dtype": "bf16",
+    "batch_size": 1,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "eps": 1e-08,
+    "lr": 0.0003,
+    "max_grad_norm": 1.0,
+    "model_config": "/workspace/NULLXES-MAGI-5.5GTBS/configs/magi_7b_moe_v0.1.yaml",
+    "profile": "/workspace/NULLXES-MAGI-5.5GTBS/configs/magi_7b_train_h200_v0.1.yaml",
+    "seed": 42,
+    "seq_len": 1024,
+    "steps": 50,
+    "use_amp": true,
+    "weight_decay": 0.1
+  },
+  "weights": "model.safetensors"
+}