Hugging Face's logo

jabbatheduck
/
ninfer-ext-models

Image-Text-to-Text
Trellis
NInfer
English
Chinese
code
ninfer-ext
exl3
nvfp4
Mixture of Experts
expert-offload
blackwell
quantization
cuda
text-generation
vision
speculative-decoding
dflash2
gemma4
qat
Model card Files
xet
 Community
jabbatheduck commited on
Commit
eec3230
·
verified ·
1 Parent(s): 864390f

Rebuild with the indexed draft head and the fork chat template

Browse files
Files changed (1) hide show
  1. qwen3_8_27b_exl3_3p5bpw.ninfer.conversion.json +53 -14
qwen3_8_27b_exl3_3p5bpw.ninfer.conversion.json CHANGED
@@ -102,6 +102,10 @@
102
  "tokenizer_config.json": "resource/text/tokenizer_config.json",
103
  "chat_template.jinja": "resource/text/chat_template.jinja",
104
  "generation_config.json": "resource/text/generation_config.json"
 
 
 
 
105
  }
106
  },
107
  "vision": {
@@ -131,8 +135,8 @@
131
  "target": "text"
132
  }
133
  },
134
- "name": null,
135
- "output": "models/qwen3_8_27b_exl3_3p5bpw.ninfer",
136
  "device": "cpu",
137
  "torch_version": "2.14.0+cpu",
138
  "cuda_version": null,
@@ -147,20 +151,22 @@
147
  "quantized": {
148
  "path": "/mnt/storage/tmp/store-35bpw/exl3.safetensors"
149
  }
150
- }
 
151
  },
152
- "preparation_seconds": 0.5710461679846048,
153
- "parameters": 1420,
154
- "uses": 784,
155
- "objects": 1058,
156
  "formats": {
157
  "exl3_mul1": 262,
158
- "q4_g64_fp16": 54,
159
  "bf16": 583,
160
  "fp32": 96,
161
  "q6_g64_fp16": 1,
162
  "q5_g64_fp16": 54,
163
- "q8_g32_fp16": 2
 
164
  },
165
  "methods": [
166
  {
@@ -18249,15 +18255,48 @@
18249
  "shape": [
18250
  5120
18251
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18252
  }
18253
  ],
18254
- "artifact_id": "a373987a2ec34e0f86b576ca85a68e87",
18255
  "files": [
18256
  {
18257
- "path": "models/qwen3_8_27b_exl3_3p5bpw.ninfer",
18258
- "payload_bytes": 14934826496
18259
  }
18260
  ],
18261
- "payload_bytes": 14934826496,
18262
- "seconds": 25.90192453295458
18263
  }
 
102
  "tokenizer_config.json": "resource/text/tokenizer_config.json",
103
  "chat_template.jinja": "resource/text/chat_template.jinja",
104
  "generation_config.json": "resource/text/generation_config.json"
105
+ },
106
+ "proposal": {
107
+ "domain": "indexed",
108
+ "rows": 131072
109
  }
110
  },
111
  "vision": {
 
135
  "target": "text"
136
  }
137
  },
138
+ "name": "qwen3.8-27b",
139
+ "output": "/mnt/storage/tmp/rb-3p5bpw.ninfer",
140
  "device": "cpu",
141
  "torch_version": "2.14.0+cpu",
142
  "cuda_version": null,
 
151
  "quantized": {
152
  "path": "/mnt/storage/tmp/store-35bpw/exl3.safetensors"
153
  }
154
+ },
155
+ "ranking": "/mnt/storage/Projects/ninfer-ext/tools/freq_corpus/fixtures/ranking/ranking.train.counts.i64"
156
  },
157
+ "preparation_seconds": 0.9912331649975386,
158
+ "parameters": 1422,
159
+ "uses": 785,
160
+ "objects": 1060,
161
  "formats": {
162
  "exl3_mul1": 262,
163
+ "q4_g64_fp16": 55,
164
  "bf16": 583,
165
  "fp32": 96,
166
  "q6_g64_fp16": 1,
167
  "q5_g64_fp16": 54,
168
+ "q8_g32_fp16": 2,
169
+ "int32": 1
170
  },
171
  "methods": [
172
  {
 
18255
  "shape": [
18256
  5120
18257
  ]
18258
+ },
18259
+ {
18260
+ "object": "weight/001052",
18261
+ "parameters": [
18262
+ "proposal/head"
18263
+ ],
18264
+ "sources": [
18265
+ "rows(/mnt/storage/models/qwen3.8/full:lm_head.weight, 131072 spans, 131072 rows)"
18266
+ ],
18267
+ "method": "grouped_absmax",
18268
+ "method_parameters": {},
18269
+ "format": "q4_g64_fp16",
18270
+ "layout": "row_split_k128_v1",
18271
+ "shape": [
18272
+ 131072,
18273
+ 5120
18274
+ ]
18275
+ },
18276
+ {
18277
+ "object": "weight/001053",
18278
+ "parameters": [
18279
+ "proposal/token_ids"
18280
+ ],
18281
+ "sources": [
18282
+ "shortlist(/mnt/storage/Projects/ninfer-ext/tools/freq_corpus/fixtures/ranking/ranking.train.counts.i64)"
18283
+ ],
18284
+ "method": "cast_direct",
18285
+ "method_parameters": {},
18286
+ "format": "int32",
18287
+ "layout": "contiguous_le_v1",
18288
+ "shape": [
18289
+ 131072
18290
+ ]
18291
  }
18292
  ],
18293
+ "artifact_id": "6cdfba7635794d12b5f691a0e221ffce",
18294
  "files": [
18295
  {
18296
+ "path": "/mnt/storage/tmp/rb-3p5bpw.ninfer",
18297
+ "payload_bytes": 15291866624
18298
  }
18299
  ],
18300
+ "payload_bytes": 15291866624,
18301
+ "seconds": 46.222997952005244
18302
  }