llama : use Q4_K for attn_v for Q2_K_S when n_gqa >= 4 (#4996)

ikawrakow · Kawrakow · web-flow · commit 2b3a665d3917 · 2024-01-17T12:36:37.000+02:00
Co-authored-by: Iwan Kawrakow &lt;iwan.kawrakow@gmail.com&gt;
diff --git a/llama.cpp b/llama.cpp
@@ -8477,7 +8477,12 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty
         }
         else if (name == "token_embd.weight") new_type = GGML_TYPE_Q2_K;
     } else if (name.find("attn_v.weight") != std::string::npos) {
-        if      (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;
+        if      (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {
+            new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;
+        }
+        else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) {
+            new_type = GGML_TYPE_Q4_K;
+        }
         else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {
             new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;
         }

Original file line number	Diff line number	Diff line change
`@@ -8477,7 +8477,12 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty`
`8477`	`8477`	`}`
`8478`	`8478`	`else if (name == "token_embd.weight") new_type = GGML_TYPE_Q2_K;`
`8479`	`8479`	`} else if (name.find("attn_v.weight") != std::string::npos) {`
`8480`		`- if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;`
	`8480`	`+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {`
	`8481`	`+ new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;`
	`8482`	`+ }`
	`8483`	`+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) {`
	`8484`	`+ new_type = GGML_TYPE_Q4_K;`
	`8485`	`+ }`
`8481`	`8486`	`else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {`
`8482`	`8487`	`new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;`
`8483`	`8488`	`}`