olive-recipes/microsoft-Phi-4-mini-instruct/olive/mixed.json at main · microsoft/olive-recipes · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
{
    "input_model": {
        "type": "HfModel",
        "model_path": "microsoft/Phi-4-mini-instruct"
    },
    "systems": {
        "local_system": {
            "type": "LocalSystem",
            "accelerators": [ { "execution_providers": [ "CUDAExecutionProvider" ] } ]
        }
    },
    "evaluators": {
        "evaluator": {
            "type": "LMEvaluator",
            "tasks": [ "arc_challenge", "arc_easy", "mmlu", "hellaswag", "mmlu_stem", "openbookqa" ],
            "batch_size": 32,
            "provider_options": { "enable_skip_layer_norm_strict_mode": "1" }
        }
    },
    "passes": {
        "q": {
            "type": "Quarot"
        },
        "s": {
            "type": "SelectiveMixedPrecision",
            "algorithm": "k_quant_mixed"
        },
        "g": {
            "type": "gptq",
            "bits": 4,
            "sym": true,
            "group_size": 32,
            "lm_head": true
        },
        "m": {
            "type": "ModelBuilder",
            "precision": "int4"
        }
    },
    "evaluator": "evaluator",
    "evaluate_input_model": false,
    "target": "local_system",
    "log_severity_level": 1,
    "output_dir": "models/qwen2.5-1.5b-mixed",
    "cache_dir": "cache",
    "no_artifacts": true
}