opt_example.sh 567 Bytes
Newer Older
Casper Hansen's avatar
Casper Hansen committed
1
MODEL=facebook/opt-6.7b
Ji Lin's avatar
Ji Lin committed
2
3

# run AWQ search (optional; we provided the pre-computed results)
Casper Hansen's avatar
Casper Hansen committed
4
5
6
python -m awq.entry --entry_type search \
    --model_path $MODEL \
    --search_path $MODEL-awq
Ji Lin's avatar
Ji Lin committed
7
8

# generate real quantized weights (w4)
Casper Hansen's avatar
Casper Hansen committed
9
10
11
12
python -m awq.entry --entry_type quant \
    --model_path $MODEL \
    --search_path $MODEL-awq/awq_model_search_result.pt \
    --quant_path $MODEL-awq
Ji Lin's avatar
Ji Lin committed
13
14

# load and evaluate the real quantized model (smaller gpu memory usage)
Casper Hansen's avatar
Casper Hansen committed
15
16
17
python -m awq.entry --entry_type perplexity \
    --quant_path $MODEL-awq \
    --quant_file awq_model_w4_g128.pt