[ft_instance_hyperparameter]
max_batch_size=1 ; Use for allocate the buffer
max_seq_len=1024 ; The sequence length of position embedding table, should move to model hyper-parameter
beam_width=1 ; beam width for beam search
top_k=1 ; k value for top k sampling
top_p=0 ; p value for top p sampling
temperature=1.0 ; Use for sampling
repetition_penalty=1.0 ; Use for sampling
presence_penalty=0.0  ; Only one of repetition_penalty and presence_penalty are allowed.
tensor_para_size=1
pipeline_para_size=1
data_type=fp16
sparse=0
int8_mode=0
enable_custom_all_reduce=0
; model_name=gpt_124M
model_name=megatron_345M
; model_name=gpt2_1.5B
; model_name=megatron_1.3B_adapter
; model_name=gpt_89B
; model_name=megatron_20B
; model_name=gpt_175B
; model_name=opt_125M
; model_name=opt_350M
; model_name=bloom_560M
; model_name=bloom_7B
; model_dir=/workspace/gpt3-6.7b/99.999/1-gpu
; model_dir=/workspace/gpt3_89b/1-gpu
; model_dir=../models/megatron-models/c-model/345m/1-gpu/
model_dir=/data/models/megatron-345m-infer/1-gpu
; model_dir=/data/models/gpt2-xl-1.5b/1-gpu
; model_dir=/data/models/bloom-560m-infer/1-gpu
; model_dir=/data/models/bloom-7b-infer/1-gpu/

len_penalty=0.0
beam_search_diversity_rate=0.0
shared_contexts_ratio=1.0

[request]
request_batch_size=1   ; determine by the request
request_output_len=128   ; determine by the request
return_log_probs=false  ; return the output log probs and cumulative log probs.
context_log_probs=false ; include input contexts in the cumulative log probability computation.
remove_padding=true
context_embeddings=true

[gpt_124M]
head_num=12
size_per_head=64
vocab_size=50257
decoder_layers=12
start_id=50256
end_id=50256
inter_size=3072
num_tasks=3 ;optional
prompt_learning_start_id=50257 ;optional
prompt_learning_type=3 ;optional

;prompt learning example (optional)
[gpt_124M_task_0] ; task_name_id = 0
task_name=sentiment
prompt_length=10
;optional
[gpt_124M_task_1] ; task_name_id = 1
task_name=intent_and_slot
prompt_length=10
;optional
[gpt_124M_task_2] ; task_name_id = 2
task_name=squad
prompt_length=16

[gpt2_1.5B]
head_num=25
size_per_head=64 
vocab_size=50257
decoder_layers=48 
start_id=50256
end_id=50256
inter_size=6400  

[megatron_345M]
head_num=16
size_per_head=64
vocab_size=50304
decoder_layers=24
start_id=50256
end_id=50256
inter_size=4096

[megatron_1.3B_adapter]
head_num=32
size_per_head=64
vocab_size=50304
decoder_layers=24
start_id=50256
end_id=50256
inter_size=8192
layernorm_eps=1e-5
adapter_inter_size=1024
has_adapters=true

[megatron_6.7B]
head_num=32
size_per_head=128
vocab_size=51200
decoder_layers=32
start_id=50256
end_id=50256
inter_size=16384

[megatron_20B]
head_num=48
size_per_head=128
vocab_size=51200
decoder_layers=44
start_id=50256
end_id=50256
inter_size=24576

[gpt_175B]
head_num=96
size_per_head=128
vocab_size=51200
decoder_layers=96
start_id=50256
end_id=50256
inter_size=49152

[gpt_89B]
head_num=96
size_per_head=128
vocab_size=51200
decoder_layers=8
inter_size=49152
start_id=50256
end_id=50256

[opt_125M]
head_num=12
size_per_head=64
vocab_size=50272
decoder_layers=12
start_id=2
end_id=2
inter_size=3072
model_variant=opt-pre ;define variant structure

[opt_350M]
head_num=16
size_per_head=64
vocab_size=50272
decoder_layers=24
start_id=2
end_id=2
inter_size=4096
model_variant=opt-post

[bloom_560M]
head_num=16
size_per_head=64
vocab_size=250880
decoder_layers=24
start_id=1
end_id=2
inter_size=4096
model_variant=bloom-pre

[bloom_7B]
head_num=32
size_per_head=128
vocab_size=250880
decoder_layers=30
start_id=1
end_id=2
inter_size=16384
model_variant=bloom-pre

[bloom_175B]
head_num=112
size_per_head=128
vocab_size=250880
decoder_layers=70
start_id=1
end_id=2
inter_size=57344
model_variant=bloom-pre

[self_defined]
head_num=16
size_per_head=64
vocab_size=30000
decoder_layers=12
start_id=50256
end_id=50256
inter_size=4096

