[ft_instance_hyperparameter]
max_batch_size=16 ; Use for allocate the buffer
max_seq_len=1024 ; The sequence length of position embedding table, should move to model hyper-parameter
beam_width=1 ; beam width for beam search
top_k=1 ; k value for top k sampling
top_p=0.0 ; p value for top p sampling
temperature=1.0 ; Use for sampling
repetition_penalty=1.0 ; Use for sampling
len_penalty=0.0
beam_search_diversity_rate=0.0
; model_name=gpt_89B
model_name=megatron_6.7B
tensor_para_size=1
pipeline_para_size=1
; model_name=megatron_345M
; model_name=megatron_6.7B
; model_name=gpt_175B
; model_name=self_defined
; model_dir=./models/megatron-models/c-model/6.7b/
; model_dir=/home/scratch.bhsueh_sw_1/models/GPT/Megatron/c-model/345m/1-gpu/
; model_dir=/home/scratch.bhsueh_sw_1/models/GPT/Megatron_FP8/iter_0750000/c-model/2-gpu/
; model_dir=/workspace/FasterTransformer/models/124m/1-gpu
; model_dir=/workspace/gpt3-8.3b-calibrated/1-gpu
model_dir=/workspace/gpt3-6.7b/99.999/1-gpu
; model_dir=/workspace/gpt3-89b/1-gpu

[request]
request_batch_size=16 # determine by the request
request_input_len=16
request_output_len=16 # determine by the request

[gpt_124M]
head_num=12
size_per_head=64
vocab_size=50257
decoder_layers=12
inter_size=3072
start_id=50256
end_id=50256

[gpt_175B]
head_num=96
size_per_head=128
vocab_size=51200
decoder_layers=96

[gpt_89B]
head_num=96
size_per_head=128
vocab_size=51200
decoder_layers=48
inter_size=49152
start_id=50256
end_id=50256

[self_defined]
head_num=16
size_per_head=64
vocab_size=30000
decoder_layers=12

[megatron_345M]
head_num=16
size_per_head=64
vocab_size=50304
decoder_layers=24

[megatron_6.7B]
head_num=32
size_per_head=128
vocab_size=50688
decoder_layers=32
inter_size=16384
start_id=50256
end_id=50256

[megatron_8.3B]
head_num=64
size_per_head=64
vocab_size=50688
decoder_layers=40
