[ft_instance_hyperparameter]
max_batch_size=8 ; Use for allocate the buffer
max_seq_len=128 ; The sequence length of position embedding table, should move to model hyper-parameter
beam_width=1 ; beam width for beam search
top_k=0 ; k value for top k sampling
top_p=0.5 ; p value for top p sampling
temperature=1.0 ; Use for sampling
repetition_penalty=2.0 ; Use for sampling
presence_penalty=0.0  ; Only one of repetition_penalty and presence_penalty are allowed.
len_penalty=0.0
beam_search_diversity_rate=0.0
data_type=fp16
sparse=0
model_name=gpt_124M
; model_name=megatron_345M
; model_name=megatron_6.7B
; model_name=gpt_175B
; model_name=self_defined
; model_dir=./models/megatron-models/c-model/6.7b/
model_dir=models/openai-gpt-models/c-model/124m/1-gpu/
shared_contexts_ratio=1.0

[request]
request_batch_size=8    ; determine by the request
request_output_len=32   ; determine by the request
return_log_probs=false  ; return the output log probs and cumulative log probs.
context_log_probs=false ; include input contexts in the cumulative log probability computation.

[gpt_124M]
head_num=12
size_per_head=64
vocab_size=50257
decoder_layers=12

[gpt_175B]
head_num=96
size_per_head=128
vocab_size=51200
decoder_layers=96

[self_defined]
head_num=16
size_per_head=64
vocab_size=30000
decoder_layers=12

[megatron_345M]
head_num=16
size_per_head=64
vocab_size=50304
decoder_layers=24

[megatron_6.7B]
head_num=32
size_per_head=128
vocab_size=51200
decoder_layers=32

