openai.go 18.7 KB
Newer Older
1
// openai package provides core transformation logic for partial compatibility with the OpenAI REST API
2
3
4
package openai

import (
5
	"encoding/base64"
6
	"encoding/json"
Michael Yang's avatar
lint  
Michael Yang committed
7
	"errors"
8
	"fmt"
royjhan's avatar
royjhan committed
9
	"log/slog"
10
11
	"math/rand"
	"net/http"
12
	"slices"
13
	"strings"
14
15
	"time"

16
	"github.com/ollama/ollama/api"
17
	"github.com/ollama/ollama/types/model"
18
19
)

20
21
var finishReasonToolCalls = "tool_calls"

22
type Error struct {
23
24
25
26
	Message string  `json:"message"`
	Type    string  `json:"type"`
	Param   any     `json:"param"`
	Code    *string `json:"code"`
27
28
29
30
31
32
33
}

type ErrorResponse struct {
	Error Error `json:"error"`
}

type Message struct {
34
35
36
37
38
39
	Role       string     `json:"role"`
	Content    any        `json:"content"`
	Reasoning  string     `json:"reasoning,omitempty"`
	ToolCalls  []ToolCall `json:"tool_calls,omitempty"`
	Name       string     `json:"name,omitempty"`
	ToolCallID string     `json:"tool_call_id,omitempty"`
40
41
42
43
44
45
46
47
48
49
50
51
52
53
}

type Choice struct {
	Index        int     `json:"index"`
	Message      Message `json:"message"`
	FinishReason *string `json:"finish_reason"`
}

type ChunkChoice struct {
	Index        int     `json:"index"`
	Delta        Message `json:"delta"`
	FinishReason *string `json:"finish_reason"`
}

54
55
56
57
58
59
type CompleteChunkChoice struct {
	Text         string  `json:"text"`
	Index        int     `json:"index"`
	FinishReason *string `json:"finish_reason"`
}

60
61
62
63
64
65
66
type Usage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}

type ResponseFormat struct {
67
68
69
70
71
	Type       string      `json:"type"`
	JsonSchema *JsonSchema `json:"json_schema,omitempty"`
}

type JsonSchema struct {
72
	Schema json.RawMessage `json:"schema"`
73
74
}

75
type EmbedRequest struct {
76
77
78
	Input      any    `json:"input"`
	Model      string `json:"model"`
	Dimensions int    `json:"dimensions,omitempty"`
79
80
}

81
82
83
84
type StreamOptions struct {
	IncludeUsage bool `json:"include_usage"`
}

Michael Yang's avatar
Michael Yang committed
85
type Reasoning struct {
86
	Effort string `json:"effort,omitempty"`
Michael Yang's avatar
Michael Yang committed
87
88
}

89
90
91
92
type ChatCompletionRequest struct {
	Model            string          `json:"model"`
	Messages         []Message       `json:"messages"`
	Stream           bool            `json:"stream"`
93
	StreamOptions    *StreamOptions  `json:"stream_options"`
94
95
96
97
98
	MaxTokens        *int            `json:"max_tokens"`
	Seed             *int            `json:"seed"`
	Stop             any             `json:"stop"`
	Temperature      *float64        `json:"temperature"`
	FrequencyPenalty *float64        `json:"frequency_penalty"`
99
	PresencePenalty  *float64        `json:"presence_penalty"`
100
101
	TopP             *float64        `json:"top_p"`
	ResponseFormat   *ResponseFormat `json:"response_format"`
royjhan's avatar
royjhan committed
102
	Tools            []api.Tool      `json:"tools"`
Michael Yang's avatar
Michael Yang committed
103
	Reasoning        *Reasoning      `json:"reasoning,omitempty"`
104
	ReasoningEffort  *string         `json:"reasoning_effort,omitempty"`
Devon Rifkin's avatar
Devon Rifkin committed
105
	DebugRenderOnly  bool            `json:"_debug_render_only"`
106
107
108
}

type ChatCompletion struct {
Devon Rifkin's avatar
Devon Rifkin committed
109
110
111
112
113
114
115
116
	Id                string         `json:"id"`
	Object            string         `json:"object"`
	Created           int64          `json:"created"`
	Model             string         `json:"model"`
	SystemFingerprint string         `json:"system_fingerprint"`
	Choices           []Choice       `json:"choices"`
	Usage             Usage          `json:"usage,omitempty"`
	DebugInfo         *api.DebugInfo `json:"_debug_info,omitempty"`
117
118
119
120
121
122
123
124
125
}

type ChatCompletionChunk struct {
	Id                string        `json:"id"`
	Object            string        `json:"object"`
	Created           int64         `json:"created"`
	Model             string        `json:"model"`
	SystemFingerprint string        `json:"system_fingerprint"`
	Choices           []ChunkChoice `json:"choices"`
126
	Usage             *Usage        `json:"usage,omitempty"`
127
128
}

129
130
// TODO (https://github.com/ollama/ollama/issues/5259): support []string, []int and [][]int
type CompletionRequest struct {
131
132
133
134
135
136
137
138
139
140
141
142
	Model            string         `json:"model"`
	Prompt           string         `json:"prompt"`
	FrequencyPenalty float32        `json:"frequency_penalty"`
	MaxTokens        *int           `json:"max_tokens"`
	PresencePenalty  float32        `json:"presence_penalty"`
	Seed             *int           `json:"seed"`
	Stop             any            `json:"stop"`
	Stream           bool           `json:"stream"`
	StreamOptions    *StreamOptions `json:"stream_options"`
	Temperature      *float32       `json:"temperature"`
	TopP             float32        `json:"top_p"`
	Suffix           string         `json:"suffix"`
Devon Rifkin's avatar
Devon Rifkin committed
143
	DebugRenderOnly  bool           `json:"_debug_render_only"`
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
}

type Completion struct {
	Id                string                `json:"id"`
	Object            string                `json:"object"`
	Created           int64                 `json:"created"`
	Model             string                `json:"model"`
	SystemFingerprint string                `json:"system_fingerprint"`
	Choices           []CompleteChunkChoice `json:"choices"`
	Usage             Usage                 `json:"usage,omitempty"`
}

type CompletionChunk struct {
	Id                string                `json:"id"`
	Object            string                `json:"object"`
	Created           int64                 `json:"created"`
	Choices           []CompleteChunkChoice `json:"choices"`
	Model             string                `json:"model"`
	SystemFingerprint string                `json:"system_fingerprint"`
163
	Usage             *Usage                `json:"usage,omitempty"`
164
165
}

royjhan's avatar
royjhan committed
166
167
type ToolCall struct {
	ID       string `json:"id"`
168
	Index    int    `json:"index"`
royjhan's avatar
royjhan committed
169
170
171
172
173
174
175
	Type     string `json:"type"`
	Function struct {
		Name      string `json:"name"`
		Arguments string `json:"arguments"`
	} `json:"function"`
}

176
177
178
179
180
181
182
type Model struct {
	Id      string `json:"id"`
	Object  string `json:"object"`
	Created int64  `json:"created"`
	OwnedBy string `json:"owned_by"`
}

183
184
185
186
187
188
type Embedding struct {
	Object    string    `json:"object"`
	Embedding []float32 `json:"embedding"`
	Index     int       `json:"index"`
}

189
190
191
192
193
type ListCompletion struct {
	Object string  `json:"object"`
	Data   []Model `json:"data"`
}

194
type EmbeddingList struct {
195
196
197
198
199
200
201
202
203
	Object string         `json:"object"`
	Data   []Embedding    `json:"data"`
	Model  string         `json:"model"`
	Usage  EmbeddingUsage `json:"usage,omitempty"`
}

type EmbeddingUsage struct {
	PromptTokens int `json:"prompt_tokens"`
	TotalTokens  int `json:"total_tokens"`
204
205
}

206
207
208
209
210
211
212
213
214
215
216
217
218
219
func NewError(code int, message string) ErrorResponse {
	var etype string
	switch code {
	case http.StatusBadRequest:
		etype = "invalid_request_error"
	case http.StatusNotFound:
		etype = "not_found_error"
	default:
		etype = "api_error"
	}

	return ErrorResponse{Error{Type: etype, Message: message}}
}

220
221
// ToUsage converts an api.ChatResponse to Usage
func ToUsage(r api.ChatResponse) Usage {
222
	return Usage{
223
224
225
		PromptTokens:     r.Metrics.PromptEvalCount,
		CompletionTokens: r.Metrics.EvalCount,
		TotalTokens:      r.Metrics.PromptEvalCount + r.Metrics.EvalCount,
226
227
228
	}
}

royjhan's avatar
royjhan committed
229
230
231
232
233
234
235
236
237
func toolCallId() string {
	const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
	b := make([]byte, 8)
	for i := range b {
		b[i] = letterBytes[rand.Intn(len(letterBytes))]
	}
	return "call_" + strings.ToLower(string(b))
}

238
239
240
func toToolCalls(tc []api.ToolCall) []ToolCall {
	toolCalls := make([]ToolCall, len(tc))
	for i, tc := range tc {
royjhan's avatar
royjhan committed
241
242
243
		toolCalls[i].ID = toolCallId()
		toolCalls[i].Type = "function"
		toolCalls[i].Function.Name = tc.Function.Name
244
		toolCalls[i].Index = tc.Function.Index
royjhan's avatar
royjhan committed
245
246
247
248
249
250
251
252
253

		args, err := json.Marshal(tc.Function.Arguments)
		if err != nil {
			slog.Error("could not marshall function arguments to json", "error", err)
			continue
		}

		toolCalls[i].Function.Arguments = string(args)
	}
254
255
	return toolCalls
}
royjhan's avatar
royjhan committed
256

257
258
// ToChatCompletion converts an api.ChatResponse to ChatCompletion
func ToChatCompletion(id string, r api.ChatResponse) ChatCompletion {
259
	toolCalls := toToolCalls(r.Message.ToolCalls)
260
261
262
263
264
265
266
	return ChatCompletion{
		Id:                id,
		Object:            "chat.completion",
		Created:           r.CreatedAt.Unix(),
		Model:             r.Model,
		SystemFingerprint: "fp_ollama",
		Choices: []Choice{{
267
			Index:   0,
Michael Yang's avatar
Michael Yang committed
268
			Message: Message{Role: r.Message.Role, Content: r.Message.Content, ToolCalls: toolCalls, Reasoning: r.Message.Thinking},
269
			FinishReason: func(reason string) *string {
270
271
272
				if len(toolCalls) > 0 {
					reason = "tool_calls"
				}
273
274
275
276
277
				if len(reason) > 0 {
					return &reason
				}
				return nil
			}(r.DoneReason),
278
		}}, Usage: ToUsage(r),
Devon Rifkin's avatar
Devon Rifkin committed
279
		DebugInfo: r.DebugInfo,
280
281
282
	}
}

283
284
// ToChunk converts an api.ChatResponse to ChatCompletionChunk
func ToChunk(id string, r api.ChatResponse, toolCallSent bool) ChatCompletionChunk {
285
	toolCalls := toToolCalls(r.Message.ToolCalls)
286
287
288
289
290
291
	return ChatCompletionChunk{
		Id:                id,
		Object:            "chat.completion.chunk",
		Created:           time.Now().Unix(),
		Model:             r.Model,
		SystemFingerprint: "fp_ollama",
292
293
		Choices: []ChunkChoice{{
			Index: 0,
Michael Yang's avatar
Michael Yang committed
294
			Delta: Message{Role: "assistant", Content: r.Message.Content, ToolCalls: toolCalls, Reasoning: r.Message.Thinking},
295
296
			FinishReason: func(reason string) *string {
				if len(reason) > 0 {
Michael Yang's avatar
Michael Yang committed
297
					if toolCallSent || len(toolCalls) > 0 {
298
299
						return &finishReasonToolCalls
					}
300
301
302
303
304
					return &reason
				}
				return nil
			}(r.DoneReason),
		}},
305
306
307
	}
}

308
309
// ToUsageGenerate converts an api.GenerateResponse to Usage
func ToUsageGenerate(r api.GenerateResponse) Usage {
310
	return Usage{
311
312
313
		PromptTokens:     r.Metrics.PromptEvalCount,
		CompletionTokens: r.Metrics.EvalCount,
		TotalTokens:      r.Metrics.PromptEvalCount + r.Metrics.EvalCount,
314
315
316
	}
}

317
318
// ToCompletion converts an api.GenerateResponse to Completion
func ToCompletion(id string, r api.GenerateResponse) Completion {
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
	return Completion{
		Id:                id,
		Object:            "text_completion",
		Created:           r.CreatedAt.Unix(),
		Model:             r.Model,
		SystemFingerprint: "fp_ollama",
		Choices: []CompleteChunkChoice{{
			Text:  r.Response,
			Index: 0,
			FinishReason: func(reason string) *string {
				if len(reason) > 0 {
					return &reason
				}
				return nil
			}(r.DoneReason),
		}},
335
		Usage: ToUsageGenerate(r),
336
337
338
	}
}

339
340
// ToCompleteChunk converts an api.GenerateResponse to CompletionChunk
func ToCompleteChunk(id string, r api.GenerateResponse) CompletionChunk {
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
	return CompletionChunk{
		Id:                id,
		Object:            "text_completion",
		Created:           time.Now().Unix(),
		Model:             r.Model,
		SystemFingerprint: "fp_ollama",
		Choices: []CompleteChunkChoice{{
			Text:  r.Response,
			Index: 0,
			FinishReason: func(reason string) *string {
				if len(reason) > 0 {
					return &reason
				}
				return nil
			}(r.DoneReason),
		}},
	}
}

360
361
// ToListCompletion converts an api.ListResponse to ListCompletion
func ToListCompletion(r api.ListResponse) ListCompletion {
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
	var data []Model
	for _, m := range r.Models {
		data = append(data, Model{
			Id:      m.Name,
			Object:  "model",
			Created: m.ModifiedAt.Unix(),
			OwnedBy: model.ParseName(m.Name).Namespace,
		})
	}

	return ListCompletion{
		Object: "list",
		Data:   data,
	}
}

378
379
// ToEmbeddingList converts an api.EmbedResponse to EmbeddingList
func ToEmbeddingList(model string, r api.EmbedResponse) EmbeddingList {
380
381
382
383
384
385
386
387
388
389
390
391
392
393
	if r.Embeddings != nil {
		var data []Embedding
		for i, e := range r.Embeddings {
			data = append(data, Embedding{
				Object:    "embedding",
				Embedding: e,
				Index:     i,
			})
		}

		return EmbeddingList{
			Object: "list",
			Data:   data,
			Model:  model,
394
395
396
397
			Usage: EmbeddingUsage{
				PromptTokens: r.PromptEvalCount,
				TotalTokens:  r.PromptEvalCount,
			},
398
399
400
401
402
403
		}
	}

	return EmbeddingList{}
}

404
405
// ToModel converts an api.ShowResponse to Model
func ToModel(r api.ShowResponse, m string) Model {
406
407
408
409
410
411
412
413
	return Model{
		Id:      m,
		Object:  "model",
		Created: r.ModifiedAt.Unix(),
		OwnedBy: model.ParseName(m).Namespace,
	}
}

414
415
// FromChatRequest converts a ChatCompletionRequest to api.ChatRequest
func FromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {
416
417
	var messages []api.Message
	for _, msg := range r.Messages {
418
419
420
421
422
423
424
		toolName := ""
		if strings.ToLower(msg.Role) == "tool" {
			toolName = msg.Name
			if toolName == "" && msg.ToolCallID != "" {
				toolName = nameFromToolCallID(r.Messages, msg.ToolCallID)
			}
		}
425
426
		switch content := msg.Content.(type) {
		case string:
427
428
429
430
			toolCalls, err := fromCompletionToolCall(msg.ToolCalls)
			if err != nil {
				return nil, err
			}
431
			messages = append(messages, api.Message{Role: msg.Role, Content: content, Thinking: msg.Reasoning, ToolCalls: toolCalls, ToolName: toolName})
432
433
434
435
		case []any:
			for _, c := range content {
				data, ok := c.(map[string]any)
				if !ok {
Michael Yang's avatar
lint  
Michael Yang committed
436
					return nil, errors.New("invalid message format")
437
438
439
440
441
				}
				switch data["type"] {
				case "text":
					text, ok := data["text"].(string)
					if !ok {
Michael Yang's avatar
lint  
Michael Yang committed
442
						return nil, errors.New("invalid message format")
443
					}
444
					messages = append(messages, api.Message{Role: msg.Role, Content: text})
445
446
447
448
				case "image_url":
					var url string
					if urlMap, ok := data["image_url"].(map[string]any); ok {
						if url, ok = urlMap["url"].(string); !ok {
Michael Yang's avatar
lint  
Michael Yang committed
449
							return nil, errors.New("invalid message format")
450
451
452
						}
					} else {
						if url, ok = data["image_url"].(string); !ok {
Michael Yang's avatar
lint  
Michael Yang committed
453
							return nil, errors.New("invalid message format")
454
455
456
						}
					}

457
					types := []string{"jpeg", "jpg", "png", "webp"}
458
					valid := false
459
460
461
462
463
					// support blank mime type to match api/chat taking just unadorned base64
					if strings.HasPrefix(url, "data:;base64,") {
						url = strings.TrimPrefix(url, "data:;base64,")
						valid = true
					}
464
465
466
467
468
469
470
471
472
473
					for _, t := range types {
						prefix := "data:image/" + t + ";base64,"
						if strings.HasPrefix(url, prefix) {
							url = strings.TrimPrefix(url, prefix)
							valid = true
							break
						}
					}

					if !valid {
Michael Yang's avatar
lint  
Michael Yang committed
474
						return nil, errors.New("invalid image input")
475
476
477
478
					}

					img, err := base64.StdEncoding.DecodeString(url)
					if err != nil {
Michael Yang's avatar
lint  
Michael Yang committed
479
						return nil, errors.New("invalid message format")
480
					}
481
482

					messages = append(messages, api.Message{Role: msg.Role, Images: []api.ImageData{img}})
483
				default:
Michael Yang's avatar
lint  
Michael Yang committed
484
					return nil, errors.New("invalid message format")
485
486
				}
			}
487
488
489
490
491
492
493
494
			// since we might have added multiple messages above, if we have tools
			// calls we'll add them to the last message
			if len(messages) > 0 && len(msg.ToolCalls) > 0 {
				toolCalls, err := fromCompletionToolCall(msg.ToolCalls)
				if err != nil {
					return nil, err
				}
				messages[len(messages)-1].ToolCalls = toolCalls
495
496
497
				if toolName != "" {
					messages[len(messages)-1].ToolName = toolName
				}
498
				messages[len(messages)-1].Thinking = msg.Reasoning
499
			}
500
		default:
501
			// content is only optional if tool calls are present
royjhan's avatar
royjhan committed
502
503
504
505
506
507
508
509
510
			if msg.ToolCalls == nil {
				return nil, fmt.Errorf("invalid message content type: %T", content)
			}

			toolCalls := make([]api.ToolCall, len(msg.ToolCalls))
			for i, tc := range msg.ToolCalls {
				toolCalls[i].Function.Name = tc.Function.Name
				err := json.Unmarshal([]byte(tc.Function.Arguments), &toolCalls[i].Function.Arguments)
				if err != nil {
Michael Yang's avatar
lint  
Michael Yang committed
511
					return nil, errors.New("invalid tool call arguments")
royjhan's avatar
royjhan committed
512
513
				}
			}
514
			messages = append(messages, api.Message{Role: msg.Role, Thinking: msg.Reasoning, ToolCalls: toolCalls})
515
		}
516
517
	}

518
	options := make(map[string]any)
519
520
521
522

	switch stop := r.Stop.(type) {
	case string:
		options["stop"] = []string{stop}
523
	case []any:
524
525
526
527
528
529
530
531
532
533
534
535
536
537
		var stops []string
		for _, s := range stop {
			if str, ok := s.(string); ok {
				stops = append(stops, str)
			}
		}
		options["stop"] = stops
	}

	if r.MaxTokens != nil {
		options["num_predict"] = *r.MaxTokens
	}

	if r.Temperature != nil {
538
		options["temperature"] = *r.Temperature
539
540
541
542
543
544
545
546
547
	} else {
		options["temperature"] = 1.0
	}

	if r.Seed != nil {
		options["seed"] = *r.Seed
	}

	if r.FrequencyPenalty != nil {
548
		options["frequency_penalty"] = *r.FrequencyPenalty
549
550
551
	}

	if r.PresencePenalty != nil {
552
		options["presence_penalty"] = *r.PresencePenalty
553
554
555
556
557
558
559
560
	}

	if r.TopP != nil {
		options["top_p"] = *r.TopP
	} else {
		options["top_p"] = 1.0
	}

561
562
563
564
565
566
567
568
	var format json.RawMessage
	if r.ResponseFormat != nil {
		switch strings.ToLower(strings.TrimSpace(r.ResponseFormat.Type)) {
		// Support the old "json_object" type for OpenAI compatibility
		case "json_object":
			format = json.RawMessage(`"json"`)
		case "json_schema":
			if r.ResponseFormat.JsonSchema != nil {
569
				format = r.ResponseFormat.JsonSchema.Schema
570
571
			}
		}
572
573
	}

Michael Yang's avatar
Michael Yang committed
574
	var think *api.ThinkValue
575
576
	var effort string

Michael Yang's avatar
Michael Yang committed
577
	if r.Reasoning != nil {
578
579
580
581
582
583
584
585
		effort = r.Reasoning.Effort
	} else if r.ReasoningEffort != nil {
		effort = *r.ReasoningEffort
	}

	if effort != "" {
		if !slices.Contains([]string{"high", "medium", "low", "none"}, effort) {
			return nil, fmt.Errorf("invalid reasoning value: '%s' (must be \"high\", \"medium\", \"low\", or \"none\")", effort)
Michael Yang's avatar
Michael Yang committed
586
		}
587

588
		if effort == "none" {
589
590
			think = &api.ThinkValue{Value: false}
		} else {
591
			think = &api.ThinkValue{Value: effort}
592
		}
Michael Yang's avatar
Michael Yang committed
593
594
	}

595
	return &api.ChatRequest{
Devon Rifkin's avatar
Devon Rifkin committed
596
597
598
599
600
601
602
603
		Model:           r.Model,
		Messages:        messages,
		Format:          format,
		Options:         options,
		Stream:          &r.Stream,
		Tools:           r.Tools,
		Think:           think,
		DebugRenderOnly: r.DebugRenderOnly,
604
	}, nil
605
606
}

607
608
609
610
611
612
613
614
615
616
617
618
619
620
func nameFromToolCallID(messages []Message, toolCallID string) string {
	// iterate backwards to be more resilient to duplicate tool call IDs (this
	// follows "last one wins")
	for i := len(messages) - 1; i >= 0; i-- {
		msg := messages[i]
		for _, tc := range msg.ToolCalls {
			if tc.ID == toolCallID {
				return tc.Function.Name
			}
		}
	}
	return ""
}

621
622
623
624
625
626
627
628
629
630
631
632
633
func fromCompletionToolCall(toolCalls []ToolCall) ([]api.ToolCall, error) {
	apiToolCalls := make([]api.ToolCall, len(toolCalls))
	for i, tc := range toolCalls {
		apiToolCalls[i].Function.Name = tc.Function.Name
		err := json.Unmarshal([]byte(tc.Function.Arguments), &apiToolCalls[i].Function.Arguments)
		if err != nil {
			return nil, errors.New("invalid tool call arguments")
		}
	}

	return apiToolCalls, nil
}

634
635
// FromCompleteRequest converts a CompletionRequest to api.GenerateRequest
func FromCompleteRequest(r CompletionRequest) (api.GenerateRequest, error) {
636
637
638
639
640
	options := make(map[string]any)

	switch stop := r.Stop.(type) {
	case string:
		options["stop"] = []string{stop}
641
642
643
644
645
646
647
648
	case []any:
		var stops []string
		for _, s := range stop {
			if str, ok := s.(string); ok {
				stops = append(stops, str)
			} else {
				return api.GenerateRequest{}, fmt.Errorf("invalid type for 'stop' field: %T", s)
			}
649
		}
650
		options["stop"] = stops
651
652
653
654
655
656
657
	}

	if r.MaxTokens != nil {
		options["num_predict"] = *r.MaxTokens
	}

	if r.Temperature != nil {
658
		options["temperature"] = *r.Temperature
659
660
661
662
663
664
665
666
	} else {
		options["temperature"] = 1.0
	}

	if r.Seed != nil {
		options["seed"] = *r.Seed
	}

667
	options["frequency_penalty"] = r.FrequencyPenalty
668

669
	options["presence_penalty"] = r.PresencePenalty
670
671
672
673
674
675
676
677

	if r.TopP != 0.0 {
		options["top_p"] = r.TopP
	} else {
		options["top_p"] = 1.0
	}

	return api.GenerateRequest{
Devon Rifkin's avatar
Devon Rifkin committed
678
679
680
681
682
683
		Model:           r.Model,
		Prompt:          r.Prompt,
		Options:         options,
		Stream:          &r.Stream,
		Suffix:          r.Suffix,
		DebugRenderOnly: r.DebugRenderOnly,
684
685
	}, nil
}