parser.go 2.88 KB
Newer Older
1
2
3
4
package parser

import (
	"bufio"
5
	"bytes"
6
	"errors"
7
	"fmt"
8
	"io"
Michael Yang's avatar
Michael Yang committed
9
	"log"
10
11
12
13
)

type Command struct {
	Name string
14
15
16
17
18
19
	Args string
}

func (c *Command) Reset() {
	c.Name = ""
	c.Args = ""
20
21
22
23
}

func Parse(reader io.Reader) ([]Command, error) {
	var commands []Command
24
	var command, modelCommand Command
25
26

	scanner := bufio.NewScanner(reader)
27
	scanner.Buffer(make([]byte, 0, bufio.MaxScanTokenSize), bufio.MaxScanTokenSize)
28
	scanner.Split(scanModelfile)
29
	for scanner.Scan() {
30
31
32
		line := scanner.Bytes()

		fields := bytes.SplitN(line, []byte(" "), 2)
Michael Yang's avatar
Michael Yang committed
33
		if len(fields) == 0 || len(fields[0]) == 0 {
34
35
36
			continue
		}

37
		switch string(bytes.ToUpper(fields[0])) {
38
39
		case "FROM":
			command.Name = "model"
40
			command.Args = string(bytes.TrimSpace(fields[1]))
41
42
			// copy command for validation
			modelCommand = command
43
44
45
46
		case "ADAPTER":
			command.Name = string(bytes.ToLower(fields[0]))
			command.Args = string(bytes.TrimSpace(fields[1]))
		case "LICENSE", "TEMPLATE", "SYSTEM", "PROMPT":
47
			command.Name = string(bytes.ToLower(fields[0]))
48
			command.Args = string(fields[1])
49
		case "PARAMETER":
50
			fields = bytes.SplitN(fields[1], []byte(" "), 2)
Michael Yang's avatar
Michael Yang committed
51
52
53
54
			if len(fields) < 2 {
				return nil, fmt.Errorf("missing value for %s", fields)
			}

55
			command.Name = string(fields[0])
56
			command.Args = string(bytes.TrimSpace(fields[1]))
57
58
		case "EMBED":
			return nil, fmt.Errorf("deprecated command: EMBED is no longer supported, use the /embed API endpoint instead")
59
		default:
Michael Yang's avatar
Michael Yang committed
60
61
62
63
			if !bytes.HasPrefix(fields[0], []byte("#")) {
				// log a warning for unknown commands
				log.Printf("WARNING: Unknown command: %s", fields[0])
			}
64
			continue
65
		}
66
67
68

		commands = append(commands, command)
		command.Reset()
69
70
	}

71
	if modelCommand.Args == "" {
72
		return nil, errors.New("no FROM line for the model was specified")
73
74
75
76
	}

	return commands, scanner.Err()
}
77
78

func scanModelfile(data []byte, atEOF bool) (advance int, token []byte, err error) {
79
80
81
82
83
84
85
86
	advance, token, err = scan([]byte(`"""`), []byte(`"""`), data, atEOF)
	if err != nil {
		return 0, nil, err
	}

	if advance > 0 && token != nil {
		return advance, token, nil
	}
87

88
89
90
91
92
93
94
95
96
97
98
99
100
	advance, token, err = scan([]byte(`"`), []byte(`"`), data, atEOF)
	if err != nil {
		return 0, nil, err
	}

	if advance > 0 && token != nil {
		return advance, token, nil
	}

	return bufio.ScanLines(data, atEOF)
}

func scan(openBytes, closeBytes, data []byte, atEOF bool) (advance int, token []byte, err error) {
101
102
	newline := bytes.IndexByte(data, '\n')

103
104
	if start := bytes.Index(data, openBytes); start >= 0 && start < newline {
		end := bytes.Index(data[start+len(openBytes):], closeBytes)
105
		if end < 0 {
106
			if atEOF {
107
				return 0, nil, fmt.Errorf("unterminated %s: expecting %s", openBytes, closeBytes)
108
109
110
			} else {
				return 0, nil, nil
			}
111
112
		}

113
		n := start + len(openBytes) + end + len(closeBytes)
Michael Yang's avatar
Michael Yang committed
114
115

		newData := data[:start]
116
		newData = append(newData, data[start+len(openBytes):n-len(closeBytes)]...)
Michael Yang's avatar
Michael Yang committed
117
		return n, newData, nil
118
119
	}

120
	return 0, nil, nil
121
}