parser_test.go 13.1 KB
Newer Older
1
package parser
2
3

import (
Michael Yang's avatar
Michael Yang committed
4
	"bytes"
5
	"encoding/binary"
Michael Yang's avatar
Michael Yang committed
6
7
	"fmt"
	"io"
8
9
	"strings"
	"testing"
10
	"unicode/utf16"
11
12

	"github.com/stretchr/testify/assert"
Michael Yang's avatar
lint  
Michael Yang committed
13
	"github.com/stretchr/testify/require"
14
15
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
16
17
)

Michael Yang's avatar
Michael Yang committed
18
func TestParseFileFile(t *testing.T) {
19
20
21
22
23
24
	input := `
FROM model1
ADAPTER adapter1
LICENSE MIT
PARAMETER param1 value1
PARAMETER param2 value2
Josh Yan's avatar
Josh Yan committed
25
26
27
28
29
30
31
TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>

{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>

{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>

{{ .Response }}<|eot_id|>"""    
32
33
34
35
`

	reader := strings.NewReader(input)

Michael Yang's avatar
Michael Yang committed
36
	modelfile, err := ParseFile(reader)
Michael Yang's avatar
lint  
Michael Yang committed
37
	require.NoError(t, err)
38
39
40
41
42
43
44

	expectedCommands := []Command{
		{Name: "model", Args: "model1"},
		{Name: "adapter", Args: "adapter1"},
		{Name: "license", Args: "MIT"},
		{Name: "param1", Args: "value1"},
		{Name: "param2", Args: "value2"},
Josh Yan's avatar
Josh Yan committed
45
		{Name: "template", Args: "{{ if .System }}<|start_header_id|>system<|end_header_id|>\n\n{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>\n\n{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>\n\n{{ .Response }}<|eot_id|>"},
46
47
	}

Michael Yang's avatar
Michael Yang committed
48
	assert.Equal(t, expectedCommands, modelfile.Commands)
49
50
}

Michael Yang's avatar
Michael Yang committed
51
func TestParseFileFrom(t *testing.T) {
Michael Yang's avatar
tests  
Michael Yang committed
52
53
54
55
56
	var cases = []struct {
		input    string
		expected []Command
		err      error
	}{
Josh Yan's avatar
Josh Yan committed
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
		{
			"FROM \"FOO  BAR  \"",
			[]Command{{Name: "model", Args: "FOO  BAR  "}},
			nil,
		},
		{
			"FROM \"FOO BAR\"\nPARAMETER param1 value1",
			[]Command{{Name: "model", Args: "FOO BAR"}, {Name: "param1", Args: "value1"}},
			nil,
		},
		{
			"FROM     FOOO BAR    ",
			[]Command{{Name: "model", Args: "FOOO BAR"}},
			nil,
		},
		{
			"FROM /what/is/the path ",
			[]Command{{Name: "model", Args: "/what/is/the path"}},
			nil,
		},
Michael Yang's avatar
tests  
Michael Yang committed
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
		{
			"FROM foo",
			[]Command{{Name: "model", Args: "foo"}},
			nil,
		},
		{
			"FROM /path/to/model",
			[]Command{{Name: "model", Args: "/path/to/model"}},
			nil,
		},
		{
			"FROM /path/to/model/fp16.bin",
			[]Command{{Name: "model", Args: "/path/to/model/fp16.bin"}},
			nil,
		},
		{
			"FROM llama3:latest",
			[]Command{{Name: "model", Args: "llama3:latest"}},
			nil,
		},
		{
			"FROM llama3:7b-instruct-q4_K_M",
			[]Command{{Name: "model", Args: "llama3:7b-instruct-q4_K_M"}},
			nil,
		},
		{
			"", nil, errMissingFrom,
		},
		{
			"PARAMETER param1 value1",
			nil,
			errMissingFrom,
		},
		{
			"PARAMETER param1 value1\nFROM foo",
			[]Command{{Name: "param1", Args: "value1"}, {Name: "model", Args: "foo"}},
			nil,
		},
Josh Yan's avatar
Josh Yan committed
115
116
		{
			"PARAMETER what the \nFROM lemons make lemonade ",
Josh Yan's avatar
Josh Yan committed
117
			[]Command{{Name: "what", Args: "the"}, {Name: "model", Args: "lemons make lemonade"}},
Josh Yan's avatar
Josh Yan committed
118
119
			nil,
		},
Michael Yang's avatar
tests  
Michael Yang committed
120
	}
121

Michael Yang's avatar
tests  
Michael Yang committed
122
123
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
124
			modelfile, err := ParseFile(strings.NewReader(c.input))
Michael Yang's avatar
lint  
Michael Yang committed
125
			require.ErrorIs(t, err, c.err)
Michael Yang's avatar
Michael Yang committed
126
127
128
			if modelfile != nil {
				assert.Equal(t, c.expected, modelfile.Commands)
			}
Michael Yang's avatar
tests  
Michael Yang committed
129
130
		})
	}
131
132
}

Michael Yang's avatar
Michael Yang committed
133
func TestParseFileParametersMissingValue(t *testing.T) {
134
135
136
137
138
139
140
	input := `
FROM foo
PARAMETER param1
`

	reader := strings.NewReader(input)

Michael Yang's avatar
Michael Yang committed
141
	_, err := ParseFile(reader)
Michael Yang's avatar
lint  
Michael Yang committed
142
	require.ErrorIs(t, err, io.ErrUnexpectedEOF)
143
}
144

Michael Yang's avatar
Michael Yang committed
145
func TestParseFileBadCommand(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
146
147
148
149
	input := `
FROM foo
BADCOMMAND param1 value1
`
Michael Yang's avatar
Michael Yang committed
150
	_, err := ParseFile(strings.NewReader(input))
Michael Yang's avatar
lint  
Michael Yang committed
151
	require.ErrorIs(t, err, errInvalidCommand)
Michael Yang's avatar
Michael Yang committed
152
153
}

Michael Yang's avatar
Michael Yang committed
154
func TestParseFileMessages(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
155
156
157
158
159
160
161
162
	var cases = []struct {
		input    string
		expected []Command
		err      error
	}{
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
163
MESSAGE system You are a file parser. Always parse things.
Michael Yang's avatar
Michael Yang committed
164
165
166
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
167
				{Name: "message", Args: "system: You are a file parser. Always parse things."},
Michael Yang's avatar
Michael Yang committed
168
169
170
171
172
			},
			nil,
		},
		{
			`
173
FROM foo
Michael Yang's avatar
Michael Yang committed
174
MESSAGE system You are a file parser. Always parse things.`,
Michael Yang's avatar
Michael Yang committed
175
176
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
177
				{Name: "message", Args: "system: You are a file parser. Always parse things."},
Michael Yang's avatar
Michael Yang committed
178
179
180
181
182
183
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
184
MESSAGE system You are a file parser. Always parse things.
185
186
MESSAGE user Hey there!
MESSAGE assistant Hello, I want to parse all the things!
Michael Yang's avatar
Michael Yang committed
187
188
189
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
190
				{Name: "message", Args: "system: You are a file parser. Always parse things."},
Michael Yang's avatar
Michael Yang committed
191
192
193
194
195
196
197
198
199
				{Name: "message", Args: "user: Hey there!"},
				{Name: "message", Args: "assistant: Hello, I want to parse all the things!"},
			},
			nil,
		},
		{
			`
FROM foo
MESSAGE system """
Michael Yang's avatar
Michael Yang committed
200
You are a multiline file parser. Always parse things.
Michael Yang's avatar
Michael Yang committed
201
202
203
204
"""
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
205
				{Name: "message", Args: "system: \nYou are a multiline file parser. Always parse things.\n"},
Michael Yang's avatar
Michael Yang committed
206
207
208
209
210
211
212
213
214
			},
			nil,
		},
		{
			`
FROM foo
MESSAGE badguy I'm a bad guy!
`,
			nil,
Michael Yang's avatar
Michael Yang committed
215
			errInvalidMessageRole,
Michael Yang's avatar
Michael Yang committed
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
		},
		{
			`
FROM foo
MESSAGE system
`,
			nil,
			io.ErrUnexpectedEOF,
		},
		{
			`
FROM foo
MESSAGE system`,
			nil,
			io.ErrUnexpectedEOF,
		},
	}
233

Michael Yang's avatar
Michael Yang committed
234
235
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
236
			modelfile, err := ParseFile(strings.NewReader(c.input))
Michael Yang's avatar
lint  
Michael Yang committed
237
			require.ErrorIs(t, err, c.err)
Michael Yang's avatar
Michael Yang committed
238
239
240
			if modelfile != nil {
				assert.Equal(t, c.expected, modelfile.Commands)
			}
Michael Yang's avatar
Michael Yang committed
241
242
243
		})
	}
}
244

Michael Yang's avatar
Michael Yang committed
245
func TestParseFileQuoted(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
246
247
248
249
250
251
252
253
	var cases = []struct {
		multiline string
		expected  []Command
		err       error
	}{
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
254
SYSTEM """
Michael Yang's avatar
Michael Yang committed
255
This is a
Michael Yang's avatar
Michael Yang committed
256
multiline system.
Michael Yang's avatar
Michael Yang committed
257
258
259
260
"""
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
261
				{Name: "system", Args: "\nThis is a\nmultiline system.\n"},
Michael Yang's avatar
Michael Yang committed
262
263
264
265
266
267
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
268
SYSTEM """
Michael Yang's avatar
Michael Yang committed
269
This is a
Michael Yang's avatar
Michael Yang committed
270
multiline system."""
Michael Yang's avatar
Michael Yang committed
271
272
273
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
274
				{Name: "system", Args: "\nThis is a\nmultiline system."},
Michael Yang's avatar
Michael Yang committed
275
276
277
278
279
280
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
281
282
SYSTEM """This is a
multiline system."""
Michael Yang's avatar
Michael Yang committed
283
284
285
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
286
				{Name: "system", Args: "This is a\nmultiline system."},
Michael Yang's avatar
Michael Yang committed
287
288
289
290
291
292
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
293
SYSTEM """This is a multiline system."""
Michael Yang's avatar
Michael Yang committed
294
295
296
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
297
				{Name: "system", Args: "This is a multiline system."},
Michael Yang's avatar
Michael Yang committed
298
299
300
301
302
303
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
304
SYSTEM """This is a multiline system.""
Michael Yang's avatar
Michael Yang committed
305
306
307
308
309
310
311
			`,
			nil,
			io.ErrUnexpectedEOF,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
312
SYSTEM "
Michael Yang's avatar
Michael Yang committed
313
314
315
316
317
318
319
			`,
			nil,
			io.ErrUnexpectedEOF,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
320
321
SYSTEM """
This is a multiline system with "quotes".
Michael Yang's avatar
Michael Yang committed
322
323
324
325
"""
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
326
				{Name: "system", Args: "\nThis is a multiline system with \"quotes\".\n"},
Michael Yang's avatar
Michael Yang committed
327
328
329
330
331
332
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
333
SYSTEM """"""
Michael Yang's avatar
Michael Yang committed
334
335
336
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
337
				{Name: "system", Args: ""},
Michael Yang's avatar
Michael Yang committed
338
339
340
341
342
343
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
344
SYSTEM ""
Michael Yang's avatar
Michael Yang committed
345
346
347
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
348
				{Name: "system", Args: ""},
Michael Yang's avatar
Michael Yang committed
349
350
351
352
353
354
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
355
SYSTEM "'"
Michael Yang's avatar
Michael Yang committed
356
357
358
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
359
				{Name: "system", Args: "'"},
Michael Yang's avatar
Michael Yang committed
360
361
362
			},
			nil,
		},
Michael Yang's avatar
tests  
Michael Yang committed
363
364
365
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
366
SYSTEM """''"'""'""'"'''''""'""'"""
Michael Yang's avatar
tests  
Michael Yang committed
367
368
369
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
370
371
372
373
374
375
376
377
378
379
380
381
382
				{Name: "system", Args: `''"'""'""'"'''''""'""'`},
			},
			nil,
		},
		{
			`
FROM foo
TEMPLATE """
{{ .Prompt }}
"""`,
			[]Command{
				{Name: "model", Args: "foo"},
				{Name: "template", Args: "\n{{ .Prompt }}\n"},
Michael Yang's avatar
tests  
Michael Yang committed
383
384
385
			},
			nil,
		},
386
387
	}

Michael Yang's avatar
Michael Yang committed
388
389
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
390
			modelfile, err := ParseFile(strings.NewReader(c.multiline))
Michael Yang's avatar
lint  
Michael Yang committed
391
			require.ErrorIs(t, err, c.err)
Michael Yang's avatar
Michael Yang committed
392
393
394
			if modelfile != nil {
				assert.Equal(t, c.expected, modelfile.Commands)
			}
Michael Yang's avatar
Michael Yang committed
395
396
		})
	}
397
398
}

Michael Yang's avatar
Michael Yang committed
399
func TestParseFileParameters(t *testing.T) {
Michael Yang's avatar
tests  
Michael Yang committed
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
	var cases = map[string]struct {
		name, value string
	}{
		"numa true":                    {"numa", "true"},
		"num_ctx 1":                    {"num_ctx", "1"},
		"num_batch 1":                  {"num_batch", "1"},
		"num_gqa 1":                    {"num_gqa", "1"},
		"num_gpu 1":                    {"num_gpu", "1"},
		"main_gpu 1":                   {"main_gpu", "1"},
		"low_vram true":                {"low_vram", "true"},
		"f16_kv true":                  {"f16_kv", "true"},
		"logits_all true":              {"logits_all", "true"},
		"vocab_only true":              {"vocab_only", "true"},
		"use_mmap true":                {"use_mmap", "true"},
		"use_mlock true":               {"use_mlock", "true"},
		"num_thread 1":                 {"num_thread", "1"},
		"num_keep 1":                   {"num_keep", "1"},
		"seed 1":                       {"seed", "1"},
		"num_predict 1":                {"num_predict", "1"},
		"top_k 1":                      {"top_k", "1"},
		"top_p 1.0":                    {"top_p", "1.0"},
		"tfs_z 1.0":                    {"tfs_z", "1.0"},
		"typical_p 1.0":                {"typical_p", "1.0"},
		"repeat_last_n 1":              {"repeat_last_n", "1"},
		"temperature 1.0":              {"temperature", "1.0"},
		"repeat_penalty 1.0":           {"repeat_penalty", "1.0"},
		"presence_penalty 1.0":         {"presence_penalty", "1.0"},
		"frequency_penalty 1.0":        {"frequency_penalty", "1.0"},
		"mirostat 1":                   {"mirostat", "1"},
		"mirostat_tau 1.0":             {"mirostat_tau", "1.0"},
		"mirostat_eta 1.0":             {"mirostat_eta", "1.0"},
		"penalize_newline true":        {"penalize_newline", "true"},
		"stop ### User:":               {"stop", "### User:"},
Josh Yan's avatar
Josh Yan committed
433
		"stop ### User: ":              {"stop", "### User:"},
Michael Yang's avatar
tests  
Michael Yang committed
434
435
436
437
438
439
440
		"stop \"### User:\"":           {"stop", "### User:"},
		"stop \"### User: \"":          {"stop", "### User: "},
		"stop \"\"\"### User:\"\"\"":   {"stop", "### User:"},
		"stop \"\"\"### User:\n\"\"\"": {"stop", "### User:\n"},
		"stop <|endoftext|>":           {"stop", "<|endoftext|>"},
		"stop <|eot_id|>":              {"stop", "<|eot_id|>"},
		"stop </s>":                    {"stop", "</s>"},
Michael Yang's avatar
Michael Yang committed
441
	}
442

Michael Yang's avatar
tests  
Michael Yang committed
443
444
	for k, v := range cases {
		t.Run(k, func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
445
446
			var b bytes.Buffer
			fmt.Fprintln(&b, "FROM foo")
Michael Yang's avatar
tests  
Michael Yang committed
447
			fmt.Fprintln(&b, "PARAMETER", k)
Michael Yang's avatar
Michael Yang committed
448
			modelfile, err := ParseFile(&b)
Michael Yang's avatar
lint  
Michael Yang committed
449
			require.NoError(t, err)
Michael Yang's avatar
tests  
Michael Yang committed
450
451
452
453

			assert.Equal(t, []Command{
				{Name: "model", Args: "foo"},
				{Name: v.name, Args: v.value},
Michael Yang's avatar
Michael Yang committed
454
			}, modelfile.Commands)
Michael Yang's avatar
Michael Yang committed
455
456
457
458
		})
	}
}

Michael Yang's avatar
Michael Yang committed
459
func TestParseFileComments(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
460
461
462
463
464
465
466
	var cases = []struct {
		input    string
		expected []Command
	}{
		{
			`
# comment
467
FROM foo
Michael Yang's avatar
Michael Yang committed
468
469
470
471
472
473
	`,
			[]Command{
				{Name: "model", Args: "foo"},
			},
		},
	}
474

Michael Yang's avatar
Michael Yang committed
475
476
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
477
			modelfile, err := ParseFile(strings.NewReader(c.input))
Michael Yang's avatar
lint  
Michael Yang committed
478
			require.NoError(t, err)
Michael Yang's avatar
Michael Yang committed
479
			assert.Equal(t, c.expected, modelfile.Commands)
Michael Yang's avatar
Michael Yang committed
480
481
		})
	}
482
}
Michael Yang's avatar
Michael Yang committed
483

Michael Yang's avatar
Michael Yang committed
484
func TestParseFileFormatParseFile(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
485
486
487
488
489
490
491
492
	var cases = []string{
		`
FROM foo
ADAPTER adapter1
LICENSE MIT
PARAMETER param1 value1
PARAMETER param2 value2
TEMPLATE template1
Michael Yang's avatar
Michael Yang committed
493
MESSAGE system You are a file parser. Always parse things.
Michael Yang's avatar
Michael Yang committed
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
MESSAGE user Hey there!
MESSAGE assistant Hello, I want to parse all the things!
`,
		`
FROM foo
ADAPTER adapter1
LICENSE MIT
PARAMETER param1 value1
PARAMETER param2 value2
TEMPLATE template1
MESSAGE system """
You are a store greeter. Always responsed with "Hello!".
"""
MESSAGE user Hey there!
MESSAGE assistant Hello, I want to parse all the things!
`,
		`
FROM foo
ADAPTER adapter1
LICENSE """
Very long and boring legal text.
Blah blah blah.
"Oh look, a quote!"
"""

PARAMETER param1 value1
PARAMETER param2 value2
TEMPLATE template1
MESSAGE system """
You are a store greeter. Always responsed with "Hello!".
"""
MESSAGE user Hey there!
MESSAGE assistant Hello, I want to parse all the things!
527
528
529
530
`,
		`
FROM foo
SYSTEM ""
Michael Yang's avatar
Michael Yang committed
531
532
533
534
535
`,
	}

	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
536
			modelfile, err := ParseFile(strings.NewReader(c))
Michael Yang's avatar
lint  
Michael Yang committed
537
			require.NoError(t, err)
Michael Yang's avatar
Michael Yang committed
538

Michael Yang's avatar
Michael Yang committed
539
			modelfile2, err := ParseFile(strings.NewReader(modelfile.String()))
Michael Yang's avatar
lint  
Michael Yang committed
540
			require.NoError(t, err)
Michael Yang's avatar
Michael Yang committed
541

Michael Yang's avatar
Michael Yang committed
542
			assert.Equal(t, modelfile, modelfile2)
Michael Yang's avatar
Michael Yang committed
543
544
545
		})
	}
}
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560

func TestParseFileUTF16ParseFile(t *testing.T) {
	data := `FROM bob
PARAMETER param1 1
PARAMETER param2 4096
SYSTEM You are a utf16 file.
`

	expected := []Command{
		{Name: "model", Args: "bob"},
		{Name: "param1", Args: "1"},
		{Name: "param2", Args: "4096"},
		{Name: "system", Args: "You are a utf16 file."},
	}

561
562
563
564
	t.Run("le", func(t *testing.T) {
		var b bytes.Buffer
		require.NoError(t, binary.Write(&b, binary.LittleEndian, []byte{0xff, 0xfe}))
		require.NoError(t, binary.Write(&b, binary.LittleEndian, utf16.Encode([]rune(data))))
565

566
567
		actual, err := ParseFile(&b)
		require.NoError(t, err)
568

569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
		assert.Equal(t, expected, actual.Commands)
	})

	t.Run("be", func(t *testing.T) {
		var b bytes.Buffer
		require.NoError(t, binary.Write(&b, binary.BigEndian, []byte{0xfe, 0xff}))
		require.NoError(t, binary.Write(&b, binary.BigEndian, utf16.Encode([]rune(data))))

		actual, err := ParseFile(&b)
		require.NoError(t, err)
		assert.Equal(t, expected, actual.Commands)
	})
}

func TestParseMultiByte(t *testing.T) {
	input := `FROM test
	SYSTEM 你好👋`

	expect := []Command{
		{Name: "model", Args: "test"},
		{Name: "system", Args: "你好👋"},
	}

	encodings := []encoding.Encoding{
		unicode.UTF8,
		unicode.UTF16(unicode.LittleEndian, unicode.UseBOM),
		unicode.UTF16(unicode.BigEndian, unicode.UseBOM),
	}

	for _, encoding := range encodings {
		t.Run(fmt.Sprintf("%s", encoding), func(t *testing.T) {
			s, err := encoding.NewEncoder().String(input)
			require.NoError(t, err)

			actual, err := ParseFile(strings.NewReader(s))
			require.NoError(t, err)

			assert.Equal(t, expect, actual.Commands)
		})
	}
609
}