parser_test.go 12.6 KB
Newer Older
1
package parser
2
3

import (
Michael Yang's avatar
Michael Yang committed
4
	"bytes"
5
	"encoding/binary"
Michael Yang's avatar
Michael Yang committed
6
7
	"fmt"
	"io"
8
9
	"strings"
	"testing"
10
	"unicode/utf16"
11
12

	"github.com/stretchr/testify/assert"
Michael Yang's avatar
lint  
Michael Yang committed
13
	"github.com/stretchr/testify/require"
14
15
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
16
17
)

Michael Yang's avatar
Michael Yang committed
18
func TestParseFileFile(t *testing.T) {
19
20
21
22
23
24
25
26
27
28
29
	input := `
FROM model1
ADAPTER adapter1
LICENSE MIT
PARAMETER param1 value1
PARAMETER param2 value2
TEMPLATE template1
`

	reader := strings.NewReader(input)

Michael Yang's avatar
Michael Yang committed
30
	modelfile, err := ParseFile(reader)
Michael Yang's avatar
lint  
Michael Yang committed
31
	require.NoError(t, err)
32
33
34
35
36
37
38
39
40
41

	expectedCommands := []Command{
		{Name: "model", Args: "model1"},
		{Name: "adapter", Args: "adapter1"},
		{Name: "license", Args: "MIT"},
		{Name: "param1", Args: "value1"},
		{Name: "param2", Args: "value2"},
		{Name: "template", Args: "template1"},
	}

Michael Yang's avatar
Michael Yang committed
42
	assert.Equal(t, expectedCommands, modelfile.Commands)
43
44
}

Michael Yang's avatar
Michael Yang committed
45
func TestParseFileFrom(t *testing.T) {
Michael Yang's avatar
tests  
Michael Yang committed
46
47
48
49
50
	var cases = []struct {
		input    string
		expected []Command
		err      error
	}{
Josh Yan's avatar
Josh Yan committed
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
		{
			"FROM \"FOO  BAR  \"",
			[]Command{{Name: "model", Args: "FOO  BAR  "}},
			nil,
		},
		{
			"FROM \"FOO BAR\"\nPARAMETER param1 value1",
			[]Command{{Name: "model", Args: "FOO BAR"}, {Name: "param1", Args: "value1"}},
			nil,
		},
		{
			"FROM     FOOO BAR    ",
			[]Command{{Name: "model", Args: "FOOO BAR"}},
			nil,
		},
		{
			"FROM /what/is/the path ",
			[]Command{{Name: "model", Args: "/what/is/the path"}},
			nil,
		},
Michael Yang's avatar
tests  
Michael Yang committed
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
		{
			"FROM foo",
			[]Command{{Name: "model", Args: "foo"}},
			nil,
		},
		{
			"FROM /path/to/model",
			[]Command{{Name: "model", Args: "/path/to/model"}},
			nil,
		},
		{
			"FROM /path/to/model/fp16.bin",
			[]Command{{Name: "model", Args: "/path/to/model/fp16.bin"}},
			nil,
		},
		{
			"FROM llama3:latest",
			[]Command{{Name: "model", Args: "llama3:latest"}},
			nil,
		},
		{
			"FROM llama3:7b-instruct-q4_K_M",
			[]Command{{Name: "model", Args: "llama3:7b-instruct-q4_K_M"}},
			nil,
		},
		{
			"", nil, errMissingFrom,
		},
		{
			"PARAMETER param1 value1",
			nil,
			errMissingFrom,
		},
		{
			"PARAMETER param1 value1\nFROM foo",
			[]Command{{Name: "param1", Args: "value1"}, {Name: "model", Args: "foo"}},
			nil,
		},
Josh Yan's avatar
Josh Yan committed
109
110
111
112
113
		{
			"PARAMETER what the \nFROM lemons make lemonade ",
			[]Command{{Name: "what", Args: "the "}, {Name: "model", Args: "lemons make lemonade"}},
			nil,
		},
Michael Yang's avatar
tests  
Michael Yang committed
114
	}
115

Michael Yang's avatar
tests  
Michael Yang committed
116
117
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
118
			modelfile, err := ParseFile(strings.NewReader(c.input))
Michael Yang's avatar
lint  
Michael Yang committed
119
			require.ErrorIs(t, err, c.err)
Michael Yang's avatar
Michael Yang committed
120
121
122
			if modelfile != nil {
				assert.Equal(t, c.expected, modelfile.Commands)
			}
Michael Yang's avatar
tests  
Michael Yang committed
123
124
		})
	}
125
126
}

Michael Yang's avatar
Michael Yang committed
127
func TestParseFileParametersMissingValue(t *testing.T) {
128
129
130
131
132
133
134
	input := `
FROM foo
PARAMETER param1
`

	reader := strings.NewReader(input)

Michael Yang's avatar
Michael Yang committed
135
	_, err := ParseFile(reader)
Michael Yang's avatar
lint  
Michael Yang committed
136
	require.ErrorIs(t, err, io.ErrUnexpectedEOF)
137
}
138

Michael Yang's avatar
Michael Yang committed
139
func TestParseFileBadCommand(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
140
141
142
143
	input := `
FROM foo
BADCOMMAND param1 value1
`
Michael Yang's avatar
Michael Yang committed
144
	_, err := ParseFile(strings.NewReader(input))
Michael Yang's avatar
lint  
Michael Yang committed
145
	require.ErrorIs(t, err, errInvalidCommand)
Michael Yang's avatar
Michael Yang committed
146
147
}

Michael Yang's avatar
Michael Yang committed
148
func TestParseFileMessages(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
149
150
151
152
153
154
155
156
	var cases = []struct {
		input    string
		expected []Command
		err      error
	}{
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
157
MESSAGE system You are a file parser. Always parse things.
Michael Yang's avatar
Michael Yang committed
158
159
160
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
161
				{Name: "message", Args: "system: You are a file parser. Always parse things."},
Michael Yang's avatar
Michael Yang committed
162
163
164
165
166
			},
			nil,
		},
		{
			`
167
FROM foo
Michael Yang's avatar
Michael Yang committed
168
MESSAGE system You are a file parser. Always parse things.`,
Michael Yang's avatar
Michael Yang committed
169
170
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
171
				{Name: "message", Args: "system: You are a file parser. Always parse things."},
Michael Yang's avatar
Michael Yang committed
172
173
174
175
176
177
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
178
MESSAGE system You are a file parser. Always parse things.
179
180
MESSAGE user Hey there!
MESSAGE assistant Hello, I want to parse all the things!
Michael Yang's avatar
Michael Yang committed
181
182
183
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
184
				{Name: "message", Args: "system: You are a file parser. Always parse things."},
Michael Yang's avatar
Michael Yang committed
185
186
187
188
189
190
191
192
193
				{Name: "message", Args: "user: Hey there!"},
				{Name: "message", Args: "assistant: Hello, I want to parse all the things!"},
			},
			nil,
		},
		{
			`
FROM foo
MESSAGE system """
Michael Yang's avatar
Michael Yang committed
194
You are a multiline file parser. Always parse things.
Michael Yang's avatar
Michael Yang committed
195
196
197
198
"""
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
199
				{Name: "message", Args: "system: \nYou are a multiline file parser. Always parse things.\n"},
Michael Yang's avatar
Michael Yang committed
200
201
202
203
204
205
206
207
208
			},
			nil,
		},
		{
			`
FROM foo
MESSAGE badguy I'm a bad guy!
`,
			nil,
Michael Yang's avatar
Michael Yang committed
209
			errInvalidMessageRole,
Michael Yang's avatar
Michael Yang committed
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
		},
		{
			`
FROM foo
MESSAGE system
`,
			nil,
			io.ErrUnexpectedEOF,
		},
		{
			`
FROM foo
MESSAGE system`,
			nil,
			io.ErrUnexpectedEOF,
		},
	}
227

Michael Yang's avatar
Michael Yang committed
228
229
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
230
			modelfile, err := ParseFile(strings.NewReader(c.input))
Michael Yang's avatar
lint  
Michael Yang committed
231
			require.ErrorIs(t, err, c.err)
Michael Yang's avatar
Michael Yang committed
232
233
234
			if modelfile != nil {
				assert.Equal(t, c.expected, modelfile.Commands)
			}
Michael Yang's avatar
Michael Yang committed
235
236
237
		})
	}
}
238

Michael Yang's avatar
Michael Yang committed
239
func TestParseFileQuoted(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
240
241
242
243
244
245
246
247
	var cases = []struct {
		multiline string
		expected  []Command
		err       error
	}{
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
248
SYSTEM """
Michael Yang's avatar
Michael Yang committed
249
This is a
Michael Yang's avatar
Michael Yang committed
250
multiline system.
Michael Yang's avatar
Michael Yang committed
251
252
253
254
"""
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
255
				{Name: "system", Args: "\nThis is a\nmultiline system.\n"},
Michael Yang's avatar
Michael Yang committed
256
257
258
259
260
261
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
262
SYSTEM """
Michael Yang's avatar
Michael Yang committed
263
This is a
Michael Yang's avatar
Michael Yang committed
264
multiline system."""
Michael Yang's avatar
Michael Yang committed
265
266
267
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
268
				{Name: "system", Args: "\nThis is a\nmultiline system."},
Michael Yang's avatar
Michael Yang committed
269
270
271
272
273
274
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
275
276
SYSTEM """This is a
multiline system."""
Michael Yang's avatar
Michael Yang committed
277
278
279
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
280
				{Name: "system", Args: "This is a\nmultiline system."},
Michael Yang's avatar
Michael Yang committed
281
282
283
284
285
286
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
287
SYSTEM """This is a multiline system."""
Michael Yang's avatar
Michael Yang committed
288
289
290
			`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
291
				{Name: "system", Args: "This is a multiline system."},
Michael Yang's avatar
Michael Yang committed
292
293
294
295
296
297
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
298
SYSTEM """This is a multiline system.""
Michael Yang's avatar
Michael Yang committed
299
300
301
302
303
304
305
			`,
			nil,
			io.ErrUnexpectedEOF,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
306
SYSTEM "
Michael Yang's avatar
Michael Yang committed
307
308
309
310
311
312
313
			`,
			nil,
			io.ErrUnexpectedEOF,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
314
315
SYSTEM """
This is a multiline system with "quotes".
Michael Yang's avatar
Michael Yang committed
316
317
318
319
"""
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
320
				{Name: "system", Args: "\nThis is a multiline system with \"quotes\".\n"},
Michael Yang's avatar
Michael Yang committed
321
322
323
324
325
326
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
327
SYSTEM """"""
Michael Yang's avatar
Michael Yang committed
328
329
330
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
331
				{Name: "system", Args: ""},
Michael Yang's avatar
Michael Yang committed
332
333
334
335
336
337
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
338
SYSTEM ""
Michael Yang's avatar
Michael Yang committed
339
340
341
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
342
				{Name: "system", Args: ""},
Michael Yang's avatar
Michael Yang committed
343
344
345
346
347
348
			},
			nil,
		},
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
349
SYSTEM "'"
Michael Yang's avatar
Michael Yang committed
350
351
352
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
353
				{Name: "system", Args: "'"},
Michael Yang's avatar
Michael Yang committed
354
355
356
			},
			nil,
		},
Michael Yang's avatar
tests  
Michael Yang committed
357
358
359
		{
			`
FROM foo
Michael Yang's avatar
Michael Yang committed
360
SYSTEM """''"'""'""'"'''''""'""'"""
Michael Yang's avatar
tests  
Michael Yang committed
361
362
363
`,
			[]Command{
				{Name: "model", Args: "foo"},
Michael Yang's avatar
Michael Yang committed
364
365
366
367
368
369
370
371
372
373
374
375
376
				{Name: "system", Args: `''"'""'""'"'''''""'""'`},
			},
			nil,
		},
		{
			`
FROM foo
TEMPLATE """
{{ .Prompt }}
"""`,
			[]Command{
				{Name: "model", Args: "foo"},
				{Name: "template", Args: "\n{{ .Prompt }}\n"},
Michael Yang's avatar
tests  
Michael Yang committed
377
378
379
			},
			nil,
		},
380
381
	}

Michael Yang's avatar
Michael Yang committed
382
383
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
384
			modelfile, err := ParseFile(strings.NewReader(c.multiline))
Michael Yang's avatar
lint  
Michael Yang committed
385
			require.ErrorIs(t, err, c.err)
Michael Yang's avatar
Michael Yang committed
386
387
388
			if modelfile != nil {
				assert.Equal(t, c.expected, modelfile.Commands)
			}
Michael Yang's avatar
Michael Yang committed
389
390
		})
	}
391
392
}

Michael Yang's avatar
Michael Yang committed
393
func TestParseFileParameters(t *testing.T) {
Michael Yang's avatar
tests  
Michael Yang committed
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
	var cases = map[string]struct {
		name, value string
	}{
		"numa true":                    {"numa", "true"},
		"num_ctx 1":                    {"num_ctx", "1"},
		"num_batch 1":                  {"num_batch", "1"},
		"num_gqa 1":                    {"num_gqa", "1"},
		"num_gpu 1":                    {"num_gpu", "1"},
		"main_gpu 1":                   {"main_gpu", "1"},
		"low_vram true":                {"low_vram", "true"},
		"f16_kv true":                  {"f16_kv", "true"},
		"logits_all true":              {"logits_all", "true"},
		"vocab_only true":              {"vocab_only", "true"},
		"use_mmap true":                {"use_mmap", "true"},
		"use_mlock true":               {"use_mlock", "true"},
		"num_thread 1":                 {"num_thread", "1"},
		"num_keep 1":                   {"num_keep", "1"},
		"seed 1":                       {"seed", "1"},
		"num_predict 1":                {"num_predict", "1"},
		"top_k 1":                      {"top_k", "1"},
		"top_p 1.0":                    {"top_p", "1.0"},
		"tfs_z 1.0":                    {"tfs_z", "1.0"},
		"typical_p 1.0":                {"typical_p", "1.0"},
		"repeat_last_n 1":              {"repeat_last_n", "1"},
		"temperature 1.0":              {"temperature", "1.0"},
		"repeat_penalty 1.0":           {"repeat_penalty", "1.0"},
		"presence_penalty 1.0":         {"presence_penalty", "1.0"},
		"frequency_penalty 1.0":        {"frequency_penalty", "1.0"},
		"mirostat 1":                   {"mirostat", "1"},
		"mirostat_tau 1.0":             {"mirostat_tau", "1.0"},
		"mirostat_eta 1.0":             {"mirostat_eta", "1.0"},
		"penalize_newline true":        {"penalize_newline", "true"},
		"stop ### User:":               {"stop", "### User:"},
		"stop ### User: ":              {"stop", "### User: "},
		"stop \"### User:\"":           {"stop", "### User:"},
		"stop \"### User: \"":          {"stop", "### User: "},
		"stop \"\"\"### User:\"\"\"":   {"stop", "### User:"},
		"stop \"\"\"### User:\n\"\"\"": {"stop", "### User:\n"},
		"stop <|endoftext|>":           {"stop", "<|endoftext|>"},
		"stop <|eot_id|>":              {"stop", "<|eot_id|>"},
		"stop </s>":                    {"stop", "</s>"},
Michael Yang's avatar
Michael Yang committed
435
	}
436

Michael Yang's avatar
tests  
Michael Yang committed
437
438
	for k, v := range cases {
		t.Run(k, func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
439
440
			var b bytes.Buffer
			fmt.Fprintln(&b, "FROM foo")
Michael Yang's avatar
tests  
Michael Yang committed
441
			fmt.Fprintln(&b, "PARAMETER", k)
Michael Yang's avatar
Michael Yang committed
442
			modelfile, err := ParseFile(&b)
Michael Yang's avatar
lint  
Michael Yang committed
443
			require.NoError(t, err)
Michael Yang's avatar
tests  
Michael Yang committed
444
445
446
447

			assert.Equal(t, []Command{
				{Name: "model", Args: "foo"},
				{Name: v.name, Args: v.value},
Michael Yang's avatar
Michael Yang committed
448
			}, modelfile.Commands)
Michael Yang's avatar
Michael Yang committed
449
450
451
452
		})
	}
}

Michael Yang's avatar
Michael Yang committed
453
func TestParseFileComments(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
454
455
456
457
458
459
460
	var cases = []struct {
		input    string
		expected []Command
	}{
		{
			`
# comment
461
FROM foo
Michael Yang's avatar
Michael Yang committed
462
463
464
465
466
467
	`,
			[]Command{
				{Name: "model", Args: "foo"},
			},
		},
	}
468

Michael Yang's avatar
Michael Yang committed
469
470
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
471
			modelfile, err := ParseFile(strings.NewReader(c.input))
Michael Yang's avatar
lint  
Michael Yang committed
472
			require.NoError(t, err)
Michael Yang's avatar
Michael Yang committed
473
			assert.Equal(t, c.expected, modelfile.Commands)
Michael Yang's avatar
Michael Yang committed
474
475
		})
	}
476
}
Michael Yang's avatar
Michael Yang committed
477

Michael Yang's avatar
Michael Yang committed
478
func TestParseFileFormatParseFile(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
479
480
481
482
483
484
485
486
	var cases = []string{
		`
FROM foo
ADAPTER adapter1
LICENSE MIT
PARAMETER param1 value1
PARAMETER param2 value2
TEMPLATE template1
Michael Yang's avatar
Michael Yang committed
487
MESSAGE system You are a file parser. Always parse things.
Michael Yang's avatar
Michael Yang committed
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
MESSAGE user Hey there!
MESSAGE assistant Hello, I want to parse all the things!
`,
		`
FROM foo
ADAPTER adapter1
LICENSE MIT
PARAMETER param1 value1
PARAMETER param2 value2
TEMPLATE template1
MESSAGE system """
You are a store greeter. Always responsed with "Hello!".
"""
MESSAGE user Hey there!
MESSAGE assistant Hello, I want to parse all the things!
`,
		`
FROM foo
ADAPTER adapter1
LICENSE """
Very long and boring legal text.
Blah blah blah.
"Oh look, a quote!"
"""

PARAMETER param1 value1
PARAMETER param2 value2
TEMPLATE template1
MESSAGE system """
You are a store greeter. Always responsed with "Hello!".
"""
MESSAGE user Hey there!
MESSAGE assistant Hello, I want to parse all the things!
521
522
523
524
`,
		`
FROM foo
SYSTEM ""
Michael Yang's avatar
Michael Yang committed
525
526
527
528
529
`,
	}

	for _, c := range cases {
		t.Run("", func(t *testing.T) {
Michael Yang's avatar
Michael Yang committed
530
			modelfile, err := ParseFile(strings.NewReader(c))
Michael Yang's avatar
lint  
Michael Yang committed
531
			require.NoError(t, err)
Michael Yang's avatar
Michael Yang committed
532

Michael Yang's avatar
Michael Yang committed
533
			modelfile2, err := ParseFile(strings.NewReader(modelfile.String()))
Michael Yang's avatar
lint  
Michael Yang committed
534
			require.NoError(t, err)
Michael Yang's avatar
Michael Yang committed
535

Michael Yang's avatar
Michael Yang committed
536
			assert.Equal(t, modelfile, modelfile2)
Michael Yang's avatar
Michael Yang committed
537
538
539
		})
	}
}
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554

func TestParseFileUTF16ParseFile(t *testing.T) {
	data := `FROM bob
PARAMETER param1 1
PARAMETER param2 4096
SYSTEM You are a utf16 file.
`

	expected := []Command{
		{Name: "model", Args: "bob"},
		{Name: "param1", Args: "1"},
		{Name: "param2", Args: "4096"},
		{Name: "system", Args: "You are a utf16 file."},
	}

555
556
557
558
	t.Run("le", func(t *testing.T) {
		var b bytes.Buffer
		require.NoError(t, binary.Write(&b, binary.LittleEndian, []byte{0xff, 0xfe}))
		require.NoError(t, binary.Write(&b, binary.LittleEndian, utf16.Encode([]rune(data))))
559

560
561
		actual, err := ParseFile(&b)
		require.NoError(t, err)
562

563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
		assert.Equal(t, expected, actual.Commands)
	})

	t.Run("be", func(t *testing.T) {
		var b bytes.Buffer
		require.NoError(t, binary.Write(&b, binary.BigEndian, []byte{0xfe, 0xff}))
		require.NoError(t, binary.Write(&b, binary.BigEndian, utf16.Encode([]rune(data))))

		actual, err := ParseFile(&b)
		require.NoError(t, err)
		assert.Equal(t, expected, actual.Commands)
	})
}

func TestParseMultiByte(t *testing.T) {
	input := `FROM test
	SYSTEM 你好👋`

	expect := []Command{
		{Name: "model", Args: "test"},
		{Name: "system", Args: "你好👋"},
	}

	encodings := []encoding.Encoding{
		unicode.UTF8,
		unicode.UTF16(unicode.LittleEndian, unicode.UseBOM),
		unicode.UTF16(unicode.BigEndian, unicode.UseBOM),
	}

	for _, encoding := range encodings {
		t.Run(fmt.Sprintf("%s", encoding), func(t *testing.T) {
			s, err := encoding.NewEncoder().String(input)
			require.NoError(t, err)

			actual, err := ParseFile(strings.NewReader(s))
			require.NoError(t, err)

			assert.Equal(t, expect, actual.Commands)
		})
	}
603
}