Unverified Commit 290cf204 authored by Parth Sareen's avatar Parth Sareen Committed by GitHub
Browse files

llama: test key order preservation in schema_to_grammar (#8078)

This change adds a test to catch a regression in schema_to_grammar where
the order of keys in the JSON schema is not preserved in the generated
grammar, which is critical for step-by-step reasoning.
parent a72f2dce
package llama package grammar
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"strings" "strings"
"testing" "testing"
"github.com/ollama/ollama/llama"
) )
// https://github.com/ollama/ollama/issues/7978 // https://github.com/ollama/ollama/issues/7978
...@@ -14,23 +16,48 @@ const issue7978JSONSchema = `{ ...@@ -14,23 +16,48 @@ const issue7978JSONSchema = `{
"steps": { "steps": {
"type": "array", "type": "array",
"items": { "items": {
"type": "object", "type": "object",
"properties": { "properties": {
"explanation": { "type": "string" }, "explanation": { "type": "string" },
"output": { "type": "string" } "output": { "type": "string" },
}, "nested": {
"required": ["explanation", "output"], "type": "object",
"additionalProperties": false "properties": {
"deep": { "type": "string" }
}
}
},
"required": ["explanation", "output"],
"additionalProperties": false
} }
}, },
"final_answer": { "type": "string" } "final_answer": { "type": "string" },
"01_numbered_key": { "type": "string" },
"numbers": {
"type": "array",
"items": { "type": "number" }
},
"booleans": {
"type": "array",
"items": { "type": "boolean" }
},
"mixed": {
"type": "array",
"items": {
"oneOf": [
{ "type": "string" },
{ "type": "number" },
{ "type": "boolean" }
]
}
}
}, },
"required": ["steps", "final_answer"], "required": ["steps", "final_answer"],
"additionalProperties": false "additionalProperties": false
}` }`
func TestIssue7978(t *testing.T) { func TestIssue7978(t *testing.T) {
g := SchemaToGrammar([]byte(issue7978JSONSchema)) g := llama.SchemaToGrammar([]byte(issue7978JSONSchema))
if g == nil { if g == nil {
t.Fatal("failed to convert JSON schema to grammar") t.Fatal("failed to convert JSON schema to grammar")
} }
...@@ -38,17 +65,21 @@ func TestIssue7978(t *testing.T) { ...@@ -38,17 +65,21 @@ func TestIssue7978(t *testing.T) {
t.Logf("grammar:\n%s", g) t.Logf("grammar:\n%s", g)
t.Log() t.Log()
var sawSteps bool var got string
s := bufio.NewScanner(bytes.NewReader(g)) s := bufio.NewScanner(bytes.NewReader(g))
for s.Scan() { for s.Scan() {
line := s.Text() line := strings.TrimSpace(s.Text())
if strings.Contains(line, "steps") { step, _, _ := strings.Cut(line, " ::= ")
sawSteps = true step = strings.TrimSpace(step)
} if step == "root" {
if strings.Contains(line, "final-answer") && !sawSteps { got = line
t.Error("expected 'steps' before 'final-answer'")
} }
} }
want := `root ::= "{" space steps-kv "," space final-answer-kv ( "," space ( 01-numbered-key-kv 01-numbered-key-rest | numbers-kv numbers-rest | booleans-kv booleans-rest | mixed-kv ) )? "}" space`
if got != want {
t.Errorf("root =\n%qwant:\n%q", got, want)
}
} }
func TestSchemaToGrammer(t *testing.T) { func TestSchemaToGrammer(t *testing.T) {
...@@ -64,7 +95,7 @@ func TestSchemaToGrammer(t *testing.T) { ...@@ -64,7 +95,7 @@ func TestSchemaToGrammer(t *testing.T) {
for _, c := range cases { for _, c := range cases {
t.Run("x", func(t *testing.T) { t.Run("x", func(t *testing.T) {
g := SchemaToGrammar([]byte(c.schema)) g := llama.SchemaToGrammar([]byte(c.schema))
if c.prefix == nil && g != nil { if c.prefix == nil && g != nil {
t.Fatalf("grammar = %v, want nil", g) t.Fatalf("grammar = %v, want nil", g)
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment