Add automated test for multimodal

A simple test case that verifies llava:7b can read text in an image

scripts/setup_integration_tests.sh

@@ -9,27 +9,30 @@ REPO=$(dirname $0)/../
 export OLLAMA_MODELS=${REPO}/test_data/models
 REGISTRY_SCHEME=https
 REGISTRY=registry.ollama.ai
-TEST_MODEL=library/orca-mini
-TEST_MODEL_TAG=latest
+TEST_MODELS=("library/orca-mini:latest" "library/llava:7b")
 ACCEPT_HEADER="Accept: application/vnd.docker.distribution.manifest.v2+json"
 
-mkdir -p ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/
-mkdir -p ${OLLAMA_MODELS}/blobs/
+for model in ${TEST_MODELS[@]}; do
+    TEST_MODEL=$(echo ${model} | cut -f1 -d:)
+    TEST_MODEL_TAG=$(echo ${model} | cut -f2 -d:)
+    mkdir -p ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/
+    mkdir -p ${OLLAMA_MODELS}/blobs/
 
-echo "Pulling manifest for ${TEST_MODEL}:${TEST_MODEL_TAG}"
-curl -s --header "${ACCEPT_HEADER}" \
-    -o ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} \
-    ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/manifests/${TEST_MODEL_TAG} 
+    echo "Pulling manifest for ${TEST_MODEL}:${TEST_MODEL_TAG}"
+    curl -s --header "${ACCEPT_HEADER}" \
+        -o ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} \
+        ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/manifests/${TEST_MODEL_TAG} 
 
-CFG_HASH=$(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".config.digest")
-echo "Pulling config blob ${CFG_HASH}"
-curl -L -C - --header "${ACCEPT_HEADER}" \
-        -o ${OLLAMA_MODELS}/blobs/${CFG_HASH} \
-        ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${CFG_HASH}
-
-for LAYER in $(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".layers[].digest" ) ; do
-    echo "Pulling blob ${LAYER}"
+    CFG_HASH=$(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".config.digest")
+    echo "Pulling config blob ${CFG_HASH}"
     curl -L -C - --header "${ACCEPT_HEADER}" \
-        -o ${OLLAMA_MODELS}/blobs/${LAYER} \
-        ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${LAYER}
+            -o ${OLLAMA_MODELS}/blobs/${CFG_HASH} \
+            ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${CFG_HASH}
+
+    for LAYER in $(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".layers[].digest" ) ; do
+        echo "Pulling blob ${LAYER}"
+        curl -L -C - --header "${ACCEPT_HEADER}" \
+            -o ${OLLAMA_MODELS}/blobs/${LAYER} \
+            ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${LAYER}
+    done
 done

server/llm_utils_test.go

@@ -7,7 +7,6 @@ import (
 	"path"
 	"runtime"
 	"testing"
-	"time"
 
 	"github.com/jmorganca/ollama/api"
 	"github.com/jmorganca/ollama/llm"
@@ -39,7 +38,6 @@ func PrepareModelForPrompts(t *testing.T, modelName string, opts api.Options) (*
 }
 
 func OneShotPromptResponse(t *testing.T, ctx context.Context, req api.GenerateRequest, model *Model, runner llm.LLM) string {
-	checkpointStart := time.Now()
 	prompt, err := model.Prompt(PromptVars{
 		System: req.System,
 		Prompt: req.Prompt,
@@ -56,12 +54,10 @@ func OneShotPromptResponse(t *testing.T, ctx context.Context, req api.GenerateRe
 			success <- true
 		}
 	}
-	checkpointLoaded := time.Now()
 	predictReq := llm.PredictOpts{
-		Prompt:           prompt,
-		Format:           req.Format,
-		CheckpointStart:  checkpointStart,
-		CheckpointLoaded: checkpointLoaded,
+		Prompt: prompt,
+		Format: req.Format,
+		Images: req.Images,
 	}
 	err = runner.Predict(ctx, predictReq, cb)
 	require.NoError(t, err, "predict call failed")