Skip to content

GitLab

Unverified Commit 5df80590 authored by Nicolas Patry's avatar Nicolas Patry Committed by GitHub
Browse files

Auto max prefill (#2797) 

* Attempt at automatic max batch prefill.

* Taking into account number of shards.

* Adding more cards.

* Adding A100 + H100

* Adding a few more cards.

* Logprobs cost too much.

* h100 better name, and keep factor of 2

* Damn inflated sparse tflops.

* Typo in h100.

* Updated the flops calculation (checked with fvcore).

* chunking by default.

* Fix prefix caching for chat completion since we removed logprobs.

* More tests.

* Dropping all the prefill logprobs.

* Add a flag that enables users to get logprobs back.

* Repairing prompt token counting.

* Fixing a few tests.

* Remove some scaffolding.

* Attempting to reduces the issues (workarounds for now).
parent 8c3669b2
Hide whitespace changes
Inline Side-by-side
Showing with 41 additions and 862 deletions
integration-tests/models/__snapshots__/test_flash_grammar_llama/test_flash_llama_grammar_load.json
View file @ 5df80590
...@@ -4,53 +4,7 @@ ...@@ -4,53 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1024,
"logprob": -10.578125,
"text": "name"
},
{
"id": 29901,
"logprob": -3.0332031,
"text": ":"
},
{
"id": 13260,
"logprob": -9.171875,
"text": "dav"
},
{
"id": 333,
"logprob": -0.04257202,
"text": "id"
},
{
"id": 29889,
"logprob": -2.4785156,
"text": "."
},
{
"id": 4876,
"logprob": -10.7890625,
"text": "email"
},
{
"id": 29901,
"logprob": -0.32495117,
"text": ":"
},
{
"id": 259,
"logprob": -9.4921875,
"text": " "
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -123,53 +77,7 @@ ...@@ -123,53 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1024,
"logprob": -10.578125,
"text": "name"
},
{
"id": 29901,
"logprob": -3.03125,
"text": ":"
},
{
"id": 13260,
"logprob": -9.171875,
"text": "dav"
},
{
"id": 333,
"logprob": -0.04244995,
"text": "id"
},
{
"id": 29889,
"logprob": -2.4863281,
"text": "."
},
{
"id": 4876,
"logprob": -10.7890625,
"text": "email"
},
{
"id": 29901,
"logprob": -0.32714844,
"text": ":"
},
{
"id": 259,
"logprob": -9.4921875,
"text": " "
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -242,53 +150,7 @@ ...@@ -242,53 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1024,
"logprob": -10.578125,
"text": "name"
},
{
"id": 29901,
"logprob": -3.0332031,
"text": ":"
},
{
"id": 13260,
"logprob": -9.171875,
"text": "dav"
},
{
"id": 333,
"logprob": -0.04257202,
"text": "id"
},
{
"id": 29889,
"logprob": -2.4785156,
"text": "."
},
{
"id": 4876,
"logprob": -10.7890625,
"text": "email"
},
{
"id": 29901,
"logprob": -0.32495117,
"text": ":"
},
{
"id": 259,
"logprob": -9.4921875,
"text": " "
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -361,53 +223,7 @@ ...@@ -361,53 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1024,
"logprob": -10.578125,
"text": "name"
},
{
"id": 29901,
"logprob": -3.0332031,
"text": ":"
},
{
"id": 13260,
"logprob": -9.171875,
"text": "dav"
},
{
"id": 333,
"logprob": -0.04257202,
"text": "id"
},
{
"id": 29889,
"logprob": -2.4785156,
"text": "."
},
{
"id": 4876,
"logprob": -10.7890625,
"text": "email"
},
{
"id": 29901,
"logprob": -0.32495117,
"text": ":"
},
{
"id": 259,
"logprob": -9.4921875,
"text": " "
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_grammar_llama/test_flash_llama_grammar_regex.json
View file @ 5df80590
...@@ -3,43 +3,7 @@ ...@@ -3,43 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 806,
"logprob": -11.890625,
"text": "Wh"
},
{
"id": 1446,
"logprob": -3.6699219,
"text": "ats"
},
{
"id": 2921,
"logprob": -7.8203125,
"text": "Go"
},
{
"id": 468,
"logprob": -8.0703125,
"text": "og"
},
{
"id": 793,
"logprob": -2.1875,
"text": "les"
},
{
"id": 16332,
"logprob": -9.7109375,
"text": "DNS"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "stop_sequence", "finish_reason": "stop_sequence",
"generated_tokens": 5, "generated_tokens": 5,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -8.6875,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.546875,
"text": "request"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json
View file @ 5df80590
...@@ -4,23 +4,7 @@ ...@@ -4,23 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -8.6875,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.546875,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -93,23 +77,7 @@ ...@@ -93,23 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -8.6875,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.546875,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -182,23 +150,7 @@ ...@@ -182,23 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -8.6875,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.546875,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -271,23 +223,7 @@ ...@@ -271,23 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -8.6875,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.546875,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_simple.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -8.6875,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.546875,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2.json
View file @ 5df80590
...@@ -3,18 +3,7 @@ ...@@ -3,18 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.4375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2_all_params.json
View file @ 5df80590
...@@ -3,18 +3,7 @@ ...@@ -3,18 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.453125,
"text": " request"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2_load.json
View file @ 5df80590
...@@ -4,18 +4,7 @@ ...@@ -4,18 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.453125,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -88,18 +77,7 @@ ...@@ -88,18 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.40625,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -172,18 +150,7 @@ ...@@ -172,18 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.421875,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -256,18 +223,7 @@ ...@@ -256,18 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.4140625,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 2323,
"logprob": -9.421875,
"text": "Test"
},
{
"id": 1715,
"logprob": -10.546875,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_all_params.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 2323,
"logprob": -9.5234375,
"text": "Test"
},
{
"id": 1715,
"logprob": -10.421875,
"text": " request"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_load.json
View file @ 5df80590
...@@ -4,23 +4,7 @@ ...@@ -4,23 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 2323,
"logprob": -9.5625,
"text": "Test"
},
{
"id": 1715,
"logprob": -10.375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -93,23 +77,7 @@ ...@@ -93,23 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 2323,
"logprob": -9.5625,
"text": "Test"
},
{
"id": 1715,
"logprob": -10.375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -182,23 +150,7 @@ ...@@ -182,23 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 2323,
"logprob": -9.5625,
"text": "Test"
},
{
"id": 1715,
"logprob": -10.375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -271,23 +223,7 @@ ...@@ -271,23 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 2323,
"logprob": -9.5625,
"text": "Test"
},
{
"id": 1715,
"logprob": -10.375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_fp8_kv_cache/test_flash_llama_fp8_kv_cache.json
View file @ 5df80590
...@@ -3,38 +3,7 @@ ...@@ -3,38 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 3923,
"logprob": -6.1875,
"text": "What"
},
{
"id": 374,
"logprob": -0.93359375,
"text": " is"
},
{
"id": 5655,
"logprob": -9.875,
"text": " deep"
},
{
"id": 6975,
"logprob": -1.1796875,
"text": " learning"
},
{
"id": 30,
"logprob": -1.75,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_fp8_kv_cache/test_flash_llama_fp8_kv_cache_all_params.json
View file @ 5df80590
...@@ -3,33 +3,7 @@ ...@@ -3,33 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 374,
"logprob": -18.0,
"text": " is"
},
{
"id": 5655,
"logprob": -11.8359375,
"text": " deep"
},
{
"id": 6975,
"logprob": -2.0703125,
"text": " learning"
},
{
"id": 30,
"logprob": -5.9765625,
"text": "?"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_fp8_kv_cache/test_flash_llama_fp8_kv_cache_load.json
View file @ 5df80590
...@@ -4,38 +4,7 @@ ...@@ -4,38 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 3923,
"logprob": -6.1875,
"text": "What"
},
{
"id": 374,
"logprob": -0.93359375,
"text": " is"
},
{
"id": 5655,
"logprob": -9.875,
"text": " deep"
},
{
"id": 6975,
"logprob": -1.1796875,
"text": " learning"
},
{
"id": 30,
"logprob": -1.75,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -108,38 +77,7 @@ ...@@ -108,38 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 3923,
"logprob": -6.21875,
"text": "What"
},
{
"id": 374,
"logprob": -0.95703125,
"text": " is"
},
{
"id": 5655,
"logprob": -9.9375,
"text": " deep"
},
{
"id": 6975,
"logprob": -1.1328125,
"text": " learning"
},
{
"id": 30,
"logprob": -1.75,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -212,38 +150,7 @@ ...@@ -212,38 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 3923,
"logprob": -6.21875,
"text": "What"
},
{
"id": 374,
"logprob": -0.95703125,
"text": " is"
},
{
"id": 5655,
"logprob": -9.9375,
"text": " deep"
},
{
"id": 6975,
"logprob": -1.1328125,
"text": " learning"
},
{
"id": 30,
"logprob": -1.75,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -316,38 +223,7 @@ ...@@ -316,38 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 128000,
"logprob": null,
"text": "<|begin_of_text|>"
},
{
"id": 3923,
"logprob": -6.21875,
"text": "What"
},
{
"id": 374,
"logprob": -0.95703125,
"text": " is"
},
{
"id": 5655,
"logprob": -9.9375,
"text": " deep"
},
{
"id": 6975,
"logprob": -1.1328125,
"text": " learning"
},
{
"id": 30,
"logprob": -1.75,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json
View file @ 5df80590
...@@ -3,18 +3,7 @@ ...@@ -3,18 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.34375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json
View file @ 5df80590
...@@ -3,18 +3,7 @@ ...@@ -3,18 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.34375,
"text": " request"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json
View file @ 5df80590
...@@ -4,18 +4,7 @@ ...@@ -4,18 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.34375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -88,18 +77,7 @@ ...@@ -88,18 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.34375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -172,18 +150,7 @@ ...@@ -172,18 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.34375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -256,18 +223,7 @@ ...@@ -256,18 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 2323,
"logprob": null,
"text": "Test"
},
{
"id": 1715,
"logprob": -11.34375,
"text": " request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -12.390625,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.0625,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin_all_params.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -12.390625,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.0625,
"text": "request"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin_load.json
View file @ 5df80590
...@@ -4,23 +4,7 @@ ...@@ -4,23 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -12.390625,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.0625,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -93,23 +77,7 @@ ...@@ -93,23 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -12.390625,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.0625,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -182,23 +150,7 @@ ...@@ -182,23 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -12.390625,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.0625,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -271,23 +223,7 @@ ...@@ -271,23 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -12.390625,
"text": "Test"
},
{
"id": 2009,
"logprob": -11.0625,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment