Skip to content

GitLab

Unverified Commit 5df80590 authored by Nicolas Patry's avatar Nicolas Patry Committed by GitHub
Browse files

Auto max prefill (#2797) 

* Attempt at automatic max batch prefill.

* Taking into account number of shards.

* Adding more cards.

* Adding A100 + H100

* Adding a few more cards.

* Logprobs cost too much.

* h100 better name, and keep factor of 2

* Damn inflated sparse tflops.

* Typo in h100.

* Updated the flops calculation (checked with fvcore).

* chunking by default.

* Fix prefix caching for chat completion since we removed logprobs.

* More tests.

* Dropping all the prefill logprobs.

* Add a flag that enables users to get logprobs back.

* Repairing prompt token counting.

* Fixing a few tests.

* Remove some scaffolding.

* Attempting to reduces the issues (workarounds for now).
parent 8c3669b2
Hide whitespace changes
Inline Side-by-side
Showing with 41 additions and 1212 deletions
integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -9.0859375,
"text": "Test"
},
{
"id": 2009,
"logprob": -16.359375,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin24_all_params.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -9.0859375,
"text": "Test"
},
{
"id": 2009,
"logprob": -16.359375,
"text": "request"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin24_load.json
View file @ 5df80590
...@@ -4,23 +4,7 @@ ...@@ -4,23 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -9.0859375,
"text": "Test"
},
{
"id": 2009,
"logprob": -16.359375,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -93,23 +77,7 @@ ...@@ -93,23 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -9.0859375,
"text": "Test"
},
{
"id": 2009,
"logprob": -16.359375,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -182,23 +150,7 @@ ...@@ -182,23 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -9.0859375,
"text": "Test"
},
{
"id": 2009,
"logprob": -16.359375,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -271,23 +223,7 @@ ...@@ -271,23 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 4321,
"logprob": -9.0859375,
"text": "Test"
},
{
"id": 2009,
"logprob": -16.359375,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_all_params.json
View file @ 5df80590
...@@ -3,33 +3,7 @@ ...@@ -3,33 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 338,
"logprob": -10.0078125,
"text": "is"
},
{
"id": 21784,
"logprob": -15.515625,
"text": "Deep"
},
{
"id": 29257,
"logprob": -2.8847656,
"text": "Learning"
},
{
"id": 29973,
"logprob": -4.140625,
"text": "?"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_load.json
View file @ 5df80590
...@@ -4,38 +4,7 @@ ...@@ -4,38 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1724,
"logprob": -10.734375,
"text": "What"
},
{
"id": 338,
"logprob": -1.5488281,
"text": "is"
},
{
"id": 21784,
"logprob": -9.2890625,
"text": "Deep"
},
{
"id": 29257,
"logprob": -1.2753906,
"text": "Learning"
},
{
"id": 29973,
"logprob": -0.48046875,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -107,38 +76,7 @@ ...@@ -107,38 +76,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1724,
"logprob": -10.734375,
"text": "What"
},
{
"id": 338,
"logprob": -1.5488281,
"text": "is"
},
{
"id": 21784,
"logprob": -9.2890625,
"text": "Deep"
},
{
"id": 29257,
"logprob": -1.2724609,
"text": "Learning"
},
{
"id": 29973,
"logprob": -0.47729492,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -210,38 +148,7 @@ ...@@ -210,38 +148,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1724,
"logprob": -10.734375,
"text": "What"
},
{
"id": 338,
"logprob": -1.5488281,
"text": "is"
},
{
"id": 21784,
"logprob": -9.2890625,
"text": "Deep"
},
{
"id": 29257,
"logprob": -1.2724609,
"text": "Learning"
},
{
"id": 29973,
"logprob": -0.47729492,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -313,38 +220,7 @@ ...@@ -313,38 +220,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1724,
"logprob": -10.734375,
"text": "What"
},
{
"id": 338,
"logprob": -1.5488281,
"text": "is"
},
{
"id": 21784,
"logprob": -9.2890625,
"text": "Deep"
},
{
"id": 29257,
"logprob": -1.2724609,
"text": "Learning"
},
{
"id": 29973,
"logprob": -0.47729492,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_simple.json
View file @ 5df80590
...@@ -3,38 +3,7 @@ ...@@ -3,38 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1724,
"logprob": -10.734375,
"text": "What"
},
{
"id": 338,
"logprob": -1.5488281,
"text": "is"
},
{
"id": 21784,
"logprob": -9.2890625,
"text": "Deep"
},
{
"id": 29257,
"logprob": -1.2753906,
"text": "Learning"
},
{
"id": 29973,
"logprob": -0.48046875,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 3735,
"logprob": -12.9140625,
"text": "Test"
},
{
"id": 2159,
"logprob": -10.7578125,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json
View file @ 5df80590
...@@ -3,23 +3,7 @@ ...@@ -3,23 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 3735,
"logprob": -12.9140625,
"text": "Test"
},
{
"id": 2159,
"logprob": -10.7578125,
"text": "request"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_load.json
View file @ 5df80590
...@@ -4,23 +4,7 @@ ...@@ -4,23 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 3735,
"logprob": -12.9140625,
"text": "Test"
},
{
"id": 2159,
"logprob": -10.7578125,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -93,23 +77,7 @@ ...@@ -93,23 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 3735,
"logprob": -12.9140625,
"text": "Test"
},
{
"id": 2159,
"logprob": -10.7578125,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -182,23 +150,7 @@ ...@@ -182,23 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 3735,
"logprob": -12.9140625,
"text": "Test"
},
{
"id": 2159,
"logprob": -10.7578125,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -271,23 +223,7 @@ ...@@ -271,23 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 3735,
"logprob": -12.9140625,
"text": "Test"
},
{
"id": 2159,
"logprob": -10.7578125,
"text": "request"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral.json
View file @ 5df80590
...@@ -3,48 +3,7 @@ ...@@ -3,48 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -6.1445312,
"text": "What"
},
{
"id": 349,
"logprob": -1.4648438,
"text": "is"
},
{
"id": 21135,
"logprob": -13.6875,
"text": "gradient"
},
{
"id": 24871,
"logprob": -1.6005859,
"text": "descent"
},
{
"id": 28804,
"logprob": -0.39526367,
"text": "?"
},
{
"id": 13,
"logprob": -0.640625,
"text": "\n"
},
{
"id": 13,
"logprob": -0.18774414,
"text": "\n"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral_all_params.json
View file @ 5df80590
...@@ -3,33 +3,7 @@ ...@@ -3,33 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 24871,
"logprob": -17.234375,
"text": "descent"
},
{
"id": 28804,
"logprob": -7.4375,
"text": "?"
},
{
"id": 13,
"logprob": -0.8046875,
"text": "\n"
},
{
"id": 13,
"logprob": -0.33032227,
"text": "\n"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral_load.json
View file @ 5df80590
...@@ -4,48 +4,7 @@ ...@@ -4,48 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -6.1445312,
"text": "What"
},
{
"id": 349,
"logprob": -1.4648438,
"text": "is"
},
{
"id": 21135,
"logprob": -13.6875,
"text": "gradient"
},
{
"id": 24871,
"logprob": -1.6005859,
"text": "descent"
},
{
"id": 28804,
"logprob": -0.39526367,
"text": "?"
},
{
"id": 13,
"logprob": -0.640625,
"text": "\n"
},
{
"id": 13,
"logprob": -0.18774414,
"text": "\n"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -118,48 +77,7 @@ ...@@ -118,48 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -6.1445312,
"text": "What"
},
{
"id": 349,
"logprob": -1.4677734,
"text": "is"
},
{
"id": 21135,
"logprob": -13.6875,
"text": "gradient"
},
{
"id": 24871,
"logprob": -1.6015625,
"text": "descent"
},
{
"id": 28804,
"logprob": -0.39453125,
"text": "?"
},
{
"id": 13,
"logprob": -0.6435547,
"text": "\n"
},
{
"id": 13,
"logprob": -0.18713379,
"text": "\n"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -232,48 +150,7 @@ ...@@ -232,48 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -6.140625,
"text": "What"
},
{
"id": 349,
"logprob": -1.4658203,
"text": "is"
},
{
"id": 21135,
"logprob": -13.6796875,
"text": "gradient"
},
{
"id": 24871,
"logprob": -1.5898438,
"text": "descent"
},
{
"id": 28804,
"logprob": -0.3955078,
"text": "?"
},
{
"id": 13,
"logprob": -0.64501953,
"text": "\n"
},
{
"id": 13,
"logprob": -0.18493652,
"text": "\n"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -346,48 +223,7 @@ ...@@ -346,48 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -6.1328125,
"text": "What"
},
{
"id": 349,
"logprob": -1.4658203,
"text": "is"
},
{
"id": 21135,
"logprob": -13.6796875,
"text": "gradient"
},
{
"id": 24871,
"logprob": -1.5947266,
"text": "descent"
},
{
"id": 28804,
"logprob": -0.39648438,
"text": "?"
},
{
"id": 13,
"logprob": -0.6464844,
"text": "\n"
},
{
"id": 13,
"logprob": -0.18688965,
"text": "\n"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral_awq/test_flash_mixtral_awq.json
View file @ 5df80590
...@@ -3,38 +3,7 @@ ...@@ -3,38 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -12.296875,
"text": "What"
},
{
"id": 349,
"logprob": -0.97216797,
"text": "is"
},
{
"id": 3534,
"logprob": -10.1796875,
"text": "deep"
},
{
"id": 5168,
"logprob": -0.9658203,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.44384766,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral_awq/test_flash_mixtral_awq_all_params.json
View file @ 5df80590
...@@ -3,33 +3,7 @@ ...@@ -3,33 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 349,
"logprob": -13.921875,
"text": "is"
},
{
"id": 3534,
"logprob": -11.2265625,
"text": "deep"
},
{
"id": 5168,
"logprob": -2.3886719,
"text": "learning"
},
{
"id": 28804,
"logprob": -4.7109375,
"text": "?"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral_awq/test_flash_mixtral_awq_load.json
View file @ 5df80590
...@@ -4,38 +4,7 @@ ...@@ -4,38 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -12.296875,
"text": "What"
},
{
"id": 349,
"logprob": -0.97216797,
"text": "is"
},
{
"id": 3534,
"logprob": -10.1796875,
"text": "deep"
},
{
"id": 5168,
"logprob": -0.9658203,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.44384766,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -108,38 +77,7 @@ ...@@ -108,38 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -12.34375,
"text": "What"
},
{
"id": 349,
"logprob": -0.96728516,
"text": "is"
},
{
"id": 3534,
"logprob": -10.1796875,
"text": "deep"
},
{
"id": 5168,
"logprob": -0.97265625,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.44189453,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -212,38 +150,7 @@ ...@@ -212,38 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -12.34375,
"text": "What"
},
{
"id": 349,
"logprob": -0.96728516,
"text": "is"
},
{
"id": 3534,
"logprob": -10.1796875,
"text": "deep"
},
{
"id": 5168,
"logprob": -0.97265625,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.44189453,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -316,38 +223,7 @@ ...@@ -316,38 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -12.34375,
"text": "What"
},
{
"id": 349,
"logprob": -0.96728516,
"text": "is"
},
{
"id": 3534,
"logprob": -10.1796875,
"text": "deep"
},
{
"id": 5168,
"logprob": -0.97265625,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.44189453,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq.json
View file @ 5df80590
...@@ -3,38 +3,7 @@ ...@@ -3,38 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -9.2890625,
"text": "What"
},
{
"id": 349,
"logprob": -1.1503906,
"text": "is"
},
{
"id": 3534,
"logprob": -9.5859375,
"text": "deep"
},
{
"id": 5168,
"logprob": -1.3945312,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.4555664,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq_all_params.json
View file @ 5df80590
...@@ -3,33 +3,7 @@ ...@@ -3,33 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 349,
"logprob": -12.0546875,
"text": "is"
},
{
"id": 3534,
"logprob": -10.53125,
"text": "deep"
},
{
"id": 5168,
"logprob": -2.71875,
"text": "learning"
},
{
"id": 28804,
"logprob": -5.0078125,
"text": "?"
}
],
"seed": 0, "seed": 0,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq_load.json
View file @ 5df80590
...@@ -4,38 +4,7 @@ ...@@ -4,38 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -9.2890625,
"text": "What"
},
{
"id": 349,
"logprob": -1.1503906,
"text": "is"
},
{
"id": 3534,
"logprob": -9.5859375,
"text": "deep"
},
{
"id": 5168,
"logprob": -1.3945312,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.4555664,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -108,38 +77,7 @@ ...@@ -108,38 +77,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -9.2890625,
"text": "What"
},
{
"id": 349,
"logprob": -1.1425781,
"text": "is"
},
{
"id": 3534,
"logprob": -9.59375,
"text": "deep"
},
{
"id": 5168,
"logprob": -1.390625,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.45532227,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -212,38 +150,7 @@ ...@@ -212,38 +150,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -9.2890625,
"text": "What"
},
{
"id": 349,
"logprob": -1.1425781,
"text": "is"
},
{
"id": 3534,
"logprob": -9.59375,
"text": "deep"
},
{
"id": 5168,
"logprob": -1.390625,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.45532227,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -316,38 +223,7 @@ ...@@ -316,38 +223,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 1,
"logprob": null,
"text": "<s>"
},
{
"id": 1824,
"logprob": -9.2890625,
"text": "What"
},
{
"id": 349,
"logprob": -1.1425781,
"text": "is"
},
{
"id": 3534,
"logprob": -9.59375,
"text": "deep"
},
{
"id": 5168,
"logprob": -1.390625,
"text": "learning"
},
{
"id": 28804,
"logprob": -0.45532227,
"text": "?"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json
View file @ 5df80590
...@@ -3,48 +3,7 @@ ...@@ -3,48 +3,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 50278,
"logprob": null,
"text": "<|USER|>"
},
{
"id": 1276,
"logprob": -4.5546875,
"text": "What"
},
{
"id": 434,
"logprob": -4.234375,
"text": "'s"
},
{
"id": 634,
"logprob": -5.1054688,
"text": " your"
},
{
"id": 12315,
"logprob": -9.953125,
"text": " mood"
},
{
"id": 3063,
"logprob": -4.0820312,
"text": " today"
},
{
"id": 32,
"logprob": -0.15148926,
"text": "?"
},
{
"id": 50279,
"logprob": -0.27026367,
"text": "<|ASSISTANT|>"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json
View file @ 5df80590
...@@ -4,48 +4,7 @@ ...@@ -4,48 +4,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 50278,
"logprob": null,
"text": "<|USER|>"
},
{
"id": 1276,
"logprob": -4.5546875,
"text": "What"
},
{
"id": 434,
"logprob": -4.234375,
"text": "'s"
},
{
"id": 634,
"logprob": -5.21875,
"text": " your"
},
{
"id": 12315,
"logprob": -9.9375,
"text": " mood"
},
{
"id": 3063,
"logprob": -4.1015625,
"text": " today"
},
{
"id": 32,
"logprob": -0.15319824,
"text": "?"
},
{
"id": 50279,
"logprob": -0.2614746,
"text": "<|ASSISTANT|>"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -117,48 +76,7 @@ ...@@ -117,48 +76,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 50278,
"logprob": null,
"text": "<|USER|>"
},
{
"id": 1276,
"logprob": -4.5546875,
"text": "What"
},
{
"id": 434,
"logprob": -4.234375,
"text": "'s"
},
{
"id": 634,
"logprob": -5.1054688,
"text": " your"
},
{
"id": 12315,
"logprob": -9.953125,
"text": " mood"
},
{
"id": 3063,
"logprob": -4.0820312,
"text": " today"
},
{
"id": 32,
"logprob": -0.15148926,
"text": "?"
},
{
"id": 50279,
"logprob": -0.27026367,
"text": "<|ASSISTANT|>"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -230,48 +148,7 @@ ...@@ -230,48 +148,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 50278,
"logprob": null,
"text": "<|USER|>"
},
{
"id": 1276,
"logprob": -4.5546875,
"text": "What"
},
{
"id": 434,
"logprob": -4.234375,
"text": "'s"
},
{
"id": 634,
"logprob": -5.21875,
"text": " your"
},
{
"id": 12315,
"logprob": -9.9375,
"text": " mood"
},
{
"id": 3063,
"logprob": -4.1015625,
"text": " today"
},
{
"id": 32,
"logprob": -0.15319824,
"text": "?"
},
{
"id": 50279,
"logprob": -0.2614746,
"text": "<|ASSISTANT|>"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
...@@ -343,48 +220,7 @@ ...@@ -343,48 +220,7 @@
"best_of_sequences": null, "best_of_sequences": null,
"finish_reason": "length", "finish_reason": "length",
"generated_tokens": 10, "generated_tokens": 10,
"prefill": [ "prefill": [],
{
"id": 50278,
"logprob": null,
"text": "<|USER|>"
},
{
"id": 1276,
"logprob": -4.5546875,
"text": "What"
},
{
"id": 434,
"logprob": -4.234375,
"text": "'s"
},
{
"id": 634,
"logprob": -5.21875,
"text": " your"
},
{
"id": 12315,
"logprob": -9.9375,
"text": " mood"
},
{
"id": 3063,
"logprob": -4.1015625,
"text": " today"
},
{
"id": 32,
"logprob": -0.15319824,
"text": "?"
},
{
"id": 50279,
"logprob": -0.2614746,
"text": "<|ASSISTANT|>"
}
],
"seed": null, "seed": null,
"tokens": [ "tokens": [
{ {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment