Skip to content

GitLab

Unverified Commit 5df80590 authored by Nicolas Patry's avatar Nicolas Patry Committed by GitHub
Browse files

Auto max prefill (#2797) 

* Attempt at automatic max batch prefill.

* Taking into account number of shards.

* Adding more cards.

* Adding A100 + H100

* Adding a few more cards.

* Logprobs cost too much.

* h100 better name, and keep factor of 2

* Damn inflated sparse tflops.

* Typo in h100.

* Updated the flops calculation (checked with fvcore).

* chunking by default.

* Fix prefix caching for chat completion since we removed logprobs.

* More tests.

* Dropping all the prefill logprobs.

* Add a flag that enables users to get logprobs back.

* Repairing prompt token counting.

* Fixing a few tests.

* Remove some scaffolding.

* Attempting to reduces the issues (workarounds for now).
parent 8c3669b2
Hide whitespace changes
Inline Side-by-side
Showing with 41 additions and 1252 deletions
integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox.json
View file @ 5df80590
......@@ -3,98 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 50278,
"logprob": null,
"text": "<|prompter|>"
},
{
"id": 1276,
"logprob": -8.03125,
"text": "What"
},
{
"id": 310,
"logprob": -5.421875,
"text": " is"
},
{
"id": 247,
"logprob": -2.1601562,
"text": " a"
},
{
"id": 1167,
"logprob": -5.4609375,
"text": " mem"
},
{
"id": 70,
"logprob": -0.005657196,
"text": "e"
},
{
"id": 13,
"logprob": -7.28125,
"text": ","
},
{
"id": 285,
"logprob": -0.2980957,
"text": " and"
},
{
"id": 752,
"logprob": -2.1679688,
"text": " what"
},
{
"id": 434,
"logprob": -5.6210938,
"text": "'s"
},
{
"id": 253,
"logprob": -0.81103516,
"text": " the"
},
{
"id": 2892,
"logprob": -6.6640625,
"text": " history"
},
{
"id": 3212,
"logprob": -2.265625,
"text": " behind"
},
{
"id": 436,
"logprob": -11.5078125,
"text": " this"
},
{
"id": 3159,
"logprob": -2.1582031,
"text": " word"
},
{
"id": 32,
"logprob": -0.008720398,
"text": "?"
},
{
"id": 0,
"logprob": -2.4726562,
"text": "<|endoftext|>"
},
{
"id": 50281,
"logprob": -18.265625,
"text": "<|assistant|>"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox_load.json
View file @ 5df80590
......@@ -4,98 +4,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 50278,
"logprob": null,
"text": "<|prompter|>"
},
{
"id": 1276,
"logprob": -8.03125,
"text": "What"
},
{
"id": 310,
"logprob": -5.421875,
"text": " is"
},
{
"id": 247,
"logprob": -2.1601562,
"text": " a"
},
{
"id": 1167,
"logprob": -5.4609375,
"text": " mem"
},
{
"id": 70,
"logprob": -0.005657196,
"text": "e"
},
{
"id": 13,
"logprob": -7.28125,
"text": ","
},
{
"id": 285,
"logprob": -0.2980957,
"text": " and"
},
{
"id": 752,
"logprob": -2.1679688,
"text": " what"
},
{
"id": 434,
"logprob": -5.6210938,
"text": "'s"
},
{
"id": 253,
"logprob": -0.81103516,
"text": " the"
},
{
"id": 2892,
"logprob": -6.6640625,
"text": " history"
},
{
"id": 3212,
"logprob": -2.265625,
"text": " behind"
},
{
"id": 436,
"logprob": -11.5078125,
"text": " this"
},
{
"id": 3159,
"logprob": -2.1582031,
"text": " word"
},
{
"id": 32,
"logprob": -0.008720398,
"text": "?"
},
{
"id": 0,
"logprob": -2.4726562,
"text": "<|endoftext|>"
},
{
"id": 50281,
"logprob": -18.265625,
"text": "<|assistant|>"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -167,98 +76,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 50278,
"logprob": null,
"text": "<|prompter|>"
},
{
"id": 1276,
"logprob": -8.03125,
"text": "What"
},
{
"id": 310,
"logprob": -5.421875,
"text": " is"
},
{
"id": 247,
"logprob": -2.1601562,
"text": " a"
},
{
"id": 1167,
"logprob": -5.4609375,
"text": " mem"
},
{
"id": 70,
"logprob": -0.005657196,
"text": "e"
},
{
"id": 13,
"logprob": -7.28125,
"text": ","
},
{
"id": 285,
"logprob": -0.2980957,
"text": " and"
},
{
"id": 752,
"logprob": -2.1679688,
"text": " what"
},
{
"id": 434,
"logprob": -5.6210938,
"text": "'s"
},
{
"id": 253,
"logprob": -0.81103516,
"text": " the"
},
{
"id": 2892,
"logprob": -6.6640625,
"text": " history"
},
{
"id": 3212,
"logprob": -2.265625,
"text": " behind"
},
{
"id": 436,
"logprob": -11.5078125,
"text": " this"
},
{
"id": 3159,
"logprob": -2.1582031,
"text": " word"
},
{
"id": 32,
"logprob": -0.008720398,
"text": "?"
},
{
"id": 0,
"logprob": -2.4726562,
"text": "<|endoftext|>"
},
{
"id": 50281,
"logprob": -18.265625,
"text": "<|assistant|>"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -330,98 +148,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 50278,
"logprob": null,
"text": "<|prompter|>"
},
{
"id": 1276,
"logprob": -8.03125,
"text": "What"
},
{
"id": 310,
"logprob": -5.421875,
"text": " is"
},
{
"id": 247,
"logprob": -2.1601562,
"text": " a"
},
{
"id": 1167,
"logprob": -5.4609375,
"text": " mem"
},
{
"id": 70,
"logprob": -0.005657196,
"text": "e"
},
{
"id": 13,
"logprob": -7.28125,
"text": ","
},
{
"id": 285,
"logprob": -0.2980957,
"text": " and"
},
{
"id": 752,
"logprob": -2.1679688,
"text": " what"
},
{
"id": 434,
"logprob": -5.6210938,
"text": "'s"
},
{
"id": 253,
"logprob": -0.81103516,
"text": " the"
},
{
"id": 2892,
"logprob": -6.6640625,
"text": " history"
},
{
"id": 3212,
"logprob": -2.265625,
"text": " behind"
},
{
"id": 436,
"logprob": -11.5078125,
"text": " this"
},
{
"id": 3159,
"logprob": -2.1582031,
"text": " word"
},
{
"id": 32,
"logprob": -0.008720398,
"text": "?"
},
{
"id": 0,
"logprob": -2.4726562,
"text": "<|endoftext|>"
},
{
"id": 50281,
"logprob": -18.265625,
"text": "<|assistant|>"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -493,98 +220,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 50278,
"logprob": null,
"text": "<|prompter|>"
},
{
"id": 1276,
"logprob": -8.03125,
"text": "What"
},
{
"id": 310,
"logprob": -5.421875,
"text": " is"
},
{
"id": 247,
"logprob": -2.1601562,
"text": " a"
},
{
"id": 1167,
"logprob": -5.4609375,
"text": " mem"
},
{
"id": 70,
"logprob": -0.005657196,
"text": "e"
},
{
"id": 13,
"logprob": -7.28125,
"text": ","
},
{
"id": 285,
"logprob": -0.2980957,
"text": " and"
},
{
"id": 752,
"logprob": -2.1679688,
"text": " what"
},
{
"id": 434,
"logprob": -5.6210938,
"text": "'s"
},
{
"id": 253,
"logprob": -0.81103516,
"text": " the"
},
{
"id": 2892,
"logprob": -6.6640625,
"text": " history"
},
{
"id": 3212,
"logprob": -2.265625,
"text": " behind"
},
{
"id": 436,
"logprob": -11.5078125,
"text": " this"
},
{
"id": 3159,
"logprob": -2.1582031,
"text": " word"
},
{
"id": 32,
"logprob": -0.008720398,
"text": "?"
},
{
"id": 0,
"logprob": -2.4726562,
"text": "<|endoftext|>"
},
{
"id": 50281,
"logprob": -18.265625,
"text": "<|assistant|>"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi.json
View file @ 5df80590
......@@ -3,18 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 14402,
"logprob": null,
"text": "Test"
},
{
"id": 2581,
"logprob": -11.6171875,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json
View file @ 5df80590
......@@ -3,18 +3,7 @@
"best_of_sequences": null,
"finish_reason": "stop_sequence",
"generated_tokens": 6,
"prefill": [
{
"id": 14402,
"logprob": null,
"text": "Test"
},
{
"id": 2581,
"logprob": -11.6171875,
"text": " request"
}
],
"prefill": [],
"seed": 0,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_load.json
View file @ 5df80590
......@@ -4,18 +4,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 14402,
"logprob": null,
"text": "Test"
},
{
"id": 2581,
"logprob": -11.6171875,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -88,18 +77,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 14402,
"logprob": null,
"text": "Test"
},
{
"id": 2581,
"logprob": -11.6171875,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -172,18 +150,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 14402,
"logprob": null,
"text": "Test"
},
{
"id": 2581,
"logprob": -11.6171875,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -256,18 +223,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 14402,
"logprob": null,
"text": "Test"
},
{
"id": 2581,
"logprob": -11.6171875,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe.json
View file @ 5df80590
......@@ -3,43 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 1724,
"logprob": null,
"text": "What"
},
{
"id": 338,
"logprob": -0.6201172,
"text": "is"
},
{
"id": 16030,
"logprob": -13.6484375,
"text": "gradient"
},
{
"id": 26815,
"logprob": -0.003894806,
"text": "descent"
},
{
"id": 29973,
"logprob": -2.6386719,
"text": "?"
},
{
"id": 13,
"logprob": -6.46875,
"text": "\n"
},
{
"id": 13,
"logprob": -6.6875,
"text": "\n"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_all_params.json
View file @ 5df80590
......@@ -3,33 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 338,
"logprob": null,
"text": "is"
},
{
"id": 16030,
"logprob": -13.328125,
"text": "gradient"
},
{
"id": 26815,
"logprob": -0.24023438,
"text": "descent"
},
{
"id": 29973,
"logprob": -3.1386719,
"text": "?"
},
{
"id": 13,
"logprob": -3.0878906,
"text": "\n"
}
],
"prefill": [],
"seed": 0,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_load.json
View file @ 5df80590
......@@ -4,43 +4,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 1724,
"logprob": null,
"text": "What"
},
{
"id": 338,
"logprob": -0.6201172,
"text": "is"
},
{
"id": 16030,
"logprob": -13.6484375,
"text": "gradient"
},
{
"id": 26815,
"logprob": -0.003894806,
"text": "descent"
},
{
"id": 29973,
"logprob": -2.6386719,
"text": "?"
},
{
"id": 13,
"logprob": -6.46875,
"text": "\n"
},
{
"id": 13,
"logprob": -6.6875,
"text": "\n"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -113,43 +77,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 1724,
"logprob": null,
"text": "What"
},
{
"id": 338,
"logprob": -0.6113281,
"text": "is"
},
{
"id": 16030,
"logprob": -13.6640625,
"text": "gradient"
},
{
"id": 26815,
"logprob": -0.003929138,
"text": "descent"
},
{
"id": 29973,
"logprob": -2.625,
"text": "?"
},
{
"id": 13,
"logprob": -6.484375,
"text": "\n"
},
{
"id": 13,
"logprob": -6.6875,
"text": "\n"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -222,43 +150,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 1724,
"logprob": null,
"text": "What"
},
{
"id": 338,
"logprob": -0.609375,
"text": "is"
},
{
"id": 16030,
"logprob": -13.671875,
"text": "gradient"
},
{
"id": 26815,
"logprob": -0.0040016174,
"text": "descent"
},
{
"id": 29973,
"logprob": -2.6230469,
"text": "?"
},
{
"id": 13,
"logprob": -6.453125,
"text": "\n"
},
{
"id": 13,
"logprob": -6.6875,
"text": "\n"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -331,43 +223,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 1724,
"logprob": null,
"text": "What"
},
{
"id": 338,
"logprob": -0.609375,
"text": "is"
},
{
"id": 16030,
"logprob": -13.6640625,
"text": "gradient"
},
{
"id": 26815,
"logprob": -0.0038967133,
"text": "descent"
},
{
"id": 29973,
"logprob": -2.6347656,
"text": "?"
},
{
"id": 13,
"logprob": -6.453125,
"text": "\n"
},
{
"id": 13,
"logprob": -6.6875,
"text": "\n"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_qwen2/test_flash_qwen2.json
View file @ 5df80590
......@@ -3,18 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 2271,
"logprob": null,
"text": "Test"
},
{
"id": 1681,
"logprob": -8.8515625,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_qwen2/test_flash_qwen2_all_params.json
View file @ 5df80590
......@@ -3,18 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 2271,
"logprob": null,
"text": "Test"
},
{
"id": 1681,
"logprob": -8.8515625,
"text": " request"
}
],
"prefill": [],
"seed": 0,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_qwen2/test_flash_qwen2_load.json
View file @ 5df80590
......@@ -4,18 +4,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 2271,
"logprob": null,
"text": "Test"
},
{
"id": 1681,
"logprob": -8.8515625,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -88,18 +77,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 2271,
"logprob": null,
"text": "Test"
},
{
"id": 1681,
"logprob": -8.8515625,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -172,18 +150,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 2271,
"logprob": null,
"text": "Test"
},
{
"id": 1681,
"logprob": -8.8515625,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -256,18 +223,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 2271,
"logprob": null,
"text": "Test"
},
{
"id": 1681,
"logprob": -8.8515625,
"text": " request"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder.json
View file @ 5df80590
......@@ -3,28 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 563,
"logprob": null,
"text": "def"
},
{
"id": 942,
"logprob": -5.1367188,
"text": " print"
},
{
"id": 62,
"logprob": -0.24450684,
"text": "_"
},
{
"id": 7196,
"logprob": -6.9609375,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder_load.json
View file @ 5df80590
......@@ -4,28 +4,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 563,
"logprob": null,
"text": "def"
},
{
"id": 942,
"logprob": -5.1367188,
"text": " print"
},
{
"id": 62,
"logprob": -0.24450684,
"text": "_"
},
{
"id": 7196,
"logprob": -6.9609375,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -97,28 +76,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 563,
"logprob": null,
"text": "def"
},
{
"id": 942,
"logprob": -5.1367188,
"text": " print"
},
{
"id": 62,
"logprob": -0.24450684,
"text": "_"
},
{
"id": 7196,
"logprob": -6.9609375,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -190,28 +148,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 563,
"logprob": null,
"text": "def"
},
{
"id": 942,
"logprob": -5.1367188,
"text": " print"
},
{
"id": 62,
"logprob": -0.24450684,
"text": "_"
},
{
"id": 7196,
"logprob": -6.9609375,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -283,28 +220,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 563,
"logprob": null,
"text": "def"
},
{
"id": 942,
"logprob": -5.1367188,
"text": " print"
},
{
"id": 62,
"logprob": -0.24450684,
"text": "_"
},
{
"id": 7196,
"logprob": -6.9609375,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder.json
View file @ 5df80590
......@@ -3,28 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 1459,
"logprob": -5.6289062,
"text": " print"
},
{
"id": 81,
"logprob": -1.6005859,
"text": "_"
},
{
"id": 7656,
"logprob": -5.9921875,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
View file @ 5df80590
......@@ -3,28 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 60,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 1459,
"logprob": -5.625,
"text": " print"
},
{
"id": 81,
"logprob": -1.6064453,
"text": "_"
},
{
"id": 7656,
"logprob": -5.9921875,
"text": "hello"
}
],
"prefill": [],
"seed": 0,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_load.json
View file @ 5df80590
......@@ -4,28 +4,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 1459,
"logprob": -5.6289062,
"text": " print"
},
{
"id": 81,
"logprob": -1.6005859,
"text": "_"
},
{
"id": 7656,
"logprob": -5.9921875,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -97,28 +76,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 1459,
"logprob": -5.6289062,
"text": " print"
},
{
"id": 81,
"logprob": -1.6005859,
"text": "_"
},
{
"id": 7656,
"logprob": -5.9921875,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -190,28 +148,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 1459,
"logprob": -5.6289062,
"text": " print"
},
{
"id": 81,
"logprob": -1.6005859,
"text": "_"
},
{
"id": 7656,
"logprob": -5.9921875,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -283,28 +220,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 1459,
"logprob": -5.6289062,
"text": " print"
},
{
"id": 81,
"logprob": -1.6005859,
"text": "_"
},
{
"id": 7656,
"logprob": -5.9921875,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2.json
View file @ 5df80590
......@@ -3,28 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 610,
"logprob": null,
"text": "def"
},
{
"id": 1489,
"logprob": -5.2617188,
"text": " print"
},
{
"id": 100,
"logprob": -0.38476562,
"text": "_"
},
{
"id": 7670,
"logprob": -7.640625,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
View file @ 5df80590
......@@ -3,28 +3,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 60,
"prefill": [
{
"id": 610,
"logprob": null,
"text": "def"
},
{
"id": 1489,
"logprob": -5.265625,
"text": " print"
},
{
"id": 100,
"logprob": -0.38305664,
"text": "_"
},
{
"id": 7670,
"logprob": -7.640625,
"text": "hello"
}
],
"prefill": [],
"seed": 0,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_load.json
View file @ 5df80590
......@@ -4,28 +4,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 610,
"logprob": null,
"text": "def"
},
{
"id": 1489,
"logprob": -5.2617188,
"text": " print"
},
{
"id": 100,
"logprob": -0.38476562,
"text": "_"
},
{
"id": 7670,
"logprob": -7.640625,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -98,28 +77,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 610,
"logprob": null,
"text": "def"
},
{
"id": 1489,
"logprob": -5.2617188,
"text": " print"
},
{
"id": 100,
"logprob": -0.38476562,
"text": "_"
},
{
"id": 7670,
"logprob": -7.640625,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -192,28 +150,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 610,
"logprob": null,
"text": "def"
},
{
"id": 1489,
"logprob": -5.2617188,
"text": " print"
},
{
"id": 100,
"logprob": -0.38476562,
"text": "_"
},
{
"id": 7670,
"logprob": -7.640625,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......@@ -286,28 +223,7 @@
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 610,
"logprob": null,
"text": "def"
},
{
"id": 1489,
"logprob": -5.2617188,
"text": " print"
},
{
"id": 100,
"logprob": -0.38476562,
"text": "_"
},
{
"id": 7670,
"logprob": -7.640625,
"text": "hello"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json
View file @ 5df80590
......@@ -3,68 +3,7 @@
"best_of_sequences": null,
"finish_reason": "eos_token",
"generated_tokens": 2,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 3226,
"logprob": -9.0234375,
"text": " ge"
},
{
"id": 21017,
"logprob": -9.0859375,
"text": "ometric"
},
{
"id": 81,
"logprob": -0.25585938,
"text": "_"
},
{
"id": 6009,
"logprob": -2.1972656,
"text": "mean"
},
{
"id": 26,
"logprob": -0.2998047,
"text": "("
},
{
"id": 62,
"logprob": -5.6445312,
"text": "L"
},
{
"id": 44,
"logprob": -3.0839844,
"text": ":"
},
{
"id": 1682,
"logprob": -0.6748047,
"text": " List"
},
{
"id": 77,
"logprob": -0.3864746,
"text": "["
},
{
"id": 1808,
"logprob": -0.9355469,
"text": "float"
},
{
"id": 10794,
"logprob": -2.5371094,
"text": "]):"
}
],
"prefill": [],
"seed": null,
"tokens": [
{
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment