Skip to content

GitLab

Unverified Commit dbdc587d authored by OlivierDehaene's avatar OlivierDehaene Committed by GitHub
Browse files

feat(integration-tests): improve comparison and health checks (#336)

parent e71471be
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1485 additions and 995 deletions
integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder.json 0 → 100644
View file @ dbdc587d
{
"details": {
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 563,
"logprob": null,
"text": "def"
},
{
"id": 942,
"logprob": -5.1367188,
"text": " print"
},
{
"id": 62,
"logprob": -0.24450684,
"text": "_"
},
{
"id": 7196,
"logprob": -6.9609375,
"text": "hello"
}
],
"seed": null,
"tokens": [
{
"id": 1241,
"logprob": -0.9863281,
"special": false,
"text": "():"
},
{
"id": 258,
"logprob": -0.21447754,
"special": false,
"text": "\n "
},
{
"id": 942,
"logprob": -0.43701172,
"special": false,
"text": " print"
},
{
"id": 372,
"logprob": -0.5361328,
"special": false,
"text": "(\""
},
{
"id": 7371,
"logprob": -0.44555664,
"special": false,
"text": "Hello"
},
{
"id": 9956,
"logprob": -1.2412109,
"special": false,
"text": " World"
},
{
"id": 8657,
"logprob": -0.7583008,
"special": false,
"text": "!\")"
},
{
"id": 185,
"logprob": -0.76171875,
"special": false,
"text": "\n"
},
{
"id": 185,
"logprob": -0.20837402,
"special": false,
"text": "\n"
},
{
"id": 1018,
"logprob": -1.2470703,
"special": false,
"text": "print"
}
]
},
"generated_text": "():\n print(\"Hello World!\")\n\nprint"
}
integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder.json 0 → 100644
View file @ dbdc587d
{
"details": {
"best_of_sequences": null,
"finish_reason": "length",
"generated_tokens": 10,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 1459,
"logprob": -5.6289062,
"text": " print"
},
{
"id": 81,
"logprob": -1.6005859,
"text": "_"
},
{
"id": 7656,
"logprob": -5.9921875,
"text": "hello"
}
],
"seed": null,
"tokens": [
{
"id": 2262,
"logprob": -0.7705078,
"special": false,
"text": "():"
},
{
"id": 284,
"logprob": -0.2590332,
"special": false,
"text": "\n "
},
{
"id": 1459,
"logprob": -0.39379883,
"special": false,
"text": " print"
},
{
"id": 440,
"logprob": -0.61376953,
"special": false,
"text": "(\""
},
{
"id": 8279,
"logprob": -0.47338867,
"special": false,
"text": "Hello"
},
{
"id": 10896,
"logprob": -1.5068359,
"special": false,
"text": " World"
},
{
"id": 657,
"logprob": -0.80810547,
"special": false,
"text": "\")"
},
{
"id": 203,
"logprob": -0.7397461,
"special": false,
"text": "\n"
},
{
"id": 203,
"logprob": -0.35229492,
"special": false,
"text": "\n"
},
{
"id": 589,
"logprob": -1.0371094,
"special": false,
"text": "def"
}
]
},
"generated_text": "():\n print(\"Hello World\")\n\ndef"
}
integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json 0 → 100644
View file @ dbdc587d
{
"details": {
"best_of_sequences": null,
"finish_reason": "eos_token",
"generated_tokens": 12,
"prefill": [
{
"id": 589,
"logprob": null,
"text": "def"
},
{
"id": 1459,
"logprob": -5.6289062,
"text": " print"
},
{
"id": 81,
"logprob": -1.6005859,
"text": "_"
},
{
"id": 7656,
"logprob": -5.9921875,
"text": "hello"
}
],
"seed": 0,
"tokens": [
{
"id": 2262,
"logprob": -0.7451172,
"special": false,
"text": "():"
},
{
"id": 284,
"logprob": -0.21325684,
"special": false,
"text": "\n "
},
{
"id": 5741,
"logprob": -5.734375,
"special": false,
"text": " logging"
},
{
"id": 32,
"logprob": 0.0,
"special": false,
"text": "."
},
{
"id": 1338,
"logprob": -0.3232422,
"special": false,
"text": "info"
},
{
"id": 463,
"logprob": -1.0380859,
"special": false,
"text": "('"
},
{
"id": 8279,
"logprob": -0.8378906,
"special": false,
"text": "Hello"
},
{
"id": 30,
"logprob": -1.9501953,
"special": false,
"text": ","
},
{
"id": 10896,
"logprob": -1.3476562,
"special": false,
"text": " World"
},
{
"id": 683,
"logprob": -1.796875,
"special": false,
"text": "')"
},
{
"id": 203,
"logprob": -0.9873047,
"special": false,
"text": "\n"
},
{
"id": 0,
"logprob": -0.7495117,
"special": true,
"text": "<|endoftext|>"
}
]
},
"generated_text": "():\n logging.info('Hello, World')\n<|endoftext|>"
}
integration-tests/models/test_bloom_560m.py
View file @ dbdc587d
This diff is collapsed.
integration-tests/models/test_flash_neox.py
View file @ dbdc587d
This diff is collapsed.
integration-tests/models/test_mt0_base.py
View file @ dbdc587d
This diff is collapsed.
integration-tests/models/utils.py deleted 100644 → 0
View file @ e71471be
import time
from aiohttp import ClientConnectorError, ClientOSError, ServerDisconnectedError
from text_generation import AsyncClient
async def health_check(client: AsyncClient, timeout: int = 60):
assert timeout > 0
for _ in range(timeout):
try:
await client.generate("test")
return
except (ClientConnectorError, ClientOSError, ServerDisconnectedError) as e:
time.sleep(1)
raise RuntimeError("Health check failed")
integration-tests/requirements.txt
View file @ dbdc587d
syrupy syrupy
text-generation==0.5.1 text-generation==0.5.2
pytest pytest
pytest-asyncio==0.17.2 pytest-asyncio==0.17.2
docker docker
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment