Unverified Commit baba0389 authored by wang.yuqi's avatar wang.yuqi Committed by GitHub
Browse files

[CI] Increase the threshold of the MTEB RERANK tests (#20615)


Signed-off-by: default avatarwang.yuqi <noooop@126.com>
parent c6c22f16
...@@ -23,7 +23,7 @@ MTEB_EMBED_TOL = 1e-4 ...@@ -23,7 +23,7 @@ MTEB_EMBED_TOL = 1e-4
# See #19344 # See #19344
MTEB_RERANK_TASKS = ["NFCorpus"] MTEB_RERANK_TASKS = ["NFCorpus"]
MTEB_RERANK_LANGS = ["en"] MTEB_RERANK_LANGS = ["en"]
MTEB_RERANK_TOL = 1e-3 MTEB_RERANK_TOL = 2e-3
class VllmMtebEncoder(mteb.Encoder): class VllmMtebEncoder(mteb.Encoder):
......
...@@ -68,7 +68,6 @@ RERANK_MODELS = [ ...@@ -68,7 +68,6 @@ RERANK_MODELS = [
enable_test=False), enable_test=False),
RerankModelInfo("BAAI/bge-reranker-v2-m3", RerankModelInfo("BAAI/bge-reranker-v2-m3",
architecture="XLMRobertaForSequenceClassification", architecture="XLMRobertaForSequenceClassification",
dtype="float32",
enable_test=False) enable_test=False)
] ]
......
...@@ -18,11 +18,8 @@ EMBEDDING_MODELS = [ ...@@ -18,11 +18,8 @@ EMBEDDING_MODELS = [
] ]
RERANK_MODELS = [ RERANK_MODELS = [
RerankModelInfo( RerankModelInfo("jinaai/jina-reranker-v2-base-multilingual",
"jinaai/jina-reranker-v2-base-multilingual", architecture="XLMRobertaForSequenceClassification")
architecture="XLMRobertaForSequenceClassification",
dtype="float32",
)
] ]
......
...@@ -12,11 +12,9 @@ from .mteb_utils import RerankModelInfo, mteb_test_rerank_models ...@@ -12,11 +12,9 @@ from .mteb_utils import RerankModelInfo, mteb_test_rerank_models
RERANK_MODELS = [ RERANK_MODELS = [
RerankModelInfo("Qwen/Qwen3-Reranker-0.6B", RerankModelInfo("Qwen/Qwen3-Reranker-0.6B",
architecture="Qwen3ForSequenceClassification", architecture="Qwen3ForSequenceClassification",
dtype="float32",
enable_test=True), enable_test=True),
RerankModelInfo("Qwen/Qwen3-Reranker-4B", RerankModelInfo("Qwen/Qwen3-Reranker-4B",
architecture="Qwen3ForSequenceClassification", architecture="Qwen3ForSequenceClassification",
dtype="float32",
enable_test=False) enable_test=False)
] ]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment