model,language,repository,task,score,coverage,files-executed,generate-tests-for-file-character-count,processing-time,response-character-count,response-no-error,response-no-excess,response-with-code
custom-codestral/codestral-latest,golang,golang/plain,write-tests,10,0,0,800,5604,2243,5,0,5
custom-codestral/codestral-latest,java,java/light,write-tests,10693,10380,83,110200,468588,163278,115,0,115
custom-codestral/codestral-latest,java,java/plain,write-tests,65,50,5,1453,7558,3279,5,0,5
ollama/codeqwen:latest,golang,golang/light,write-tests,1142,880,26,102166,6576932,143822,115,7,114
ollama/codeqwen:latest,golang,golang/plain,write-tests,33,20,3,1112,100705,2747,5,0,5
ollama/codeqwen:latest,java,java/light,write-tests,4559,4280,39,111778,6586080,158931,115,10,115
ollama/codeqwen:latest,java,java/plain,write-tests,22,10,1,1415,130480,3569,5,1,5
ollama/deepseek-coder-v2:16b-lite-instruct-fp16,golang,golang/light,write-tests,716,360,13,109584,6144217,111771,115,113,115
ollama/deepseek-coder-v2:16b-lite-instruct-fp16,golang,golang/plain,write-tests,16,0,1,853,52526,912,5,5,5
ollama/deepseek-coder-v2:16b-lite-instruct-fp16,java,java/plain,write-tests,15,0,0,997,52823,1062,5,5,5
ollama/deepseek-coder-v2:16b-lite-instruct-q8_0,golang,golang/plain,write-tests,15,0,0,808,37659,875,5,5,5
ollama/deepseek-coder-v2:16b-lite-instruct-q8_0,java,java/plain,write-tests,15,0,0,1139,41729,1204,5,5,5
ollama/falcon2:11b-q8_0,golang,golang/light,write-tests,220,50,5,59532,7330504,88162,113,0,52
ollama/falcon2:11b-q8_0,golang,golang/plain,write-tests,9,0,1,2019,224884,2513,5,0,3
ollama/falcon2:11b-q8_0,java,java/plain,write-tests,7,0,0,1895,176765,2360,5,0,2
ollama/granite-code:34b,golang,golang/light,write-tests,1870,1580,51,88121,25069582,104858,115,36,88
ollama/granite-code:34b,golang,golang/plain,write-tests,20,10,1,983,286457,1317,5,1,3
ollama/granite-code:34b-instruct-f16,golang,golang/light,write-tests,2407,2080,68,81006,55499774,93166,115,49,95
ollama/granite-code:34b-instruct-f16,golang,golang/plain,write-tests,26,10,2,765,490149,851,5,4,5
ollama/granite-code:34b-instruct-f16,java,java/light,write-tests,4436,4120,44,104658,52509778,110846,115,70,87
ollama/granite-code:34b-instruct-f16,java,java/plain,write-tests,23,10,2,1043,530825,1079,5,3,3
ollama/granite-code:34b,java,java/light,write-tests,5191,4860,50,95527,19933781,100485,115,76,90
ollama/granite-code:34b,java,java/plain,write-tests,23,10,1,1058,278518,1405,5,3,4
ollama/granite-code:3b-instruct-q8_0,golang,golang/light,write-tests,480,300,17,99456,4841933,100952,115,19,29
ollama/granite-code:3b-instruct-q8_0,golang,golang/plain,write-tests,9,0,1,1102,62323,1587,5,1,2
ollama/granite-code:3b-instruct-q8_0,java,java/light,write-tests,7083,6830,78,78624,4259723,108799,115,10,50
ollama/granite-code:3b-instruct-q8_0,java,java/plain,write-tests,46,30,5,1520,87186,2677,5,1,5
openrouter/01-ai/yi-34b-chat,golang,golang/light,write-tests,2673,2260,73,122408,1817375,124666,115,111,114
openrouter/01-ai/yi-34b-chat,golang,golang/plain,write-tests,48,30,3,975,12741,1037,5,5,5
openrouter/01-ai/yi-34b-chat,java,java/light,write-tests,7157,6760,58,192467,2122127,196690,115,110,114
openrouter/01-ai/yi-34b-chat,java,java/plain,write-tests,25,10,1,1981,26175,2801,5,4,5
openrouter/01-ai/yi-6b,golang,golang/plain,write-tests,5,0,0,648,29306,7437,3,0,2
openrouter/01-ai/yi-6b,java,java/plain,write-tests,9,0,0,3958,46532,12455,5,0,4
openrouter/allenai/olmo-7b-instruct,golang,golang/plain,write-tests,12,0,0,2745,21410,3941,5,2,5
openrouter/allenai/olmo-7b-instruct,java,java/plain,write-tests,10,0,0,2381,26183,6385,5,0,5
openrouter/alpindale/goliath-120b,golang,golang/light,write-tests,277,100,4,73757,2177772,81587,115,17,41
openrouter/alpindale/goliath-120b,golang,golang/plain,write-tests,22,10,1,1550,63083,2659,5,2,4
openrouter/alpindale/goliath-120b,java,java/plain,write-tests,9,0,0,3055,90529,4305,5,0,4
openrouter/anthropic/claude-1.2,golang,golang/light,write-tests,982,750,18,70179,720571,71195,115,49,50
openrouter/anthropic/claude-1.2,golang,golang/plain,write-tests,37,20,2,441,11042,523,5,5,5
openrouter/anthropic/claude-1.2,java,java/light,write-tests,5726,5400,54,88970,677806,90728,115,73,84
openrouter/anthropic/claude-1.2,java,java/plain,write-tests,57,40,4,940,11175,1000,5,4,4
openrouter/anthropic/claude-2.0,golang,golang/light,write-tests,2275,1910,50,72011,733980,79203,115,85,115
openrouter/anthropic/claude-2.0,golang,golang/plain,write-tests,46,30,3,389,13267,916,5,3,5
openrouter/anthropic/claude-2.0,java,java/light,write-tests,4659,4380,48,94519,883829,119110,115,1,115
openrouter/anthropic/claude-2.0,java,java/plain,write-tests,65,50,5,1155,24954,2764,5,0,5
openrouter/anthropic/claude-2.1,golang,golang/light,write-tests,3232,2910,86,64652,858292,87560,115,6,115
openrouter/anthropic/claude-2.1,golang,golang/plain,write-tests,65,50,5,382,22686,1887,5,0,5
openrouter/anthropic/claude-2.1,java,java/light,write-tests,2034,1790,14,91331,999178,126678,115,0,115
openrouter/anthropic/claude-2.1,java,java/plain,write-tests,43,30,3,1118,34181,3010,5,0,5
openrouter/anthropic/claude-3.5-sonnet,golang,golang/light,write-tests,4694,4270,110,103739,751154,106666,115,89,110
openrouter/anthropic/claude-3.5-sonnet,golang,golang/plain,write-tests,70,50,5,618,12578,676,5,5,5
openrouter/anthropic/claude-3.5-sonnet,java,java/light,write-tests,14833,14490,107,144418,829801,150199,115,33,88
openrouter/anthropic/claude-3.5-sonnet,java,java/plain,write-tests,67,50,5,1141,14322,1277,5,3,4
openrouter/anthropic/claude-3-haiku,golang,golang/light,write-tests,2147,1870,48,113372,524390,171395,115,0,114
openrouter/anthropic/claude-3-haiku,golang,golang/plain,write-tests,43,30,3,401,9441,1401,5,0,5
openrouter/anthropic/claude-3-haiku,java,java/light,write-tests,13829,13490,108,120288,495688,187013,115,1,115
openrouter/anthropic/claude-3-haiku,java,java/plain,write-tests,65,50,5,1062,11194,2532,5,0,5
openrouter/anthropic/claude-3-opus,golang,golang/light,write-tests,4493,4070,107,98984,1666517,112990,115,86,115
openrouter/anthropic/claude-3-opus,golang,golang/plain,write-tests,66,50,5,372,18098,807,5,1,5
openrouter/anthropic/claude-3-opus,java,java/light,write-tests,15330,14980,115,133112,2183935,197344,115,5,115
openrouter/anthropic/claude-3-opus,java,java/plain,write-tests,65,50,5,684,24421,1221,5,0,5
openrouter/anthropic/claude-3-sonnet,golang,golang/light,write-tests,3490,3160,94,106460,1048740,170024,115,6,115
openrouter/anthropic/claude-3-sonnet,golang,golang/plain,write-tests,69,50,5,415,11208,712,5,4,5
openrouter/anthropic/claude-3-sonnet,java,java/light,write-tests,13480,13120,103,146897,1069805,195393,115,27,115
openrouter/anthropic/claude-3-sonnet,java,java/plain,write-tests,65,50,5,1222,26793,3506,5,0,5
openrouter/anthropic/claude-instant-1.0,golang,golang/plain,write-tests,5,0,0,795,8374,805,5,0,0
openrouter/anthropic/claude-instant-1.0,java,java/light,write-tests,4939,4780,34,103109,394943,103398,115,5,5
openrouter/anthropic/claude-instant-1.0,java,java/plain,write-tests,27,20,2,881,8359,891,5,0,0
openrouter/anthropic/claude-instant-1.1,golang,golang/light,write-tests,2417,2100,64,70467,363717,74513,115,56,82
openrouter/anthropic/claude-instant-1.1,golang,golang/plain,write-tests,36,20,2,397,7579,565,5,4,5
openrouter/anthropic/claude-instant-1.1,java,java/light,write-tests,2631,2300,27,81280,379108,92721,115,74,115
openrouter/anthropic/claude-instant-1.1,java,java/plain,write-tests,26,10,1,926,6884,991,5,5,5
openrouter/austism/chronos-hermes-13b,golang,golang/light,write-tests,706,430,15,88860,1044606,102920,115,42,104
openrouter/austism/chronos-hermes-13b,golang,golang/plain,write-tests,13,0,1,936,14595,1767,5,2,5
openrouter/austism/chronos-hermes-13b,java,java/plain,write-tests,11,0,0,2106,25942,3291,5,1,5
openrouter/bigcode/starcoder2-15b-instruct,golang,golang/plain,write-tests,10,0,0,748,13639,2714,5,0,5
openrouter/bigcode/starcoder2-15b-instruct,java,java/light,write-tests,2889,2620,26,74388,654447,155315,115,13,115
openrouter/bigcode/starcoder2-15b-instruct,java,java/plain,write-tests,21,10,1,1079,13524,3153,5,0,5
openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/light,write-tests,1892,1590,43,87061,1338542,124020,115,31,113
openrouter/cognitivecomputations/dolphin-mixtral-8x22b,golang,golang/plain,write-tests,22,10,2,677,18846,1905,5,0,5
openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/light,write-tests,1905,1630,17,100022,1156440,138450,115,41,102
openrouter/cognitivecomputations/dolphin-mixtral-8x22b,java,java/plain,write-tests,12,0,1,1296,30834,3668,5,1,5
openrouter/cognitivecomputations/dolphin-mixtral-8x7b,golang,golang/plain,write-tests,15,0,0,1602,12332,1676,5,5,5
openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/light,write-tests,5074,4710,49,203781,1153598,211779,115,88,112
openrouter/cognitivecomputations/dolphin-mixtral-8x7b,java,java/plain,write-tests,47,30,3,2017,10702,2354,5,4,5
openrouter/cohere/command-r-plus,golang,golang/light,write-tests,885,530,12,120774,1218176,122379,115,114,114
openrouter/cohere/command-r-plus,golang,golang/plain,write-tests,26,10,1,900,8946,970,5,5,5
openrouter/cohere/command-r-plus,java,java/light,write-tests,11731,11300,94,149567,953962,151670,115,107,115
openrouter/cohere/command-r-plus,java,java/plain,write-tests,70,50,5,1247,9424,1307,5,5,5
openrouter/databricks/dbrx-instruct,golang,golang/light,write-tests,2163,1860,53,89763,533449,93082,95,73,82
openrouter/databricks/dbrx-instruct,golang,golang/plain,write-tests,28,20,2,213,3935,248,3,1,2
openrouter/databricks/dbrx-instruct,java,java/light,write-tests,4722,4440,35,84748,399567,91525,98,68,81
openrouter/databricks/dbrx-instruct,java,java/plain,write-tests,57,40,4,763,5884,813,5,4,4
openrouter/deepseek/deepseek-chat,golang,golang/light,write-tests,2417,2010,66,128991,3175117,130579,115,113,113
openrouter/deepseek/deepseek-chat,golang,golang/plain,write-tests,70,50,5,370,19077,440,5,5,5
openrouter/deepseek/deepseek-chat,java,java/light,write-tests,13719,13270,108,159577,3124177,160939,115,113,113
openrouter/deepseek/deepseek-chat,java,java/plain,write-tests,70,50,5,940,30618,1000,5,5,5
openrouter/deepseek/deepseek-coder,golang,golang/light,write-tests,4380,3940,105,127058,3084898,128728,115,110,110
openrouter/deepseek/deepseek-coder,golang,golang/plain,write-tests,70,50,5,370,18753,445,5,5,5
openrouter/deepseek/deepseek-coder,java,java/light,write-tests,15460,15000,115,168715,2706856,170210,115,115,115
openrouter/deepseek/deepseek-coder,java,java/plain,write-tests,70,50,5,685,21340,750,5,5,5
openrouter/fireworks/firellava-13b,golang,golang/plain,write-tests,11,0,0,1615,9922,2430,5,1,5
openrouter/fireworks/firellava-13b,java,java/light,write-tests,4925,4590,43,147335,661976,166289,115,63,114
openrouter/fireworks/firellava-13b,java,java/plain,write-tests,23,10,1,2183,11035,2581,5,2,5
openrouter/google/gemini-flash-1.5,golang,golang/light,write-tests,4105,3660,104,115926,629509,117520,115,113,113
openrouter/google/gemini-flash-1.5,golang,golang/plain,write-tests,70,50,5,370,6299,440,5,5,5
openrouter/google/gemini-flash-1.5,java,java/light,write-tests,13434,12990,99,138161,523509,139541,115,115,115
openrouter/google/gemini-flash-1.5,java,java/plain,write-tests,70,50,5,818,7363,878,5,5,5
openrouter/google/gemini-pro-1.5,golang,golang/light,write-tests,4229,3780,104,88374,880070,90013,115,115,115
openrouter/google/gemini-pro-1.5,golang,golang/plain,write-tests,70,50,5,370,8806,444,5,5,5
openrouter/google/gemini-pro-1.5,java,java/light,write-tests,10906,10470,91,108297,824409,109683,115,115,115
openrouter/google/gemini-pro-1.5,java,java/plain,write-tests,70,50,5,810,11185,870,5,5,5
openrouter/google/gemma-7b-it,golang,golang/light,write-tests,1390,1020,30,76200,452235,78535,115,110,115
openrouter/google/gemma-7b-it,golang,golang/plain,write-tests,25,10,1,586,6212,958,5,4,5
openrouter/google/gemma-7b-it,java,java/light,write-tests,3927,3560,26,102667,456540,105409,115,111,115
openrouter/google/gemma-7b-it,java,java/plain,write-tests,26,10,1,685,5890,745,5,5,5
openrouter/google/palm-2-chat-bison,golang,golang/light,write-tests,1845,1480,46,111230,375861,112484,115,102,102
openrouter/google/palm-2-chat-bison,golang,golang/plain,write-tests,59,40,4,481,5073,540,5,5,5
openrouter/google/palm-2-chat-bison,java,java/light,write-tests,2232,1880,13,113187,346876,114649,115,112,112
openrouter/google/palm-2-chat-bison,java,java/plain,write-tests,37,20,2,813,6810,878,5,5,5
openrouter/google/palm-2-codechat-bison,golang,golang/light,write-tests,1464,1100,41,110461,360994,112382,115,102,106
openrouter/google/palm-2-codechat-bison,golang,golang/plain,write-tests,59,40,4,665,5984,740,5,5,5
openrouter/google/palm-2-codechat-bison,java,java/light,write-tests,2675,2340,20,132017,364598,133359,115,100,100
openrouter/google/palm-2-codechat-bison,java,java/plain,write-tests,59,40,4,701,3918,766,5,5,5
openrouter/huggingfaceh4/zephyr-7b-beta,golang,golang/plain,write-tests,0,0,0,0,0,0,0,0,0
openrouter/huggingfaceh4/zephyr-7b-beta,java,java/plain,write-tests,0,0,0,0,0,0,0,0,0
openrouter/intel/neural-chat-7b,golang,golang/plain,write-tests,5,0,0,4056,47982,4061,5,0,0
openrouter/intel/neural-chat-7b,java,java/light,write-tests,3902,3750,37,187853,917061,188016,115,0,0
openrouter/intel/neural-chat-7b,java,java/plain,write-tests,16,10,1,2570,14338,2575,5,0,0
openrouter/liuhaotian/llava-13b,golang,golang/plain,write-tests,0,0,0,0,0,0,0,0,0
openrouter/liuhaotian/llava-13b,java,java/plain,write-tests,0,0,0,0,0,0,0,0,0
openrouter/liuhaotian/llava-yi-34b,golang,golang/plain,write-tests,11,0,0,828,10318,1575,5,1,5
openrouter/liuhaotian/llava-yi-34b,java,java/light,write-tests,2726,2420,21,142756,858339,163456,115,56,114
openrouter/liuhaotian/llava-yi-34b,java,java/plain,write-tests,22,10,1,1476,14034,2773,5,2,4
openrouter/meta-llama/codellama-34b-instruct,golang,golang/light,write-tests,1855,1550,52,93432,1225930,127858,115,32,106
openrouter/meta-llama/codellama-34b-instruct,golang,golang/plain,write-tests,24,10,1,671,13600,1698,5,3,5
openrouter/meta-llama/codellama-34b-instruct,java,java/light,write-tests,6901,6610,55,97066,1104353,148701,115,11,110
openrouter/meta-llama/codellama-34b-instruct,java,java/plain,write-tests,33,20,2,1446,19665,2839,5,1,5
openrouter/meta-llama/llama-3-70b-instruct,golang,golang/light,write-tests,3692,3280,94,77815,1008739,80713,115,88,115
openrouter/meta-llama/llama-3-70b-instruct,golang,golang/plain,write-tests,70,50,5,375,7414,429,5,5,5
openrouter/meta-llama/llama-3-70b-instruct,java,java/light,write-tests,12704,12330,96,124952,1120277,131118,115,48,115
openrouter/meta-llama/llama-3-70b-instruct,java,java/plain,write-tests,56,40,4,1387,12903,1603,5,2,5
openrouter/meta-llama/llama-3-8b-instruct,golang,golang/plain,write-tests,13,0,0,1259,8610,1420,5,3,5
openrouter/meta-llama/llama-3-8b-instruct,java,java/light,write-tests,6400,6090,56,134493,387054,142428,115,32,107
openrouter/meta-llama/llama-3-8b-instruct,java,java/plain,write-tests,34,20,3,1404,7174,1995,5,1,5
openrouter/microsoft/phi-3-medium-128k-instruct,golang,golang/plain,write-tests,0,0,0,0,0,0,0,0,0
openrouter/microsoft/phi-3-medium-128k-instruct,java,java/plain,write-tests,0,0,0,0,0,0,0,0,0
openrouter/microsoft/phi-3-medium-4k-instruct,golang,golang/plain,write-tests,9,0,0,2487,11825,2516,5,2,2
openrouter/microsoft/phi-3-medium-4k-instruct,java,java/light,write-tests,5692,5320,52,148769,692551,157448,115,90,115
openrouter/microsoft/phi-3-medium-4k-instruct,java,java/plain,write-tests,33,20,2,4167,22394,6104,5,1,5
openrouter/microsoft/phi-3-mini-128k-instruct,golang,golang/plain,write-tests,0,0,0,0,0,0,0,0,0
openrouter/microsoft/phi-3-mini-128k-instruct,java,java/plain,write-tests,0,0,0,0,0,0,0,0,0
openrouter/microsoft/wizardlm-2-7b,golang,golang/plain,write-tests,10,0,0,2671,18735,7017,5,0,5
openrouter/microsoft/wizardlm-2-7b,java,java/light,write-tests,5936,5660,46,311674,1149528,410253,115,0,115
openrouter/microsoft/wizardlm-2-7b,java,java/plain,write-tests,21,10,1,5545,35629,14316,5,0,5
openrouter/microsoft/wizardlm-2-8x22b,golang,golang/light,write-tests,2749,2440,79,152231,3382366,241131,115,0,115
openrouter/microsoft/wizardlm-2-8x22b,golang,golang/plain,write-tests,65,50,5,1531,40884,4177,5,0,5
openrouter/microsoft/wizardlm-2-8x22b,java,java/light,write-tests,13130,12800,100,208201,3059641,299371,115,0,115
openrouter/microsoft/wizardlm-2-8x22b,java,java/plain,write-tests,65,50,5,1951,56252,6875,5,0,5
openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/light,write-tests,1527,1250,37,115379,859165,160290,115,13,112
openrouter/mistralai/mistral-7b-instruct-v0.3,golang,golang/plain,write-tests,43,30,3,1406,12352,3348,5,0,5
openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/light,write-tests,7870,7570,65,153118,875056,220888,115,7,113
openrouter/mistralai/mistral-7b-instruct-v0.3,java,java/plain,write-tests,43,30,3,1826,23894,6980,5,0,5
openrouter/mistralai/mistral-large,golang,golang/plain,write-tests,10,0,0,802,37634,4776,5,0,5
openrouter/mistralai/mistral-large,java,java/light,write-tests,13277,12940,107,109972,1385913,171646,115,0,115
openrouter/mistralai/mistral-large,java,java/plain,write-tests,65,50,5,699,27340,3356,5,0,5
openrouter/mistralai/mistral-medium,golang,golang/light,write-tests,2689,2370,74,103334,1540298,155390,115,15,115
openrouter/mistralai/mistral-medium,golang,golang/plain,write-tests,65,50,5,449,21213,3387,5,0,5
openrouter/mistralai/mistral-medium,java,java/light,write-tests,12253,11920,99,151196,1766379,216326,115,4,115
openrouter/mistralai/mistral-medium,java,java/plain,write-tests,65,50,5,1196,20492,2874,5,0,5
openrouter/mistralai/mistral-small,golang,golang/light,write-tests,2235,1870,63,114634,1594233,129280,115,72,115
openrouter/mistralai/mistral-small,golang,golang/plain,write-tests,67,50,5,375,11133,1196,5,2,5
openrouter/mistralai/mistral-small,java,java/light,write-tests,10909,10550,95,152399,1658399,184976,115,34,115
openrouter/mistralai/mistral-small,java,java/plain,write-tests,66,50,5,1234,18941,2214,5,1,5
openrouter/mistralai/mistral-tiny,golang,golang/light,write-tests,1508,1240,35,121129,847432,171540,115,3,115
openrouter/mistralai/mistral-tiny,golang,golang/plain,write-tests,32,20,2,1151,13573,3964,5,0,5
openrouter/mistralai/mistral-tiny,java,java/light,write-tests,7396,7100,66,150461,870221,234512,115,0,115
openrouter/mistralai/mistral-tiny,java,java/plain,write-tests,54,40,4,1663,18121,5567,5,0,5
openrouter/mistralai/mixtral-8x22b-instruct,golang,golang/plain,write-tests,10,0,0,2129,48360,7614,5,0,5
openrouter/mistralai/mixtral-8x22b-instruct,java,java/light,write-tests,12911,12550,104,137237,1011539,181343,115,27,115
openrouter/mistralai/mixtral-8x22b-instruct,java,java/plain,write-tests,65,50,5,1182,18898,2968,5,0,5
openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/light,write-tests,2465,2120,65,133437,785198,159578,115,52,113
openrouter/mistralai/mixtral-8x7b-instruct,golang,golang/plain,write-tests,14,0,1,1472,8540,2674,5,3,5
openrouter/mistralai/mixtral-8x7b-instruct,java,java/light,write-tests,11521,11130,97,148446,726097,164002,115,64,115
openrouter/mistralai/mixtral-8x7b-instruct,java,java/plain,write-tests,65,50,5,1262,13394,2514,5,0,5
openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/light,write-tests,942,720,22,78261,1620320,79277,115,42,43
openrouter/neversleep/llama-3-lumimaid-70b,golang,golang/plain,write-tests,33,20,2,884,19817,947,5,3,3
openrouter/neversleep/llama-3-lumimaid-70b,java,java/light,write-tests,3757,3530,25,106448,1671549,108967,115,42,45
openrouter/neversleep/llama-3-lumimaid-70b,java,java/plain,write-tests,19,10,1,1281,21385,1503,5,1,2
openrouter/neversleep/llama-3-lumimaid-8b,golang,golang/plain,write-tests,10,0,0,1443,16302,1738,5,2,3
openrouter/neversleep/llama-3-lumimaid-8b,java,java/light,write-tests,3020,2710,28,149745,942512,156125,115,71,96
openrouter/neversleep/llama-3-lumimaid-8b,java,java/plain,write-tests,36,20,2,4794,20991,5072,5,4,5
openrouter/nousresearch/hermes-2-pro-llama-3-8b,golang,golang/plain,write-tests,8,0,0,1837,11419,3110,5,0,3
openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/light,write-tests,8232,7880,68,138934,538773,168730,115,59,110
openrouter/nousresearch/hermes-2-pro-llama-3-8b,java,java/plain,write-tests,34,20,3,1047,21585,6653,5,1,5
openrouter/nousresearch/nous-capybara-34b,golang,golang/plain,write-tests,15,0,0,775,7428,830,5,5,5
openrouter/nousresearch/nous-capybara-34b,java,java/light,write-tests,3660,3300,33,178491,947570,181896,115,101,111
openrouter/nousresearch/nous-capybara-34b,java,java/plain,write-tests,25,10,1,2419,13808,2571,5,4,5
openrouter/nousresearch/nous-capybara-7b,golang,golang/plain,write-tests,10,0,0,732,12871,986,5,2,3
openrouter/nousresearch/nous-capybara-7b,java,java/light,write-tests,6549,6210,57,104103,961295,113468,115,76,91
openrouter/nousresearch/nous-capybara-7b,java,java/plain,write-tests,22,10,1,7048,45837,7102,5,3,3
openrouter/nousresearch/nous-hermes-2-mistral-7b-dpo,golang,golang/plain,write-tests,11,0,0,1193,9913,2929,5,1,5
openrouter/nousresearch/nous-hermes-2-mistral-7b-dpo,java,java/light,write-tests,7860,7500,69,148776,419297,169238,115,65,111
openrouter/nousresearch/nous-hermes-2-mistral-7b-dpo,java,java/plain,write-tests,58,40,4,1354,5280,1543,5,4,5
openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/light,write-tests,1903,1610,49,124350,1213729,160793,115,31,98
openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,golang,golang/plain,write-tests,32,20,3,1142,16951,3580,5,0,4
openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/light,write-tests,9076,8740,81,175456,1122654,225112,115,29,111
openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo,java,java/plain,write-tests,45,30,5,1505,18132,3528,5,0,5
openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-sft,golang,golang/plain,write-tests,7,0,0,3278,13485,3304,5,1,1
openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-sft,java,java/light,write-tests,7118,6760,68,142701,485877,153593,115,70,105
openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-sft,java,java/plain,write-tests,70,50,5,1206,5731,1266,5,5,5
openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/light,write-tests,1720,1390,48,203572,3266918,226202,115,81,86
openrouter/nousresearch/nous-hermes-yi-34b,golang,golang/plain,write-tests,25,10,2,1172,28814,1220,5,4,4
openrouter/nousresearch/nous-hermes-yi-34b,java,java/light,write-tests,3767,3490,29,290760,3342362,293205,115,64,69
openrouter/nousresearch/nous-hermes-yi-34b,java,java/plain,write-tests,42,30,3,868,18338,892,5,2,2
openrouter/openai/gpt-4,golang,golang/light,write-tests,3484,3080,85,85785,1957962,94543,115,89,115
openrouter/openai/gpt-4,golang,golang/plain,write-tests,68,50,5,412,19715,745,5,3,5
openrouter/openai/gpt-4,java,java/light,write-tests,14576,14140,112,109617,1816016,117885,115,94,115
openrouter/openai/gpt-4,java,java/plain,write-tests,70,50,5,1211,22583,1271,5,5,5
openrouter/openai/gpt-4o,golang,golang/light,write-tests,3746,3310,91,78841,664069,80454,115,115,115
openrouter/openai/gpt-4o,golang,golang/plain,write-tests,70,50,5,370,6452,440,5,5,5
openrouter/openai/gpt-4o,java,java/light,write-tests,15350,14890,115,122188,720181,123569,115,115,115
openrouter/openai/gpt-4o,java,java/plain,write-tests,70,50,5,822,10904,882,5,5,5
openrouter/openai/gpt-4-turbo,golang,golang/light,write-tests,3171,2750,76,102972,1595053,104561,115,115,115
openrouter/openai/gpt-4-turbo,golang,golang/plain,write-tests,70,50,5,658,13750,720,5,5,5
openrouter/openai/gpt-4-turbo,java,java/light,write-tests,7988,7580,65,145297,1626835,147211,115,113,115
openrouter/openai/gpt-4-turbo,java,java/plain,write-tests,70,50,5,1095,15195,1155,5,5,5
openrouter/openchat/openchat-8b,golang,golang/light,write-tests,1902,1590,47,87217,330365,106622,114,47,104
openrouter/openchat/openchat-8b,golang,golang/plain,write-tests,43,30,3,945,6245,1858,5,1,4
openrouter/openchat/openchat-8b,java,java/light,write-tests,5977,5690,53,124851,363928,153126,112,18,104
openrouter/openchat/openchat-8b,java,java/plain,write-tests,22,10,2,1984,39855,8520,5,0,5
openrouter/open-orca/mistral-7b-openorca,golang,golang/plain,write-tests,8,0,0,888,4849,911,4,2,2
openrouter/open-orca/mistral-7b-openorca,java,java/light,write-tests,5467,5150,44,132250,383712,136280,115,74,84
openrouter/open-orca/mistral-7b-openorca,java,java/plain,write-tests,23,10,2,1652,6235,1690,5,3,3
openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/light,write-tests,3979,3540,102,78680,794481,80473,115,107,115
openrouter/perplexity/llama-3-sonar-large-32k-chat,golang,golang/plain,write-tests,70,50,5,370,5524,424,5,5,5
openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/light,write-tests,13071,12700,98,125732,867300,132028,115,43,115
openrouter/perplexity/llama-3-sonar-large-32k-chat,java,java/plain,write-tests,68,50,5,1357,10949,1562,5,3,5
openrouter/perplexity/llama-3-sonar-small-32k-chat,golang,golang/plain,write-tests,9,0,0,1324,7004,1353,5,2,2
openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/light,write-tests,6625,6280,49,141378,469646,145682,115,75,106
openrouter/perplexity/llama-3-sonar-small-32k-chat,java,java/plain,write-tests,24,10,1,1586,5915,1775,5,3,5
openrouter/phind/phind-codellama-34b,golang,golang/light,write-tests,3135,2720,78,80709,860054,85071,115,107,115
openrouter/phind/phind-codellama-34b,golang,golang/plain,write-tests,59,40,4,798,8962,870,5,5,5
openrouter/phind/phind-codellama-34b,java,java/light,write-tests,12017,11590,100,103845,806436,111029,115,97,115
openrouter/phind/phind-codellama-34b,java,java/plain,write-tests,70,50,5,1285,11851,1345,5,5,5
openrouter/qwen/qwen-110b-chat,golang,golang/light,write-tests,2335,2030,62,102746,1072982,151179,115,13,115
openrouter/qwen/qwen-110b-chat,golang,golang/plain,write-tests,21,10,1,3296,32593,6370,5,0,5
openrouter/qwen/qwen-110b-chat,java,java/light,write-tests,1747,1480,10,137919,1108130,181750,115,28,114
openrouter/qwen/qwen-110b-chat,java,java/plain,write-tests,21,10,1,1913,31383,6028,5,0,5
openrouter/qwen/qwen-14b-chat,golang,golang/light,write-tests,873,620,22,162455,998485,223582,115,1,115
openrouter/qwen/qwen-14b-chat,golang,golang/plain,write-tests,32,20,2,1286,12595,3834,5,0,5
openrouter/qwen/qwen-14b-chat,java,java/light,write-tests,3368,3110,28,161406,908450,253741,115,0,115
openrouter/qwen/qwen-14b-chat,java,java/plain,write-tests,21,10,1,2519,18408,5965,5,0,5
openrouter/qwen/qwen-2-72b-instruct,golang,golang/plain,write-tests,15,0,0,798,58560,869,5,5,5
openrouter/qwen/qwen-2-72b-instruct,java,java/light,write-tests,12176,11730,101,109315,1706394,110725,115,115,115
openrouter/qwen/qwen-2-72b-instruct,java,java/plain,write-tests,70,50,5,1727,9444,1789,5,5,5
openrouter/qwen/qwen-32b-chat,golang,golang/light,write-tests,1211,940,30,122200,1096586,152664,115,17,109
openrouter/qwen/qwen-32b-chat,golang,golang/plain,write-tests,43,30,3,640,16806,2916,5,0,5
openrouter/qwen/qwen-32b-chat,java,java/light,write-tests,7278,6950,67,137768,920518,163234,115,33,113
openrouter/qwen/qwen-32b-chat,java,java/plain,write-tests,54,40,4,1657,24898,4970,5,0,5
openrouter/qwen/qwen-4b-chat,golang,golang/plain,write-tests,13,0,0,1502,6558,2073,5,3,5
openrouter/qwen/qwen-4b-chat,java,java/light,write-tests,721,440,4,160313,535892,172729,115,62,100
openrouter/qwen/qwen-4b-chat,java,java/plain,write-tests,34,20,2,2780,8721,3760,5,2,5
openrouter/qwen/qwen-72b-chat,golang,golang/light,write-tests,2530,2230,67,87121,1277271,116896,115,4,114
openrouter/qwen/qwen-72b-chat,golang,golang/plain,write-tests,32,20,2,933,75088,2078,5,0,5
openrouter/qwen/qwen-72b-chat,java,java/light,write-tests,9975,9610,84,129068,1145755,149624,115,51,115
openrouter/qwen/qwen-72b-chat,java,java/plain,write-tests,54,40,4,1175,92724,4270,5,0,5
openrouter/qwen/qwen-7b-chat,golang,golang/plain,write-tests,10,0,0,2793,15122,5678,5,0,5
openrouter/qwen/qwen-7b-chat,java,java/plain,write-tests,10,0,0,5350,18001,9045,5,0,5
openrouter/recursal/eagle-7b,golang,golang/plain,write-tests,9,0,0,2478,29079,3064,5,0,4
openrouter/recursal/eagle-7b,java,java/light,write-tests,1786,1540,22,86308,1290418,146705,115,20,89
openrouter/recursal/eagle-7b,java,java/plain,write-tests,32,20,3,1090,57326,7707,5,0,4
openrouter/recursal/rwkv-5-3b-ai-town,golang,golang/plain,write-tests,10,0,0,275,15207,575,5,0,5
openrouter/recursal/rwkv-5-3b-ai-town,java,java/plain,write-tests,10,0,0,930,25278,1315,5,0,5
openrouter/rwkv/rwkv-5-world-3b,golang,golang/plain,write-tests,15,0,0,518,19347,582,5,5,5
openrouter/rwkv/rwkv-5-world-3b,java,java/plain,write-tests,15,0,0,1190,32835,1260,5,5,5
openrouter/snowflake/snowflake-arctic-instruct,golang,golang/light,write-tests,3016,2640,77,67781,535683,84257,115,70,114
openrouter/snowflake/snowflake-arctic-instruct,golang,golang/plain,write-tests,46,30,3,428,6816,940,5,3,5
openrouter/snowflake/snowflake-arctic-instruct,java,java/light,write-tests,10162,9810,89,82848,543904,114245,115,36,112
openrouter/snowflake/snowflake-arctic-instruct,java,java/plain,write-tests,66,50,5,1141,12271,2463,5,1,5
openrouter/teknium/openhermes-2.5-mistral-7b,golang,golang/plain,write-tests,13,0,0,980,10249,1511,5,3,5
openrouter/teknium/openhermes-2.5-mistral-7b,java,java/light,write-tests,9398,9010,81,121506,473230,131093,115,81,111
openrouter/teknium/openhermes-2.5-mistral-7b,java,java/plain,write-tests,47,30,3,2962,13750,3337,5,4,5
openrouter/teknium/openhermes-2-mistral-7b,golang,golang/plain,write-tests,14,0,0,1633,8178,1917,5,4,5
openrouter/teknium/openhermes-2-mistral-7b,java,java/light,write-tests,6635,6300,60,130839,381356,147907,115,55,105
openrouter/teknium/openhermes-2-mistral-7b,java,java/plain,write-tests,36,20,3,1903,7019,1951,5,4,4
openrouter/togethercomputer/stripedhyena-hessian-7b,golang,golang/plain,write-tests,5,0,0,8013,27085,8882,4,0,1
openrouter/togethercomputer/stripedhyena-hessian-7b,java,java/plain,write-tests,6,0,0,3087,14438,4634,5,0,1
openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/light,write-tests,1035,710,22,91746,449905,105236,115,75,113
openrouter/togethercomputer/stripedhyena-nous-7b,golang,golang/plain,write-tests,24,10,1,1547,9682,2631,5,3,5
openrouter/togethercomputer/stripedhyena-nous-7b,java,java/light,write-tests,5096,4750,52,106428,431161,127731,115,67,112
openrouter/togethercomputer/stripedhyena-nous-7b,java,java/plain,write-tests,14,0,1,3022,12392,3264,5,3,5
openrouter/undi95/toppy-m-7b,golang,golang/plain,write-tests,12,0,0,2300,12493,2874,5,3,4
openrouter/undi95/toppy-m-7b,java,java/light,write-tests,3825,3540,28,135475,514422,177124,115,31,111
openrouter/undi95/toppy-m-7b,java,java/plain,write-tests,21,10,1,2200,16178,4166,5,0,5
openrouter/xwin-lm/xwin-lm-70b,golang,golang/light,write-tests,427,250,13,73431,2801748,92691,115,0,49
openrouter/xwin-lm/xwin-lm-70b,golang,golang/plain,write-tests,21,10,1,711,58655,2609,5,0,5
openrouter/xwin-lm/xwin-lm-70b,java,java/light,write-tests,2496,2300,27,109769,2695450,133800,115,0,54
openrouter/xwin-lm/xwin-lm-70b,java,java/plain,write-tests,32,20,2,945,69134,3427,5,0,5
symflower/symbolic-execution,golang,golang/light,write-tests,345,0,0,0,49735,0,115,115,115
symflower/symbolic-execution,golang,golang/plain,write-tests,70,50,5,1270,3415,1270,5,5,5
symflower/symbolic-execution,java,java/light,write-tests,14060,13600,115,132675,1314850,132675,115,115,115
symflower/symbolic-execution,java,java/plain,write-tests,70,50,5,695,83595,695,5,5,5
