Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
llama-7B_lambada_0-shot.json 1.95 KB
Newer Older
Julen Etxaniz's avatar
add bloom, xglm and llama results  
Julen Etxaniz committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80 
{
  "results": {
    "lambada_openai_mt_it": {
      "ppl": 3653680.5734235523,
      "ppl_stderr": 197082.9860932525,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "lambada_standard": {
      "ppl": 2460346.8572795168,
      "ppl_stderr": 81216.56551688322,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "lambada_openai_mt_es": {
      "ppl": 3818890.4545065593,
      "ppl_stderr": 197999.05318216747,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "lambada_openai": {
      "ppl": 2817465.092468485,
      "ppl_stderr": 138319.08822004002,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "lambada_openai_mt_fr": {
      "ppl": 2111186.115467981,
      "ppl_stderr": 111724.42842108487,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "lambada_openai_mt_de": {
      "ppl": 1805613.6770583114,
      "ppl_stderr": 97892.78908113715,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "lambada_standard_cloze": {
      "ppl": 6710057.2411258025,
      "ppl_stderr": 169833.90998542923,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "lambada_openai_mt_en": {
      "ppl": 2817465.092468485,
      "ppl_stderr": 138319.08822004002,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "lambada_openai_cloze": {
      "ppl": 255777.71115985065,
      "ppl_stderr": 11345.7709705634,
      "acc": 0.00038812342324859306,
      "acc_stderr": 0.00027441806845051746
    }
  },
  "versions": {
    "lambada_openai_mt_it": 0,
    "lambada_standard": 0,
    "lambada_openai_mt_es": 0,
    "lambada_openai": 0,
    "lambada_openai_mt_fr": 0,
    "lambada_openai_mt_de": 0,
    "lambada_standard_cloze": 0,
    "lambada_openai_mt_en": 0,
    "lambada_openai_cloze": 0
  },
  "config": {
    "model": "hf-causal",
    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
    "num_fewshot": 0,
    "batch_size": "auto",
    "device": "cuda",
    "no_cache": true,
    "limit": null,
    "bootstrap_iters": 100000,
    "description_dict": {}
  }
}