Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
mpt-7b_lambada_0-shot.json 2.2 KB
Newer Older
Rayyyyy's avatar
first add  
Rayyyyy committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80 
{
  "results": {
    "lambada_openai": {
      "ppl": 3.8685393479651173,
      "ppl_stderr": 0.08079962928213236,
      "acc": 0.6834853483407723,
      "acc_stderr": 0.006479978824925181
    },
    "lambada_openai_cloze": {
      "ppl": 26.557641771948866,
      "ppl_stderr": 0.7040059319391023,
      "acc": 0.3964680768484378,
      "acc_stderr": 0.006815007030417622
    },
    "lambada_openai_mt_de": {
      "ppl": 70.11746381055114,
      "ppl_stderr": 4.044349022196945,
      "acc": 0.33766737822627596,
      "acc_stderr": 0.006588623616680426
    },
    "lambada_openai_mt_en": {
      "ppl": 3.8685393479651173,
      "ppl_stderr": 0.08079962928213236,
      "acc": 0.6834853483407723,
      "acc_stderr": 0.006479978824925181
    },
    "lambada_openai_mt_es": {
      "ppl": 67.22873557270539,
      "ppl_stderr": 3.6853291413042055,
      "acc": 0.3694934989326606,
      "acc_stderr": 0.006724504027913079
    },
    "lambada_openai_mt_fr": {
      "ppl": 42.93424455960906,
      "ppl_stderr": 2.3689294613751506,
      "acc": 0.4302348146710654,
      "acc_stderr": 0.006897835015074962
    },
    "lambada_openai_mt_it": {
      "ppl": 65.76277239773485,
      "ppl_stderr": 3.8678074045404967,
      "acc": 0.392004657481079,
      "acc_stderr": 0.006801548708056975
    },
    "lambada_standard": {
      "ppl": 4.922814417244969,
      "ppl_stderr": 0.10844800336124351,
      "acc": 0.619056860081506,
      "acc_stderr": 0.006765617200231529
    },
    "lambada_standard_cloze": {
      "ppl": 109.09886744922697,
      "ppl_stderr": 3.0351063735026287,
      "acc": 0.1674752571317679,
      "acc_stderr": 0.005202187177767864
    }
  },
  "versions": {
    "lambada_openai": 0,
    "lambada_openai_cloze": 0,
    "lambada_openai_mt_de": 0,
    "lambada_openai_mt_en": 0,
    "lambada_openai_mt_es": 0,
    "lambada_openai_mt_fr": 0,
    "lambada_openai_mt_it": 0,
    "lambada_standard": 0,
    "lambada_standard_cloze": 0
  },
  "config": {
    "model": "hf-causal-experimental",
    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
    "num_fewshot": 0,
    "batch_size": "auto",
    "device": "cuda",
    "no_cache": true,
    "limit": null,
    "bootstrap_iters": 100000,
    "description_dict": {}
  }
}