Add Mistral-ORPO-Beta to AlpacaEval

tatsu-lab · Mar 16, 2024 · 55da734 · 55da734
1 parent 850a570
commit 55da734
Show file tree

Hide file tree

Showing 7 changed files with 80,982 additions and 0 deletions.
diff --git a/results/mistral-orpo-beta/alpaca_eval_gpt4/annotations.json b/results/mistral-orpo-beta/alpaca_eval_gpt4/annotations.json
diff --git a/results/mistral-orpo-beta/model_outputs.json b/results/mistral-orpo-beta/model_outputs.json
diff --git a/results/mistral-orpo-beta/weighted_alpaca_eval_gpt4_turbo/annotations.json b/results/mistral-orpo-beta/weighted_alpaca_eval_gpt4_turbo/annotations.json
diff --git a/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv b/src/alpaca_eval/leaderboards/data_AlpacaEval/alpaca_eval_gpt4_leaderboard.csv
@@ -21,6 +21,7 @@ xwinlm-13b-v0.1,91.76029963,0.968139439,734,65,2,801,community,1894,
 ultralm-13b-best-of-16,91.54228856,0.981927769,736,68,0,804,community,1980,
 cut-13b,91.35572139303484,0.9897323784630048,734,69,1,804,community,1637,
 claude-2,91.35572139,0.989732378,734,69,1,804,minimal,1069,
+mistral-orpo-beta,91.15815691158157,0.9986146698437476,731,70,2,803,community,1636,91.15815691158157
 pairrm-tulu-2-13b,91.055900621118,1.0064549282620592,733,72,0,805,community,1454,
 cohere,90.62111801242236,1.022512757790303,728,74,3,805,verified,1983,
 zephyr-7b-beta,90.5977584059776,1.0287080531312012,727,75,1,803,community,1444,

diff --git a/...lpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv b/...lpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv
@@ -50,6 +50,7 @@ pairrm-zephyr-7b-beta,12.841278255668827,1.0535874941883565,98,706,1,805,12.2360
 gpt-3.5-turbo-1106_verbose,12.763169810361555,1.044246819210942,94,709,2,805,11.801242236024844,dev,1058
 vicuna-33b-v1.3,12.70594792156943,0.9992557843028602,90,711,4,805,11.428571428571429,verified,1479
 deita-7b-v1.0,12.646639472549904,1.0352555320822787,96,708,1,805,11.987577639751551,community,1417
+mistral-orpo-beta,12.565408794446087,0.9929774686149236,95,707,3,805,11.987577639751551,community,1636
 deepseek-llm-67b-chat,12.093422264943865,1.0173843632915984,90,713,2,805,11.304347826086955,community,1151
 wizardlm-13b-v1.2,12.02748034290454,0.9717618177440568,82,720,3,805,10.372670807453416,community,1635
 ultralm-13b-best-of-16,11.307314947672785,0.9418434058702704,80,723,2,805,10.062111801242237,community,1980

diff --git a/src/alpaca_eval/models_configs/mistral-orpo-beta/configs.yaml b/src/alpaca_eval/models_configs/mistral-orpo-beta/configs.yaml
@@ -0,0 +1,12 @@
+mistral-orpo-beta:
+  prompt_template: "mistral-orpo-beta/prompt.txt"
+  fn_completions: "huggingface_local_completions"
+  completions_kwargs:
+    model_name: "kaist-ai/mistral-orpo-beta"
+    model_kwargs:
+      torch_dtype: 'bfloat16'
+    max_new_tokens: 2048
+    temperature: 0.7
+    do_sample: True
+  pretty_name: "Mistral-ORPO-Beta"
+  link: "https://huggingface.co/kaist-ai/mistral-orpo-beta"
diff --git a/src/alpaca_eval/models_configs/mistral-orpo-beta/prompt.txt b/src/alpaca_eval/models_configs/mistral-orpo-beta/prompt.txt
@@ -0,0 +1,3 @@
+<|user|>
+{instruction}</s>
+<|assistant|>