-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathrun.sh
50 lines (48 loc) · 6.14 KB
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
#!/bin/bash
# Bayesian Dawid Skene
# no prior
python main.py --estimator None --dataset Hanna --calibrator BayesianDawidSkene --compare_models All --plot_dir results/plots/hanna/no_prior
python main.py --estimator None --dataset Meva --calibrator BayesianDawidSkene --compare_models All --plot_dir results/plots/meva/no_prior
python main.py --estimator None --dataset SummEval --calibrator BayesianDawidSkene --compare_models All --plot_dir results/plots/summeval/no_prior
python main.py --estimator None --dataset LLMBar --calibrator BayesianDawidSkene --compare_models All --plot_dir results/plots/llmbar/no_prior
# in-distribution prior -- vary prior ratio
python main.py --estimator BetaBernoulli --dataset Hanna --calibrator BayesianDawidSkene --compare_models All --q_prior_data_ratio 0.2 --plot_dir results/plots/hanna/in_dist_0.2
python main.py --estimator BetaBernoulli --dataset Meva --calibrator BayesianDawidSkene --compare_models All --q_prior_data_ratio 0.2 --plot_dir results/plots/meva/in_dist_0.2
python main.py --estimator BetaBernoulli --dataset SummEval --calibrator BayesianDawidSkene --compare_models All --q_prior_data_ratio 0.2 --plot_dir results/plots/summeval/in_dist_0.2
python main.py --estimator BetaBernoulli --dataset LLMBar --calibrator BayesianDawidSkene --compare_models All --q_prior_data_ratio 0.2 --plot_dir results/plots/llmbar/in_dist_0.2
# in-distribution prior -- cross-validation
python main.py --estimator BetaBernoulli --dataset Hanna --calibrator BayesianDawidSkene --compare_models All --q_prior_cv_folds 5 --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset Meva --calibrator BayesianDawidSkene --compare_models All --q_prior_cv_folds 5 --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset SummEval --calibrator BayesianDawidSkene --compare_models All --q_prior_cv_folds 5 --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset LLMBar --calibrator BayesianDawidSkene --compare_models All --q_prior_cv_folds 5 --plot_dir /dev/null
# out-of-distribution prior
python main.py --estimator BetaBernoulli --dataset Hanna --calibrator BayesianDawidSkene --compare_models All --q_prior_ood --plot_dir results/plots/hanna/out_dist
python main.py --estimator BetaBernoulli --dataset Meva --calibrator BayesianDawidSkene --compare_models All --q_prior_ood --plot_dir results/plots/meva/out_dist
python main.py --estimator BetaBernoulli --dataset SummEval --calibrator BayesianDawidSkene --compare_models All --q_prior_ood --plot_dir results/plots/summeval/out_dist
python main.py --estimator BetaBernoulli --dataset LLMBar --calibrator BayesianDawidSkene --compare_models All --q_prior_ood --plot_dir results/plots/llmbar/out_dist
# golden labels -- vary prior ratio
python main.py --estimator BetaBernoulli --dataset Hanna --calibrator BayesianDawidSkene --compare_models All --q_prior_data_ratio 0.2 --q_prior_data_usage gold_labels --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset Meva --calibrator BayesianDawidSkene --compare_models All --q_prior_data_ratio 0.2 --q_prior_data_usage gold_labels --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset SummEval --calibrator BayesianDawidSkene --compare_models All --q_prior_data_ratio 0.2 --q_prior_data_usage gold_labels --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset LLMBar --calibrator BayesianDawidSkene --compare_models All --q_prior_data_ratio 0.2 --q_prior_data_usage gold_labels --plot_dir /dev/null
# golden labels -- cross-validation
python main.py --estimator BetaBernoulli --dataset Hanna --calibrator BayesianDawidSkene --compare_models All --q_prior_cv_folds 5 --q_prior_data_usage gold_labels --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset Meva --calibrator BayesianDawidSkene --compare_models All --q_prior_cv_folds 5 --q_prior_data_usage gold_labels --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset SummEval --calibrator BayesianDawidSkene --compare_models All --q_prior_cv_folds 5 --q_prior_data_usage gold_labels --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset LLMBar --calibrator BayesianDawidSkene --compare_models All --q_prior_cv_folds 5 --q_prior_data_usage gold_labels --plot_dir /dev/null
# bwrs
# in-distribution prior -- vary prior ratio
python main.py --estimator BetaBernoulli --dataset Hanna --calibrator None --compare_models All --q_prior_data_ratio 0.2 --plot_dir results/plots/hanna-ours/in_dist_0.2
python main.py --estimator BetaBernoulli --dataset Meva --calibrator None --compare_models All --q_prior_data_ratio 0.2 --plot_dir results/plots/meva-ours/in_dist_0.2
python main.py --estimator BetaBernoulli --dataset SummEval --calibrator None --compare_models All --q_prior_data_ratio 0.2 --plot_dir results/plots/summeval-ours/in_dist_0.2
python main.py --estimator BetaBernoulli --dataset LLMBar --calibrator None --compare_models All --q_prior_data_ratio 0.2 --plot_dir results/plots/llmbar-ours/in_dist_0.2
# in-distribution prior -- cross-validation
python main.py --estimator BetaBernoulli --dataset Hanna --calibrator None --compare_models All --q_prior_cv_folds 5 --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset Meva --calibrator None --compare_models All --q_prior_cv_folds 5 --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset SummEval --calibrator None --compare_models All --q_prior_cv_folds 5 --plot_dir /dev/null
python main.py --estimator BetaBernoulli --dataset LLMBar --calibrator None --compare_models All --q_prior_cv_folds 5 --plot_dir /dev/null
# out-of-distribution prior
python main.py --estimator BetaBernoulli --dataset Hanna --calibrator None --compare_models All --q_prior_ood --plot_dir results/plots/hanna-ours/out_dist
python main.py --estimator BetaBernoulli --dataset Meva --calibrator None --compare_models All --q_prior_ood --plot_dir results/plots/meva-ours/out_dist
python main.py --estimator BetaBernoulli --dataset SummEval --calibrator None --compare_models All --q_prior_ood --plot_dir results/plots/summeval-ours/out_dist
python main.py --estimator BetaBernoulli --dataset LLMBar --calibrator None --compare_models All --q_prior_ood --plot_dir results/plots/llmbar-ours/out_dist