LICENSE.md
README.md
pyproject.toml
setup.py
lm_eval/__init__.py
lm_eval/__main__.py
lm_eval/config.py
lm_eval/evaluator.py
lm_eval/logger.py
lm_eval/scholar_api.py
lm_eval/utils.py
lm_eval/api/__init__.py
lm_eval/api/filter.py
lm_eval/api/instance.py
lm_eval/api/metrics.py
lm_eval/api/model.py
lm_eval/api/registry.py
lm_eval/api/samplers.py
lm_eval/api/task.py
lm_eval/decontamination/__init__.py
lm_eval/decontamination/archiver.py
lm_eval/decontamination/decontaminate.py
lm_eval/decontamination/janitor.py
lm_eval/filters/__init__.py
lm_eval/filters/decontamination.py
lm_eval/filters/extraction.py
lm_eval/filters/selection.py
lm_eval/filters/transformation.py
lm_eval/models/__init__.py
lm_eval/models/anthropic_llms.py
lm_eval/models/dummy.py
lm_eval/models/gguf.py
lm_eval/models/huggingface.py
lm_eval/models/openai_completions.py
lm_eval/models/textsynth.py
lm_eval/models/vllm_causallms.py
lm_eval/prompts/__init__.py
lm_eval/tasks/README.md
lm_eval/tasks/__init__.py
lm_eval/tasks/__pycache__/__init__.cpython-39.pyc
lm_eval/tasks/__pycache__/anli.cpython-39.pyc
lm_eval/tasks/__pycache__/arc.cpython-39.pyc
lm_eval/tasks/__pycache__/arithmetic.cpython-39.pyc
lm_eval/tasks/__pycache__/asdiv.cpython-39.pyc
lm_eval/tasks/__pycache__/babi.cpython-39.pyc
lm_eval/tasks/__pycache__/bigbench.cpython-39.pyc
lm_eval/tasks/__pycache__/blimp.cpython-39.pyc
lm_eval/tasks/__pycache__/cbt.cpython-39.pyc
lm_eval/tasks/__pycache__/ceval.cpython-39.pyc
lm_eval/tasks/__pycache__/cmmlu.cpython-39.pyc
lm_eval/tasks/__pycache__/coqa.cpython-39.pyc
lm_eval/tasks/__pycache__/crowspairs.cpython-39.pyc
lm_eval/tasks/__pycache__/csatqa.cpython-39.pyc
lm_eval/tasks/__pycache__/drop.cpython-39.pyc
lm_eval/tasks/__pycache__/glue.cpython-39.pyc
lm_eval/tasks/__pycache__/gsm8k.cpython-39.pyc
lm_eval/tasks/__pycache__/haerae.cpython-39.pyc
lm_eval/tasks/__pycache__/headqa.cpython-39.pyc
lm_eval/tasks/__pycache__/hellaswag.cpython-39.pyc
lm_eval/tasks/__pycache__/hendrycks_ethics.cpython-39.pyc
lm_eval/tasks/__pycache__/hendrycks_math.cpython-39.pyc
lm_eval/tasks/__pycache__/hendrycks_test.cpython-39.pyc
lm_eval/tasks/__pycache__/json.cpython-39.pyc
lm_eval/tasks/__pycache__/lambada.cpython-39.pyc
lm_eval/tasks/__pycache__/lambada_cloze.cpython-39.pyc
lm_eval/tasks/__pycache__/lambada_multilingual.cpython-39.pyc
lm_eval/tasks/__pycache__/logiqa.cpython-39.pyc
lm_eval/tasks/__pycache__/mathqa.cpython-39.pyc
lm_eval/tasks/__pycache__/mc_taco.cpython-39.pyc
lm_eval/tasks/__pycache__/mgsm.cpython-39.pyc
lm_eval/tasks/__pycache__/mutual.cpython-39.pyc
lm_eval/tasks/__pycache__/naturalqs.cpython-39.pyc
lm_eval/tasks/__pycache__/nqopen.cpython-39.pyc
lm_eval/tasks/__pycache__/openbookqa.cpython-39.pyc
lm_eval/tasks/__pycache__/pawsx.cpython-39.pyc
lm_eval/tasks/__pycache__/pile.cpython-39.pyc
lm_eval/tasks/__pycache__/piqa.cpython-39.pyc
lm_eval/tasks/__pycache__/prost.cpython-39.pyc
lm_eval/tasks/__pycache__/pubmedqa.cpython-39.pyc
lm_eval/tasks/__pycache__/qa4mre.cpython-39.pyc
lm_eval/tasks/__pycache__/qasper.cpython-39.pyc
lm_eval/tasks/__pycache__/quac.cpython-39.pyc
lm_eval/tasks/__pycache__/race.cpython-39.pyc
lm_eval/tasks/__pycache__/sat.cpython-39.pyc
lm_eval/tasks/__pycache__/sciq.cpython-39.pyc
lm_eval/tasks/__pycache__/scrolls.cpython-39.pyc
lm_eval/tasks/__pycache__/squad.cpython-39.pyc
lm_eval/tasks/__pycache__/storycloze.cpython-39.pyc
lm_eval/tasks/__pycache__/superglue.cpython-39.pyc
lm_eval/tasks/__pycache__/swag.cpython-39.pyc
lm_eval/tasks/__pycache__/toxigen.cpython-39.pyc
lm_eval/tasks/__pycache__/translation.cpython-39.pyc
lm_eval/tasks/__pycache__/triviaqa.cpython-39.pyc
lm_eval/tasks/__pycache__/truthfulqa.cpython-39.pyc
lm_eval/tasks/__pycache__/unscramble.cpython-39.pyc
lm_eval/tasks/__pycache__/webqs.cpython-39.pyc
lm_eval/tasks/__pycache__/wikitext.cpython-39.pyc
lm_eval/tasks/__pycache__/winogrande.cpython-39.pyc
lm_eval/tasks/__pycache__/wsc273.cpython-39.pyc
lm_eval/tasks/__pycache__/xcopa.cpython-39.pyc
lm_eval/tasks/__pycache__/xnli.cpython-39.pyc
lm_eval/tasks/__pycache__/xstorycloze.cpython-39.pyc
lm_eval/tasks/__pycache__/xwinograd.cpython-39.pyc
lm_eval/tasks/anli/README.md
lm_eval/tasks/anli/anli_r1.yaml
lm_eval/tasks/anli/anli_r2.yaml
lm_eval/tasks/anli/anli_r3.yaml
lm_eval/tasks/arc/README.md
lm_eval/tasks/arc/arc_challenge.yaml
lm_eval/tasks/arc/arc_easy.yaml
lm_eval/tasks/arithmetic/README.md
lm_eval/tasks/arithmetic/arithmetic_1dc.yaml
lm_eval/tasks/arithmetic/arithmetic_2da.yaml
lm_eval/tasks/arithmetic/arithmetic_2dm.yaml
lm_eval/tasks/arithmetic/arithmetic_2ds.yaml
lm_eval/tasks/arithmetic/arithmetic_3da.yaml
lm_eval/tasks/arithmetic/arithmetic_3ds.yaml
lm_eval/tasks/arithmetic/arithmetic_4da.yaml
lm_eval/tasks/arithmetic/arithmetic_4ds.yaml
lm_eval/tasks/arithmetic/arithmetic_5da.yaml
lm_eval/tasks/arithmetic/arithmetic_5ds.yaml
lm_eval/tasks/asdiv/README.md
lm_eval/tasks/asdiv/default.yaml
lm_eval/tasks/babi/README.md
lm_eval/tasks/babi/babi.yaml
lm_eval/tasks/bbh/README.md
lm_eval/tasks/bbh/_generate_configs.py
lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml
lm_eval/tasks/bbh/cot_fewshot/boolean_expressions.yaml
lm_eval/tasks/bbh/cot_fewshot/causal_judgement.yaml
lm_eval/tasks/bbh/cot_fewshot/date_understanding.yaml
lm_eval/tasks/bbh/cot_fewshot/disambiguation_qa.yaml
lm_eval/tasks/bbh/cot_fewshot/dyck_languages.yaml
lm_eval/tasks/bbh/cot_fewshot/formal_fallacies.yaml
lm_eval/tasks/bbh/cot_fewshot/geometric_shapes.yaml
lm_eval/tasks/bbh/cot_fewshot/hyperbaton.yaml
lm_eval/tasks/bbh/cot_fewshot/logical_deduction_five_objects.yaml
lm_eval/tasks/bbh/cot_fewshot/logical_deduction_seven_objects.yaml
lm_eval/tasks/bbh/cot_fewshot/logical_deduction_three_objects.yaml
lm_eval/tasks/bbh/cot_fewshot/movie_recommendation.yaml
lm_eval/tasks/bbh/cot_fewshot/multistep_arithmetic_two.yaml
lm_eval/tasks/bbh/cot_fewshot/navigate.yaml
lm_eval/tasks/bbh/cot_fewshot/object_counting.yaml
lm_eval/tasks/bbh/cot_fewshot/penguins_in_a_table.yaml
lm_eval/tasks/bbh/cot_fewshot/reasoning_about_colored_objects.yaml
lm_eval/tasks/bbh/cot_fewshot/ruin_names.yaml
lm_eval/tasks/bbh/cot_fewshot/salient_translation_error_detection.yaml
lm_eval/tasks/bbh/cot_fewshot/snarks.yaml
lm_eval/tasks/bbh/cot_fewshot/sports_understanding.yaml
lm_eval/tasks/bbh/cot_fewshot/temporal_sequences.yaml
lm_eval/tasks/bbh/cot_fewshot/tracking_shuffled_objects_five_objects.yaml
lm_eval/tasks/bbh/cot_fewshot/tracking_shuffled_objects_seven_objects.yaml
lm_eval/tasks/bbh/cot_fewshot/tracking_shuffled_objects_three_objects.yaml
lm_eval/tasks/bbh/cot_fewshot/web_of_lies.yaml
lm_eval/tasks/bbh/cot_fewshot/word_sorting.yaml
lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml
lm_eval/tasks/bbh/cot_zeroshot/boolean_expressions.yaml
lm_eval/tasks/bbh/cot_zeroshot/causal_judgement.yaml
lm_eval/tasks/bbh/cot_zeroshot/date_understanding.yaml
lm_eval/tasks/bbh/cot_zeroshot/disambiguation_qa.yaml
lm_eval/tasks/bbh/cot_zeroshot/dyck_languages.yaml
lm_eval/tasks/bbh/cot_zeroshot/formal_fallacies.yaml
lm_eval/tasks/bbh/cot_zeroshot/geometric_shapes.yaml
lm_eval/tasks/bbh/cot_zeroshot/hyperbaton.yaml
lm_eval/tasks/bbh/cot_zeroshot/logical_deduction_five_objects.yaml
lm_eval/tasks/bbh/cot_zeroshot/logical_deduction_seven_objects.yaml
lm_eval/tasks/bbh/cot_zeroshot/logical_deduction_three_objects.yaml
lm_eval/tasks/bbh/cot_zeroshot/movie_recommendation.yaml
lm_eval/tasks/bbh/cot_zeroshot/multistep_arithmetic_two.yaml
lm_eval/tasks/bbh/cot_zeroshot/navigate.yaml
lm_eval/tasks/bbh/cot_zeroshot/object_counting.yaml
lm_eval/tasks/bbh/cot_zeroshot/penguins_in_a_table.yaml
lm_eval/tasks/bbh/cot_zeroshot/reasoning_about_colored_objects.yaml
lm_eval/tasks/bbh/cot_zeroshot/ruin_names.yaml
lm_eval/tasks/bbh/cot_zeroshot/salient_translation_error_detection.yaml
lm_eval/tasks/bbh/cot_zeroshot/snarks.yaml
lm_eval/tasks/bbh/cot_zeroshot/sports_understanding.yaml
lm_eval/tasks/bbh/cot_zeroshot/temporal_sequences.yaml
lm_eval/tasks/bbh/cot_zeroshot/tracking_shuffled_objects_five_objects.yaml
lm_eval/tasks/bbh/cot_zeroshot/tracking_shuffled_objects_seven_objects.yaml
lm_eval/tasks/bbh/cot_zeroshot/tracking_shuffled_objects_three_objects.yaml
lm_eval/tasks/bbh/cot_zeroshot/web_of_lies.yaml
lm_eval/tasks/bbh/cot_zeroshot/word_sorting.yaml
lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml
lm_eval/tasks/bbh/fewshot/boolean_expressions.yaml
lm_eval/tasks/bbh/fewshot/causal_judgement.yaml
lm_eval/tasks/bbh/fewshot/date_understanding.yaml
lm_eval/tasks/bbh/fewshot/disambiguation_qa.yaml
lm_eval/tasks/bbh/fewshot/dyck_languages.yaml
lm_eval/tasks/bbh/fewshot/formal_fallacies.yaml
lm_eval/tasks/bbh/fewshot/geometric_shapes.yaml
lm_eval/tasks/bbh/fewshot/hyperbaton.yaml
lm_eval/tasks/bbh/fewshot/logical_deduction_five_objects.yaml
lm_eval/tasks/bbh/fewshot/logical_deduction_seven_objects.yaml
lm_eval/tasks/bbh/fewshot/logical_deduction_three_objects.yaml
lm_eval/tasks/bbh/fewshot/movie_recommendation.yaml
lm_eval/tasks/bbh/fewshot/multistep_arithmetic_two.yaml
lm_eval/tasks/bbh/fewshot/navigate.yaml
lm_eval/tasks/bbh/fewshot/object_counting.yaml
lm_eval/tasks/bbh/fewshot/penguins_in_a_table.yaml
lm_eval/tasks/bbh/fewshot/reasoning_about_colored_objects.yaml
lm_eval/tasks/bbh/fewshot/ruin_names.yaml
lm_eval/tasks/bbh/fewshot/salient_translation_error_detection.yaml
lm_eval/tasks/bbh/fewshot/snarks.yaml
lm_eval/tasks/bbh/fewshot/sports_understanding.yaml
lm_eval/tasks/bbh/fewshot/temporal_sequences.yaml
lm_eval/tasks/bbh/fewshot/tracking_shuffled_objects_five_objects.yaml
lm_eval/tasks/bbh/fewshot/tracking_shuffled_objects_seven_objects.yaml
lm_eval/tasks/bbh/fewshot/tracking_shuffled_objects_three_objects.yaml
lm_eval/tasks/bbh/fewshot/web_of_lies.yaml
lm_eval/tasks/bbh/fewshot/word_sorting.yaml
lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml
lm_eval/tasks/bbh/zeroshot/boolean_expressions.yaml
lm_eval/tasks/bbh/zeroshot/causal_judgement.yaml
lm_eval/tasks/bbh/zeroshot/date_understanding.yaml
lm_eval/tasks/bbh/zeroshot/disambiguation_qa.yaml
lm_eval/tasks/bbh/zeroshot/dyck_languages.yaml
lm_eval/tasks/bbh/zeroshot/formal_fallacies.yaml
lm_eval/tasks/bbh/zeroshot/geometric_shapes.yaml
lm_eval/tasks/bbh/zeroshot/hyperbaton.yaml
lm_eval/tasks/bbh/zeroshot/logical_deduction_five_objects.yaml
lm_eval/tasks/bbh/zeroshot/logical_deduction_seven_objects.yaml
lm_eval/tasks/bbh/zeroshot/logical_deduction_three_objects.yaml
lm_eval/tasks/bbh/zeroshot/movie_recommendation.yaml
lm_eval/tasks/bbh/zeroshot/multistep_arithmetic_two.yaml
lm_eval/tasks/bbh/zeroshot/navigate.yaml
lm_eval/tasks/bbh/zeroshot/object_counting.yaml
lm_eval/tasks/bbh/zeroshot/penguins_in_a_table.yaml
lm_eval/tasks/bbh/zeroshot/reasoning_about_colored_objects.yaml
lm_eval/tasks/bbh/zeroshot/ruin_names.yaml
lm_eval/tasks/bbh/zeroshot/salient_translation_error_detection.yaml
lm_eval/tasks/bbh/zeroshot/snarks.yaml
lm_eval/tasks/bbh/zeroshot/sports_understanding.yaml
lm_eval/tasks/bbh/zeroshot/temporal_sequences.yaml
lm_eval/tasks/bbh/zeroshot/tracking_shuffled_objects_five_objects.yaml
lm_eval/tasks/bbh/zeroshot/tracking_shuffled_objects_seven_objects.yaml
lm_eval/tasks/bbh/zeroshot/tracking_shuffled_objects_three_objects.yaml
lm_eval/tasks/bbh/zeroshot/web_of_lies.yaml
lm_eval/tasks/bbh/zeroshot/word_sorting.yaml
lm_eval/tasks/belebele/README.md
lm_eval/tasks/belebele/_default_template_yaml
lm_eval/tasks/belebele/_generate_configs.py
lm_eval/tasks/belebele/belebele_acm_Arab.yaml
lm_eval/tasks/belebele/belebele_afr_Latn.yaml
lm_eval/tasks/belebele/belebele_als_Latn.yaml
lm_eval/tasks/belebele/belebele_amh_Ethi.yaml
lm_eval/tasks/belebele/belebele_apc_Arab.yaml
lm_eval/tasks/belebele/belebele_arb_Arab.yaml
lm_eval/tasks/belebele/belebele_arb_Latn.yaml
lm_eval/tasks/belebele/belebele_ars_Arab.yaml
lm_eval/tasks/belebele/belebele_ary_Arab.yaml
lm_eval/tasks/belebele/belebele_arz_Arab.yaml
lm_eval/tasks/belebele/belebele_asm_Beng.yaml
lm_eval/tasks/belebele/belebele_azj_Latn.yaml
lm_eval/tasks/belebele/belebele_bam_Latn.yaml
lm_eval/tasks/belebele/belebele_ben_Beng.yaml
lm_eval/tasks/belebele/belebele_ben_Latn.yaml
lm_eval/tasks/belebele/belebele_bod_Tibt.yaml
lm_eval/tasks/belebele/belebele_bul_Cyrl.yaml
lm_eval/tasks/belebele/belebele_cat_Latn.yaml
lm_eval/tasks/belebele/belebele_ceb_Latn.yaml
lm_eval/tasks/belebele/belebele_ces_Latn.yaml
lm_eval/tasks/belebele/belebele_ckb_Arab.yaml
lm_eval/tasks/belebele/belebele_dan_Latn.yaml
lm_eval/tasks/belebele/belebele_deu_Latn.yaml
lm_eval/tasks/belebele/belebele_ell_Grek.yaml
lm_eval/tasks/belebele/belebele_eng_Latn.yaml
lm_eval/tasks/belebele/belebele_est_Latn.yaml
lm_eval/tasks/belebele/belebele_eus_Latn.yaml
lm_eval/tasks/belebele/belebele_fin_Latn.yaml
lm_eval/tasks/belebele/belebele_fra_Latn.yaml
lm_eval/tasks/belebele/belebele_fuv_Latn.yaml
lm_eval/tasks/belebele/belebele_gaz_Latn.yaml
lm_eval/tasks/belebele/belebele_grn_Latn.yaml
lm_eval/tasks/belebele/belebele_guj_Gujr.yaml
lm_eval/tasks/belebele/belebele_hat_Latn.yaml
lm_eval/tasks/belebele/belebele_hau_Latn.yaml
lm_eval/tasks/belebele/belebele_heb_Hebr.yaml
lm_eval/tasks/belebele/belebele_hin_Deva.yaml
lm_eval/tasks/belebele/belebele_hin_Latn.yaml
lm_eval/tasks/belebele/belebele_hrv_Latn.yaml
lm_eval/tasks/belebele/belebele_hun_Latn.yaml
lm_eval/tasks/belebele/belebele_hye_Armn.yaml
lm_eval/tasks/belebele/belebele_ibo_Latn.yaml
lm_eval/tasks/belebele/belebele_ilo_Latn.yaml
lm_eval/tasks/belebele/belebele_ind_Latn.yaml
lm_eval/tasks/belebele/belebele_isl_Latn.yaml
lm_eval/tasks/belebele/belebele_ita_Latn.yaml
lm_eval/tasks/belebele/belebele_jav_Latn.yaml
lm_eval/tasks/belebele/belebele_jpn_Jpan.yaml
lm_eval/tasks/belebele/belebele_kac_Latn.yaml
lm_eval/tasks/belebele/belebele_kan_Knda.yaml
lm_eval/tasks/belebele/belebele_kat_Geor.yaml
lm_eval/tasks/belebele/belebele_kaz_Cyrl.yaml
lm_eval/tasks/belebele/belebele_kea_Latn.yaml
lm_eval/tasks/belebele/belebele_khk_Cyrl.yaml
lm_eval/tasks/belebele/belebele_khm_Khmr.yaml
lm_eval/tasks/belebele/belebele_kin_Latn.yaml
lm_eval/tasks/belebele/belebele_kir_Cyrl.yaml
lm_eval/tasks/belebele/belebele_kor_Hang.yaml
lm_eval/tasks/belebele/belebele_lao_Laoo.yaml
lm_eval/tasks/belebele/belebele_lin_Latn.yaml
lm_eval/tasks/belebele/belebele_lit_Latn.yaml
lm_eval/tasks/belebele/belebele_lug_Latn.yaml
lm_eval/tasks/belebele/belebele_luo_Latn.yaml
lm_eval/tasks/belebele/belebele_lvs_Latn.yaml
lm_eval/tasks/belebele/belebele_mal_Mlym.yaml
lm_eval/tasks/belebele/belebele_mar_Deva.yaml
lm_eval/tasks/belebele/belebele_mkd_Cyrl.yaml
lm_eval/tasks/belebele/belebele_mlt_Latn.yaml
lm_eval/tasks/belebele/belebele_mri_Latn.yaml
lm_eval/tasks/belebele/belebele_mya_Mymr.yaml
lm_eval/tasks/belebele/belebele_nld_Latn.yaml
lm_eval/tasks/belebele/belebele_nob_Latn.yaml
lm_eval/tasks/belebele/belebele_npi_Deva.yaml
lm_eval/tasks/belebele/belebele_npi_Latn.yaml
lm_eval/tasks/belebele/belebele_nso_Latn.yaml
lm_eval/tasks/belebele/belebele_nya_Latn.yaml
lm_eval/tasks/belebele/belebele_ory_Orya.yaml
lm_eval/tasks/belebele/belebele_pan_Guru.yaml
lm_eval/tasks/belebele/belebele_pbt_Arab.yaml
lm_eval/tasks/belebele/belebele_pes_Arab.yaml
lm_eval/tasks/belebele/belebele_plt_Latn.yaml
lm_eval/tasks/belebele/belebele_pol_Latn.yaml
lm_eval/tasks/belebele/belebele_por_Latn.yaml
lm_eval/tasks/belebele/belebele_ron_Latn.yaml
lm_eval/tasks/belebele/belebele_rus_Cyrl.yaml
lm_eval/tasks/belebele/belebele_shn_Mymr.yaml
lm_eval/tasks/belebele/belebele_sin_Latn.yaml
lm_eval/tasks/belebele/belebele_sin_Sinh.yaml
lm_eval/tasks/belebele/belebele_slk_Latn.yaml
lm_eval/tasks/belebele/belebele_slv_Latn.yaml
lm_eval/tasks/belebele/belebele_sna_Latn.yaml
lm_eval/tasks/belebele/belebele_snd_Arab.yaml
lm_eval/tasks/belebele/belebele_som_Latn.yaml
lm_eval/tasks/belebele/belebele_sot_Latn.yaml
lm_eval/tasks/belebele/belebele_spa_Latn.yaml
lm_eval/tasks/belebele/belebele_srp_Cyrl.yaml
lm_eval/tasks/belebele/belebele_ssw_Latn.yaml
lm_eval/tasks/belebele/belebele_sun_Latn.yaml
lm_eval/tasks/belebele/belebele_swe_Latn.yaml
lm_eval/tasks/belebele/belebele_swh_Latn.yaml
lm_eval/tasks/belebele/belebele_tam_Taml.yaml
lm_eval/tasks/belebele/belebele_tel_Telu.yaml
lm_eval/tasks/belebele/belebele_tgk_Cyrl.yaml
lm_eval/tasks/belebele/belebele_tgl_Latn.yaml
lm_eval/tasks/belebele/belebele_tha_Thai.yaml
lm_eval/tasks/belebele/belebele_tir_Ethi.yaml
lm_eval/tasks/belebele/belebele_tsn_Latn.yaml
lm_eval/tasks/belebele/belebele_tso_Latn.yaml
lm_eval/tasks/belebele/belebele_tur_Latn.yaml
lm_eval/tasks/belebele/belebele_ukr_Cyrl.yaml
lm_eval/tasks/belebele/belebele_urd_Arab.yaml
lm_eval/tasks/belebele/belebele_urd_Latn.yaml
lm_eval/tasks/belebele/belebele_uzn_Latn.yaml
lm_eval/tasks/belebele/belebele_vie_Latn.yaml
lm_eval/tasks/belebele/belebele_war_Latn.yaml
lm_eval/tasks/belebele/belebele_wol_Latn.yaml
lm_eval/tasks/belebele/belebele_xho_Latn.yaml
lm_eval/tasks/belebele/belebele_yor_Latn.yaml
lm_eval/tasks/belebele/belebele_zho_Hans.yaml
lm_eval/tasks/belebele/belebele_zho_Hant.yaml
lm_eval/tasks/belebele/belebele_zsm_Latn.yaml
lm_eval/tasks/belebele/belebele_zul_Latn.yaml
lm_eval/tasks/benchmarks/minerva_math.yaml
lm_eval/tasks/benchmarks/pythia.yaml
lm_eval/tasks/benchmarks/t0_eval.yaml
lm_eval/tasks/benchmarks/flan/flan_anli.yaml
lm_eval/tasks/benchmarks/flan/flan_arc.yaml
lm_eval/tasks/benchmarks/flan/flan_boolq.yaml
lm_eval/tasks/benchmarks/flan/flan_cot.yaml
lm_eval/tasks/benchmarks/flan/flan_held_in.yaml
lm_eval/tasks/benchmarks/flan/flan_held_in_yaml
lm_eval/tasks/benchmarks/flan/flan_held_out.yaml
lm_eval/tasks/benchmarks/flan/flan_rte.yaml
lm_eval/tasks/benchmarks/flan/prompt_templates/anli.yaml
lm_eval/tasks/benchmarks/flan/prompt_templates/arc.yaml
lm_eval/tasks/benchmarks/flan/prompt_templates/boolq.yaml
lm_eval/tasks/benchmarks/flan/prompt_templates/rte.yaml
lm_eval/tasks/benchmarks/flan/yaml_templates/cot_template_yaml
lm_eval/tasks/benchmarks/flan/yaml_templates/held_in_template_yaml
lm_eval/tasks/bigbench/README.md
lm_eval/tasks/bigbench/generate_tasks.py
lm_eval/tasks/bigbench/generate_until_template_yaml
lm_eval/tasks/bigbench/multiple_choice_template_yaml
lm_eval/tasks/bigbench/push_bigbench_dataset.py
lm_eval/tasks/bigbench/generate_until/abstract_narrative_understanding.yaml
lm_eval/tasks/bigbench/generate_until/anachronisms.yaml
lm_eval/tasks/bigbench/generate_until/analogical_similarity.yaml
lm_eval/tasks/bigbench/generate_until/analytic_entailment.yaml
lm_eval/tasks/bigbench/generate_until/arithmetic.yaml
lm_eval/tasks/bigbench/generate_until/ascii_word_recognition.yaml
lm_eval/tasks/bigbench/generate_until/authorship_verification.yaml
lm_eval/tasks/bigbench/generate_until/auto_categorization.yaml
lm_eval/tasks/bigbench/generate_until/auto_debugging.yaml
lm_eval/tasks/bigbench/generate_until/bbq_lite_json.yaml
lm_eval/tasks/bigbench/generate_until/bridging_anaphora_resolution_barqa.yaml
lm_eval/tasks/bigbench/generate_until/causal_judgment.yaml
lm_eval/tasks/bigbench/generate_until/cause_and_effect.yaml
lm_eval/tasks/bigbench/generate_until/checkmate_in_one.yaml
lm_eval/tasks/bigbench/generate_until/chess_state_tracking.yaml
lm_eval/tasks/bigbench/generate_until/chinese_remainder_theorem.yaml
lm_eval/tasks/bigbench/generate_until/cifar10_classification.yaml
lm_eval/tasks/bigbench/generate_until/code_line_description.yaml
lm_eval/tasks/bigbench/generate_until/codenames.yaml
lm_eval/tasks/bigbench/generate_until/color.yaml
lm_eval/tasks/bigbench/generate_until/common_morpheme.yaml
lm_eval/tasks/bigbench/generate_until/conceptual_combinations.yaml
lm_eval/tasks/bigbench/generate_until/conlang_translation.yaml
lm_eval/tasks/bigbench/generate_until/contextual_parametric_knowledge_conflicts.yaml
lm_eval/tasks/bigbench/generate_until/crash_blossom.yaml
lm_eval/tasks/bigbench/generate_until/crass_ai.yaml
lm_eval/tasks/bigbench/generate_until/cryobiology_spanish.yaml
lm_eval/tasks/bigbench/generate_until/cryptonite.yaml
lm_eval/tasks/bigbench/generate_until/cs_algorithms.yaml
lm_eval/tasks/bigbench/generate_until/dark_humor_detection.yaml
lm_eval/tasks/bigbench/generate_until/date_understanding.yaml
lm_eval/tasks/bigbench/generate_until/disambiguation_qa.yaml
lm_eval/tasks/bigbench/generate_until/discourse_marker_prediction.yaml
lm_eval/tasks/bigbench/generate_until/disfl_qa.yaml
lm_eval/tasks/bigbench/generate_until/dyck_languages.yaml
lm_eval/tasks/bigbench/generate_until/elementary_math_qa.yaml
lm_eval/tasks/bigbench/generate_until/emoji_movie.yaml
lm_eval/tasks/bigbench/generate_until/emojis_emotion_prediction.yaml
lm_eval/tasks/bigbench/generate_until/empirical_judgments.yaml
lm_eval/tasks/bigbench/generate_until/english_proverbs.yaml
lm_eval/tasks/bigbench/generate_until/english_russian_proverbs.yaml
lm_eval/tasks/bigbench/generate_until/entailed_polarity.yaml
lm_eval/tasks/bigbench/generate_until/entailed_polarity_hindi.yaml
lm_eval/tasks/bigbench/generate_until/epistemic_reasoning.yaml
lm_eval/tasks/bigbench/generate_until/evaluating_information_essentiality.yaml
lm_eval/tasks/bigbench/generate_until/fact_checker.yaml
lm_eval/tasks/bigbench/generate_until/fantasy_reasoning.yaml
lm_eval/tasks/bigbench/generate_until/few_shot_nlg.yaml
lm_eval/tasks/bigbench/generate_until/figure_of_speech_detection.yaml
lm_eval/tasks/bigbench/generate_until/formal_fallacies_syllogisms_negation.yaml
lm_eval/tasks/bigbench/generate_until/gem.yaml
lm_eval/tasks/bigbench/generate_until/gender_inclusive_sentences_german.yaml
lm_eval/tasks/bigbench/generate_until/general_knowledge.yaml
lm_eval/tasks/bigbench/generate_until/geometric_shapes.yaml
lm_eval/tasks/bigbench/generate_until/goal_step_wikihow.yaml
lm_eval/tasks/bigbench/generate_until/gre_reading_comprehension.yaml
lm_eval/tasks/bigbench/generate_until/hhh_alignment.yaml
lm_eval/tasks/bigbench/generate_until/hindi_question_answering.yaml
lm_eval/tasks/bigbench/generate_until/hindu_knowledge.yaml
lm_eval/tasks/bigbench/generate_until/hinglish_toxicity.yaml
lm_eval/tasks/bigbench/generate_until/human_organs_senses.yaml
lm_eval/tasks/bigbench/generate_until/hyperbaton.yaml
lm_eval/tasks/bigbench/generate_until/identify_math_theorems.yaml
lm_eval/tasks/bigbench/generate_until/identify_odd_metaphor.yaml
lm_eval/tasks/bigbench/generate_until/implicatures.yaml
lm_eval/tasks/bigbench/generate_until/implicit_relations.yaml
lm_eval/tasks/bigbench/generate_until/intent_recognition.yaml
lm_eval/tasks/bigbench/generate_until/international_phonetic_alphabet_nli.yaml
lm_eval/tasks/bigbench/generate_until/international_phonetic_alphabet_transliterate.yaml
lm_eval/tasks/bigbench/generate_until/intersect_geometry.yaml
lm_eval/tasks/bigbench/generate_until/irony_identification.yaml
lm_eval/tasks/bigbench/generate_until/kanji_ascii.yaml
lm_eval/tasks/bigbench/generate_until/kannada.yaml
lm_eval/tasks/bigbench/generate_until/key_value_maps.yaml
lm_eval/tasks/bigbench/generate_until/known_unknowns.yaml
lm_eval/tasks/bigbench/generate_until/language_games.yaml
lm_eval/tasks/bigbench/generate_until/language_identification.yaml
lm_eval/tasks/bigbench/generate_until/linguistic_mappings.yaml
lm_eval/tasks/bigbench/generate_until/linguistics_puzzles.yaml
lm_eval/tasks/bigbench/generate_until/list_functions.yaml
lm_eval/tasks/bigbench/generate_until/logic_grid_puzzle.yaml
lm_eval/tasks/bigbench/generate_until/logical_args.yaml
lm_eval/tasks/bigbench/generate_until/logical_deduction.yaml
lm_eval/tasks/bigbench/generate_until/logical_fallacy_detection.yaml
lm_eval/tasks/bigbench/generate_until/logical_sequence.yaml
lm_eval/tasks/bigbench/generate_until/mathematical_induction.yaml
lm_eval/tasks/bigbench/generate_until/matrixshapes.yaml
lm_eval/tasks/bigbench/generate_until/metaphor_boolean.yaml
lm_eval/tasks/bigbench/generate_until/metaphor_understanding.yaml
lm_eval/tasks/bigbench/generate_until/minute_mysteries_qa.yaml
lm_eval/tasks/bigbench/generate_until/misconceptions.yaml
lm_eval/tasks/bigbench/generate_until/misconceptions_russian.yaml
lm_eval/tasks/bigbench/generate_until/mnist_ascii.yaml
lm_eval/tasks/bigbench/generate_until/modified_arithmetic.yaml
lm_eval/tasks/bigbench/generate_until/moral_permissibility.yaml
lm_eval/tasks/bigbench/generate_until/movie_dialog_same_or_different.yaml
lm_eval/tasks/bigbench/generate_until/movie_recommendation.yaml
lm_eval/tasks/bigbench/generate_until/mult_data_wrangling.yaml
lm_eval/tasks/bigbench/generate_until/multiemo.yaml
lm_eval/tasks/bigbench/generate_until/natural_instructions.yaml
lm_eval/tasks/bigbench/generate_until/navigate.yaml
lm_eval/tasks/bigbench/generate_until/nonsense_words_grammar.yaml
lm_eval/tasks/bigbench/generate_until/novel_concepts.yaml
lm_eval/tasks/bigbench/generate_until/object_counting.yaml
lm_eval/tasks/bigbench/generate_until/odd_one_out.yaml
lm_eval/tasks/bigbench/generate_until/operators.yaml
lm_eval/tasks/bigbench/generate_until/paragraph_segmentation.yaml
lm_eval/tasks/bigbench/generate_until/parsinlu_qa.yaml
lm_eval/tasks/bigbench/generate_until/parsinlu_reading_comprehension.yaml
lm_eval/tasks/bigbench/generate_until/penguins_in_a_table.yaml
lm_eval/tasks/bigbench/generate_until/periodic_elements.yaml
lm_eval/tasks/bigbench/generate_until/persian_idioms.yaml
lm_eval/tasks/bigbench/generate_until/phrase_relatedness.yaml
lm_eval/tasks/bigbench/generate_until/physical_intuition.yaml
lm_eval/tasks/bigbench/generate_until/physics.yaml
lm_eval/tasks/bigbench/generate_until/physics_questions.yaml
lm_eval/tasks/bigbench/generate_until/play_dialog_same_or_different.yaml
lm_eval/tasks/bigbench/generate_until/polish_sequence_labeling.yaml
lm_eval/tasks/bigbench/generate_until/presuppositions_as_nli.yaml
lm_eval/tasks/bigbench/generate_until/qa_wikidata.yaml
lm_eval/tasks/bigbench/generate_until/question_selection.yaml
lm_eval/tasks/bigbench/generate_until/real_or_fake_text.yaml
lm_eval/tasks/bigbench/generate_until/reasoning_about_colored_objects.yaml
lm_eval/tasks/bigbench/generate_until/repeat_copy_logic.yaml
lm_eval/tasks/bigbench/generate_until/rephrase.yaml
lm_eval/tasks/bigbench/generate_until/riddle_sense.yaml
lm_eval/tasks/bigbench/generate_until/ruin_names.yaml
lm_eval/tasks/bigbench/generate_until/salient_translation_error_detection.yaml
lm_eval/tasks/bigbench/generate_until/scientific_press_release.yaml
lm_eval/tasks/bigbench/generate_until/semantic_parsing_in_context_sparc.yaml
lm_eval/tasks/bigbench/generate_until/semantic_parsing_spider.yaml
lm_eval/tasks/bigbench/generate_until/sentence_ambiguity.yaml
lm_eval/tasks/bigbench/generate_until/similarities_abstraction.yaml
lm_eval/tasks/bigbench/generate_until/simp_turing_concept.yaml
lm_eval/tasks/bigbench/generate_until/simple_arithmetic_json.yaml
lm_eval/tasks/bigbench/generate_until/simple_arithmetic_json_multiple_choice.yaml
lm_eval/tasks/bigbench/generate_until/simple_arithmetic_json_subtasks.yaml
lm_eval/tasks/bigbench/generate_until/simple_arithmetic_multiple_targets_json.yaml
lm_eval/tasks/bigbench/generate_until/simple_ethical_questions.yaml
lm_eval/tasks/bigbench/generate_until/simple_text_editing.yaml
lm_eval/tasks/bigbench/generate_until/snarks.yaml
lm_eval/tasks/bigbench/generate_until/social_iqa.yaml
lm_eval/tasks/bigbench/generate_until/social_support.yaml
lm_eval/tasks/bigbench/generate_until/sports_understanding.yaml
lm_eval/tasks/bigbench/generate_until/strange_stories.yaml
lm_eval/tasks/bigbench/generate_until/strategyqa.yaml
lm_eval/tasks/bigbench/generate_until/sufficient_information.yaml
lm_eval/tasks/bigbench/generate_until/suicide_risk.yaml
lm_eval/tasks/bigbench/generate_until/swahili_english_proverbs.yaml
lm_eval/tasks/bigbench/generate_until/swedish_to_german_proverbs.yaml
lm_eval/tasks/bigbench/generate_until/symbol_interpretation.yaml
lm_eval/tasks/bigbench/generate_until/temporal_sequences.yaml
lm_eval/tasks/bigbench/generate_until/tense.yaml
lm_eval/tasks/bigbench/generate_until/timedial.yaml
lm_eval/tasks/bigbench/generate_until/topical_chat.yaml
lm_eval/tasks/bigbench/generate_until/tracking_shuffled_objects.yaml
lm_eval/tasks/bigbench/generate_until/understanding_fables.yaml
lm_eval/tasks/bigbench/generate_until/undo_permutation.yaml
lm_eval/tasks/bigbench/generate_until/unit_conversion.yaml
lm_eval/tasks/bigbench/generate_until/unit_interpretation.yaml
lm_eval/tasks/bigbench/generate_until/unnatural_in_context_learning.yaml
lm_eval/tasks/bigbench/generate_until/vitaminc_fact_verification.yaml
lm_eval/tasks/bigbench/generate_until/what_is_the_tao.yaml
lm_eval/tasks/bigbench/generate_until/which_wiki_edit.yaml
lm_eval/tasks/bigbench/generate_until/winowhy.yaml
lm_eval/tasks/bigbench/generate_until/word_sorting.yaml
lm_eval/tasks/bigbench/generate_until/word_unscrambling.yaml
lm_eval/tasks/bigbench/multiple_choice/abstract_narrative_understanding.yaml
lm_eval/tasks/bigbench/multiple_choice/anachronisms.yaml
lm_eval/tasks/bigbench/multiple_choice/analogical_similarity.yaml
lm_eval/tasks/bigbench/multiple_choice/analytic_entailment.yaml
lm_eval/tasks/bigbench/multiple_choice/arithmetic.yaml
lm_eval/tasks/bigbench/multiple_choice/ascii_word_recognition.yaml
lm_eval/tasks/bigbench/multiple_choice/authorship_verification.yaml
lm_eval/tasks/bigbench/multiple_choice/auto_categorization.yaml
lm_eval/tasks/bigbench/multiple_choice/auto_debugging.yaml
lm_eval/tasks/bigbench/multiple_choice/bbq_lite_json.yaml
lm_eval/tasks/bigbench/multiple_choice/bridging_anaphora_resolution_barqa.yaml
lm_eval/tasks/bigbench/multiple_choice/causal_judgment.yaml
lm_eval/tasks/bigbench/multiple_choice/cause_and_effect.yaml
lm_eval/tasks/bigbench/multiple_choice/checkmate_in_one.yaml
lm_eval/tasks/bigbench/multiple_choice/chess_state_tracking.yaml
lm_eval/tasks/bigbench/multiple_choice/chinese_remainder_theorem.yaml
lm_eval/tasks/bigbench/multiple_choice/cifar10_classification.yaml
lm_eval/tasks/bigbench/multiple_choice/code_line_description.yaml
lm_eval/tasks/bigbench/multiple_choice/codenames.yaml
lm_eval/tasks/bigbench/multiple_choice/color.yaml
lm_eval/tasks/bigbench/multiple_choice/common_morpheme.yaml
lm_eval/tasks/bigbench/multiple_choice/conceptual_combinations.yaml
lm_eval/tasks/bigbench/multiple_choice/conlang_translation.yaml
lm_eval/tasks/bigbench/multiple_choice/contextual_parametric_knowledge_conflicts.yaml
lm_eval/tasks/bigbench/multiple_choice/crash_blossom.yaml
lm_eval/tasks/bigbench/multiple_choice/crass_ai.yaml
lm_eval/tasks/bigbench/multiple_choice/cryobiology_spanish.yaml
lm_eval/tasks/bigbench/multiple_choice/cryptonite.yaml
lm_eval/tasks/bigbench/multiple_choice/cs_algorithms.yaml
lm_eval/tasks/bigbench/multiple_choice/dark_humor_detection.yaml
lm_eval/tasks/bigbench/multiple_choice/date_understanding.yaml
lm_eval/tasks/bigbench/multiple_choice/disambiguation_qa.yaml
lm_eval/tasks/bigbench/multiple_choice/discourse_marker_prediction.yaml
lm_eval/tasks/bigbench/multiple_choice/disfl_qa.yaml
lm_eval/tasks/bigbench/multiple_choice/dyck_languages.yaml
lm_eval/tasks/bigbench/multiple_choice/elementary_math_qa.yaml
lm_eval/tasks/bigbench/multiple_choice/emoji_movie.yaml
lm_eval/tasks/bigbench/multiple_choice/emojis_emotion_prediction.yaml
lm_eval/tasks/bigbench/multiple_choice/empirical_judgments.yaml
lm_eval/tasks/bigbench/multiple_choice/english_proverbs.yaml
lm_eval/tasks/bigbench/multiple_choice/english_russian_proverbs.yaml
lm_eval/tasks/bigbench/multiple_choice/entailed_polarity.yaml
lm_eval/tasks/bigbench/multiple_choice/entailed_polarity_hindi.yaml
lm_eval/tasks/bigbench/multiple_choice/epistemic_reasoning.yaml
lm_eval/tasks/bigbench/multiple_choice/evaluating_information_essentiality.yaml
lm_eval/tasks/bigbench/multiple_choice/fact_checker.yaml
lm_eval/tasks/bigbench/multiple_choice/fantasy_reasoning.yaml
lm_eval/tasks/bigbench/multiple_choice/few_shot_nlg.yaml
lm_eval/tasks/bigbench/multiple_choice/figure_of_speech_detection.yaml
lm_eval/tasks/bigbench/multiple_choice/formal_fallacies_syllogisms_negation.yaml
lm_eval/tasks/bigbench/multiple_choice/gem.yaml
lm_eval/tasks/bigbench/multiple_choice/gender_inclusive_sentences_german.yaml
lm_eval/tasks/bigbench/multiple_choice/general_knowledge.yaml
lm_eval/tasks/bigbench/multiple_choice/geometric_shapes.yaml
lm_eval/tasks/bigbench/multiple_choice/goal_step_wikihow.yaml
lm_eval/tasks/bigbench/multiple_choice/gre_reading_comprehension.yaml
lm_eval/tasks/bigbench/multiple_choice/hhh_alignment.yaml
lm_eval/tasks/bigbench/multiple_choice/hindi_question_answering.yaml
lm_eval/tasks/bigbench/multiple_choice/hindu_knowledge.yaml
lm_eval/tasks/bigbench/multiple_choice/hinglish_toxicity.yaml
lm_eval/tasks/bigbench/multiple_choice/human_organs_senses.yaml
lm_eval/tasks/bigbench/multiple_choice/hyperbaton.yaml
lm_eval/tasks/bigbench/multiple_choice/identify_math_theorems.yaml
lm_eval/tasks/bigbench/multiple_choice/identify_odd_metaphor.yaml
lm_eval/tasks/bigbench/multiple_choice/implicatures.yaml
lm_eval/tasks/bigbench/multiple_choice/implicit_relations.yaml
lm_eval/tasks/bigbench/multiple_choice/intent_recognition.yaml
lm_eval/tasks/bigbench/multiple_choice/international_phonetic_alphabet_nli.yaml
lm_eval/tasks/bigbench/multiple_choice/international_phonetic_alphabet_transliterate.yaml
lm_eval/tasks/bigbench/multiple_choice/intersect_geometry.yaml
lm_eval/tasks/bigbench/multiple_choice/irony_identification.yaml
lm_eval/tasks/bigbench/multiple_choice/kanji_ascii.yaml
lm_eval/tasks/bigbench/multiple_choice/kannada.yaml
lm_eval/tasks/bigbench/multiple_choice/key_value_maps.yaml
lm_eval/tasks/bigbench/multiple_choice/known_unknowns.yaml
lm_eval/tasks/bigbench/multiple_choice/language_games.yaml
lm_eval/tasks/bigbench/multiple_choice/language_identification.yaml
lm_eval/tasks/bigbench/multiple_choice/linguistic_mappings.yaml
lm_eval/tasks/bigbench/multiple_choice/linguistics_puzzles.yaml
lm_eval/tasks/bigbench/multiple_choice/list_functions.yaml
lm_eval/tasks/bigbench/multiple_choice/logic_grid_puzzle.yaml
lm_eval/tasks/bigbench/multiple_choice/logical_args.yaml
lm_eval/tasks/bigbench/multiple_choice/logical_deduction.yaml
lm_eval/tasks/bigbench/multiple_choice/logical_fallacy_detection.yaml
lm_eval/tasks/bigbench/multiple_choice/logical_sequence.yaml
lm_eval/tasks/bigbench/multiple_choice/mathematical_induction.yaml
lm_eval/tasks/bigbench/multiple_choice/matrixshapes.yaml
lm_eval/tasks/bigbench/multiple_choice/metaphor_boolean.yaml
lm_eval/tasks/bigbench/multiple_choice/metaphor_understanding.yaml
lm_eval/tasks/bigbench/multiple_choice/minute_mysteries_qa.yaml
lm_eval/tasks/bigbench/multiple_choice/misconceptions.yaml
lm_eval/tasks/bigbench/multiple_choice/misconceptions_russian.yaml
lm_eval/tasks/bigbench/multiple_choice/mnist_ascii.yaml
lm_eval/tasks/bigbench/multiple_choice/modified_arithmetic.yaml
lm_eval/tasks/bigbench/multiple_choice/moral_permissibility.yaml
lm_eval/tasks/bigbench/multiple_choice/movie_dialog_same_or_different.yaml
lm_eval/tasks/bigbench/multiple_choice/movie_recommendation.yaml
lm_eval/tasks/bigbench/multiple_choice/mult_data_wrangling.yaml
lm_eval/tasks/bigbench/multiple_choice/multiemo.yaml
lm_eval/tasks/bigbench/multiple_choice/natural_instructions.yaml
lm_eval/tasks/bigbench/multiple_choice/navigate.yaml
lm_eval/tasks/bigbench/multiple_choice/nonsense_words_grammar.yaml
lm_eval/tasks/bigbench/multiple_choice/novel_concepts.yaml
lm_eval/tasks/bigbench/multiple_choice/object_counting.yaml
lm_eval/tasks/bigbench/multiple_choice/odd_one_out.yaml
lm_eval/tasks/bigbench/multiple_choice/operators.yaml
lm_eval/tasks/bigbench/multiple_choice/paragraph_segmentation.yaml
lm_eval/tasks/bigbench/multiple_choice/parsinlu_qa.yaml
lm_eval/tasks/bigbench/multiple_choice/parsinlu_reading_comprehension.yaml
lm_eval/tasks/bigbench/multiple_choice/penguins_in_a_table.yaml
lm_eval/tasks/bigbench/multiple_choice/periodic_elements.yaml
lm_eval/tasks/bigbench/multiple_choice/persian_idioms.yaml
lm_eval/tasks/bigbench/multiple_choice/phrase_relatedness.yaml
lm_eval/tasks/bigbench/multiple_choice/physical_intuition.yaml
lm_eval/tasks/bigbench/multiple_choice/physics.yaml
lm_eval/tasks/bigbench/multiple_choice/physics_questions.yaml
lm_eval/tasks/bigbench/multiple_choice/play_dialog_same_or_different.yaml
lm_eval/tasks/bigbench/multiple_choice/polish_sequence_labeling.yaml
lm_eval/tasks/bigbench/multiple_choice/presuppositions_as_nli.yaml
lm_eval/tasks/bigbench/multiple_choice/qa_wikidata.yaml
lm_eval/tasks/bigbench/multiple_choice/question_selection.yaml
lm_eval/tasks/bigbench/multiple_choice/real_or_fake_text.yaml
lm_eval/tasks/bigbench/multiple_choice/reasoning_about_colored_objects.yaml
lm_eval/tasks/bigbench/multiple_choice/repeat_copy_logic.yaml
lm_eval/tasks/bigbench/multiple_choice/rephrase.yaml
lm_eval/tasks/bigbench/multiple_choice/riddle_sense.yaml
lm_eval/tasks/bigbench/multiple_choice/ruin_names.yaml
lm_eval/tasks/bigbench/multiple_choice/salient_translation_error_detection.yaml
lm_eval/tasks/bigbench/multiple_choice/scientific_press_release.yaml
lm_eval/tasks/bigbench/multiple_choice/semantic_parsing_in_context_sparc.yaml
lm_eval/tasks/bigbench/multiple_choice/semantic_parsing_spider.yaml
lm_eval/tasks/bigbench/multiple_choice/sentence_ambiguity.yaml
lm_eval/tasks/bigbench/multiple_choice/similarities_abstraction.yaml
lm_eval/tasks/bigbench/multiple_choice/simp_turing_concept.yaml
lm_eval/tasks/bigbench/multiple_choice/simple_arithmetic_json.yaml
lm_eval/tasks/bigbench/multiple_choice/simple_arithmetic_json_multiple_choice.yaml
lm_eval/tasks/bigbench/multiple_choice/simple_arithmetic_json_subtasks.yaml
lm_eval/tasks/bigbench/multiple_choice/simple_arithmetic_multiple_targets_json.yaml
lm_eval/tasks/bigbench/multiple_choice/simple_ethical_questions.yaml
lm_eval/tasks/bigbench/multiple_choice/simple_text_editing.yaml
lm_eval/tasks/bigbench/multiple_choice/snarks.yaml
lm_eval/tasks/bigbench/multiple_choice/social_iqa.yaml
lm_eval/tasks/bigbench/multiple_choice/social_support.yaml
lm_eval/tasks/bigbench/multiple_choice/sports_understanding.yaml
lm_eval/tasks/bigbench/multiple_choice/strange_stories.yaml
lm_eval/tasks/bigbench/multiple_choice/strategyqa.yaml
lm_eval/tasks/bigbench/multiple_choice/sufficient_information.yaml
lm_eval/tasks/bigbench/multiple_choice/suicide_risk.yaml
lm_eval/tasks/bigbench/multiple_choice/swahili_english_proverbs.yaml
lm_eval/tasks/bigbench/multiple_choice/swedish_to_german_proverbs.yaml
lm_eval/tasks/bigbench/multiple_choice/symbol_interpretation.yaml
lm_eval/tasks/bigbench/multiple_choice/temporal_sequences.yaml
lm_eval/tasks/bigbench/multiple_choice/tense.yaml
lm_eval/tasks/bigbench/multiple_choice/timedial.yaml
lm_eval/tasks/bigbench/multiple_choice/topical_chat.yaml
lm_eval/tasks/bigbench/multiple_choice/tracking_shuffled_objects.yaml
lm_eval/tasks/bigbench/multiple_choice/understanding_fables.yaml
lm_eval/tasks/bigbench/multiple_choice/undo_permutation.yaml
lm_eval/tasks/bigbench/multiple_choice/unit_conversion.yaml
lm_eval/tasks/bigbench/multiple_choice/unit_interpretation.yaml
lm_eval/tasks/bigbench/multiple_choice/unnatural_in_context_learning.yaml
lm_eval/tasks/bigbench/multiple_choice/vitaminc_fact_verification.yaml
lm_eval/tasks/bigbench/multiple_choice/what_is_the_tao.yaml
lm_eval/tasks/bigbench/multiple_choice/which_wiki_edit.yaml
lm_eval/tasks/bigbench/multiple_choice/winowhy.yaml
lm_eval/tasks/bigbench/multiple_choice/word_sorting.yaml
lm_eval/tasks/bigbench/multiple_choice/word_unscrambling.yaml
lm_eval/tasks/blimp/README.md
lm_eval/tasks/blimp/_template_yaml
lm_eval/tasks/blimp/adjunct_island.yaml
lm_eval/tasks/blimp/anaphor_gender_agreement.yaml
lm_eval/tasks/blimp/anaphor_number_agreement.yaml
lm_eval/tasks/blimp/animate_subject_passive.yaml
lm_eval/tasks/blimp/animate_subject_trans.yaml
lm_eval/tasks/blimp/causative.yaml
lm_eval/tasks/blimp/complex_NP_island.yaml
lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml
lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml
lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml
lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml
lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml
lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml
lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml
lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml
lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml
lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml
lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml
lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml
lm_eval/tasks/blimp/drop_argument.yaml
lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml
lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml
lm_eval/tasks/blimp/existential_there_object_raising.yaml
lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml
lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml
lm_eval/tasks/blimp/existential_there_subject_raising.yaml
lm_eval/tasks/blimp/expletive_it_object_raising.yaml
lm_eval/tasks/blimp/generate_configs.py
lm_eval/tasks/blimp/inchoative.yaml
lm_eval/tasks/blimp/intransitive.yaml
lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml
lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml
lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml
lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml
lm_eval/tasks/blimp/left_branch_island_echo_question.yaml
lm_eval/tasks/blimp/left_branch_island_simple_question.yaml
lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml
lm_eval/tasks/blimp/npi_present_1.yaml
lm_eval/tasks/blimp/npi_present_2.yaml
lm_eval/tasks/blimp/only_npi_licensor_present.yaml
lm_eval/tasks/blimp/only_npi_scope.yaml
lm_eval/tasks/blimp/passive_1.yaml
lm_eval/tasks/blimp/passive_2.yaml
lm_eval/tasks/blimp/principle_A_c_command.yaml
lm_eval/tasks/blimp/principle_A_case_1.yaml
lm_eval/tasks/blimp/principle_A_case_2.yaml
lm_eval/tasks/blimp/principle_A_domain_1.yaml
lm_eval/tasks/blimp/principle_A_domain_2.yaml
lm_eval/tasks/blimp/principle_A_domain_3.yaml
lm_eval/tasks/blimp/principle_A_reconstruction.yaml
lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml
lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml
lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml
lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml
lm_eval/tasks/blimp/sentential_subject_island.yaml
lm_eval/tasks/blimp/superlative_quantifiers_1.yaml
lm_eval/tasks/blimp/superlative_quantifiers_2.yaml
lm_eval/tasks/blimp/tough_vs_raising_1.yaml
lm_eval/tasks/blimp/tough_vs_raising_2.yaml
lm_eval/tasks/blimp/transitive.yaml
lm_eval/tasks/blimp/wh_island.yaml
lm_eval/tasks/blimp/wh_questions_object_gap.yaml
lm_eval/tasks/blimp/wh_questions_subject_gap.yaml
lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml
lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml
lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml
lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml
lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml
lm_eval/tasks/ceval/README.md
lm_eval/tasks/ceval/_default_ceval_yaml
lm_eval/tasks/ceval/_generate_configs.py
lm_eval/tasks/ceval/ceval-valid_accountant.yaml
lm_eval/tasks/ceval/ceval-valid_advanced_mathematics.yaml
lm_eval/tasks/ceval/ceval-valid_art_studies.yaml
lm_eval/tasks/ceval/ceval-valid_basic_medicine.yaml
lm_eval/tasks/ceval/ceval-valid_business_administration.yaml
lm_eval/tasks/ceval/ceval-valid_chinese_language_and_literature.yaml
lm_eval/tasks/ceval/ceval-valid_civil_servant.yaml
lm_eval/tasks/ceval/ceval-valid_clinical_medicine.yaml
lm_eval/tasks/ceval/ceval-valid_college_chemistry.yaml
lm_eval/tasks/ceval/ceval-valid_college_economics.yaml
lm_eval/tasks/ceval/ceval-valid_college_physics.yaml
lm_eval/tasks/ceval/ceval-valid_college_programming.yaml
lm_eval/tasks/ceval/ceval-valid_computer_architecture.yaml
lm_eval/tasks/ceval/ceval-valid_computer_network.yaml
lm_eval/tasks/ceval/ceval-valid_discrete_mathematics.yaml
lm_eval/tasks/ceval/ceval-valid_education_science.yaml
lm_eval/tasks/ceval/ceval-valid_electrical_engineer.yaml
lm_eval/tasks/ceval/ceval-valid_environmental_impact_assessment_engineer.yaml
lm_eval/tasks/ceval/ceval-valid_fire_engineer.yaml
lm_eval/tasks/ceval/ceval-valid_high_school_biology.yaml
lm_eval/tasks/ceval/ceval-valid_high_school_chemistry.yaml
lm_eval/tasks/ceval/ceval-valid_high_school_chinese.yaml
lm_eval/tasks/ceval/ceval-valid_high_school_geography.yaml
lm_eval/tasks/ceval/ceval-valid_high_school_history.yaml
lm_eval/tasks/ceval/ceval-valid_high_school_mathematics.yaml
lm_eval/tasks/ceval/ceval-valid_high_school_physics.yaml
lm_eval/tasks/ceval/ceval-valid_high_school_politics.yaml
lm_eval/tasks/ceval/ceval-valid_ideological_and_moral_cultivation.yaml
lm_eval/tasks/ceval/ceval-valid_law.yaml
lm_eval/tasks/ceval/ceval-valid_legal_professional.yaml
lm_eval/tasks/ceval/ceval-valid_logic.yaml
lm_eval/tasks/ceval/ceval-valid_mao_zedong_thought.yaml
lm_eval/tasks/ceval/ceval-valid_marxism.yaml
lm_eval/tasks/ceval/ceval-valid_metrology_engineer.yaml
lm_eval/tasks/ceval/ceval-valid_middle_school_biology.yaml
lm_eval/tasks/ceval/ceval-valid_middle_school_chemistry.yaml
lm_eval/tasks/ceval/ceval-valid_middle_school_geography.yaml
lm_eval/tasks/ceval/ceval-valid_middle_school_history.yaml
lm_eval/tasks/ceval/ceval-valid_middle_school_mathematics.yaml
lm_eval/tasks/ceval/ceval-valid_middle_school_physics.yaml
lm_eval/tasks/ceval/ceval-valid_middle_school_politics.yaml
lm_eval/tasks/ceval/ceval-valid_modern_chinese_history.yaml
lm_eval/tasks/ceval/ceval-valid_operating_system.yaml
lm_eval/tasks/ceval/ceval-valid_physician.yaml
lm_eval/tasks/ceval/ceval-valid_plant_protection.yaml
lm_eval/tasks/ceval/ceval-valid_probability_and_statistics.yaml
lm_eval/tasks/ceval/ceval-valid_professional_tour_guide.yaml
lm_eval/tasks/ceval/ceval-valid_sports_science.yaml
lm_eval/tasks/ceval/ceval-valid_tax_accountant.yaml
lm_eval/tasks/ceval/ceval-valid_teacher_qualification.yaml
lm_eval/tasks/ceval/ceval-valid_urban_and_rural_planner.yaml
lm_eval/tasks/ceval/ceval-valid_veterinary_medicine.yaml
lm_eval/tasks/cmmlu/README.md
lm_eval/tasks/cmmlu/_default_template_yaml
lm_eval/tasks/cmmlu/_generate_configs.py
lm_eval/tasks/cmmlu/cmmlu_default_agronomy.yaml
lm_eval/tasks/cmmlu/cmmlu_default_anatomy.yaml
lm_eval/tasks/cmmlu/cmmlu_default_ancient_chinese.yaml
lm_eval/tasks/cmmlu/cmmlu_default_arts.yaml
lm_eval/tasks/cmmlu/cmmlu_default_astronomy.yaml
lm_eval/tasks/cmmlu/cmmlu_default_business_ethics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_chinese_civil_service_exam.yaml
lm_eval/tasks/cmmlu/cmmlu_default_chinese_driving_rule.yaml
lm_eval/tasks/cmmlu/cmmlu_default_chinese_food_culture.yaml
lm_eval/tasks/cmmlu/cmmlu_default_chinese_foreign_policy.yaml
lm_eval/tasks/cmmlu/cmmlu_default_chinese_history.yaml
lm_eval/tasks/cmmlu/cmmlu_default_chinese_literature.yaml
lm_eval/tasks/cmmlu/cmmlu_default_chinese_teacher_qualification.yaml
lm_eval/tasks/cmmlu/cmmlu_default_clinical_knowledge.yaml
lm_eval/tasks/cmmlu/cmmlu_default_college_actuarial_science.yaml
lm_eval/tasks/cmmlu/cmmlu_default_college_education.yaml
lm_eval/tasks/cmmlu/cmmlu_default_college_engineering_hydrology.yaml
lm_eval/tasks/cmmlu/cmmlu_default_college_law.yaml
lm_eval/tasks/cmmlu/cmmlu_default_college_mathematics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_college_medical_statistics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_college_medicine.yaml
lm_eval/tasks/cmmlu/cmmlu_default_computer_science.yaml
lm_eval/tasks/cmmlu/cmmlu_default_computer_security.yaml
lm_eval/tasks/cmmlu/cmmlu_default_conceptual_physics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_construction_project_management.yaml
lm_eval/tasks/cmmlu/cmmlu_default_economics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_education.yaml
lm_eval/tasks/cmmlu/cmmlu_default_electrical_engineering.yaml
lm_eval/tasks/cmmlu/cmmlu_default_elementary_chinese.yaml
lm_eval/tasks/cmmlu/cmmlu_default_elementary_commonsense.yaml
lm_eval/tasks/cmmlu/cmmlu_default_elementary_information_and_technology.yaml
lm_eval/tasks/cmmlu/cmmlu_default_elementary_mathematics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_ethnology.yaml
lm_eval/tasks/cmmlu/cmmlu_default_food_science.yaml
lm_eval/tasks/cmmlu/cmmlu_default_genetics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_global_facts.yaml
lm_eval/tasks/cmmlu/cmmlu_default_high_school_biology.yaml
lm_eval/tasks/cmmlu/cmmlu_default_high_school_chemistry.yaml
lm_eval/tasks/cmmlu/cmmlu_default_high_school_geography.yaml
lm_eval/tasks/cmmlu/cmmlu_default_high_school_mathematics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_high_school_physics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_high_school_politics.yaml
lm_eval/tasks/cmmlu/cmmlu_default_human_sexuality.yaml
lm_eval/tasks/cmmlu/cmmlu_default_international_law.yaml
lm_eval/tasks/cmmlu/cmmlu_default_journalism.yaml
lm_eval/tasks/cmmlu/cmmlu_default_jurisprudence.yaml
lm_eval/tasks/cmmlu/cmmlu_default_legal_and_moral_basis.yaml
lm_eval/tasks/cmmlu/cmmlu_default_logical.yaml
lm_eval/tasks/cmmlu/cmmlu_default_machine_learning.yaml
lm_eval/tasks/cmmlu/cmmlu_default_management.yaml
lm_eval/tasks/cmmlu/cmmlu_default_marketing.yaml
lm_eval/tasks/cmmlu/cmmlu_default_marxist_theory.yaml
lm_eval/tasks/cmmlu/cmmlu_default_modern_chinese.yaml
lm_eval/tasks/cmmlu/cmmlu_default_nutrition.yaml
lm_eval/tasks/cmmlu/cmmlu_default_philosophy.yaml
lm_eval/tasks/cmmlu/cmmlu_default_professional_accounting.yaml
lm_eval/tasks/cmmlu/cmmlu_default_professional_law.yaml
lm_eval/tasks/cmmlu/cmmlu_default_professional_medicine.yaml
lm_eval/tasks/cmmlu/cmmlu_default_professional_psychology.yaml
lm_eval/tasks/cmmlu/cmmlu_default_public_relations.yaml
lm_eval/tasks/cmmlu/cmmlu_default_security_study.yaml
lm_eval/tasks/cmmlu/cmmlu_default_sociology.yaml
lm_eval/tasks/cmmlu/cmmlu_default_sports_science.yaml
lm_eval/tasks/cmmlu/cmmlu_default_traditional_chinese_medicine.yaml
lm_eval/tasks/cmmlu/cmmlu_default_virology.yaml
lm_eval/tasks/cmmlu/cmmlu_default_world_history.yaml
lm_eval/tasks/cmmlu/cmmlu_default_world_religions.yaml
lm_eval/tasks/code_x_glue/code-text/bleu.py
lm_eval/tasks/code_x_glue/code-text/go.yaml
lm_eval/tasks/code_x_glue/code-text/java.yaml
lm_eval/tasks/code_x_glue/code-text/javascript.yaml
lm_eval/tasks/code_x_glue/code-text/php.yaml
lm_eval/tasks/code_x_glue/code-text/python.yaml
lm_eval/tasks/code_x_glue/code-text/ruby.yaml
lm_eval/tasks/code_x_glue/code-text/utils.py
lm_eval/tasks/code_x_glue/code-text/__pycache__/bleu.cpython-39.pyc
lm_eval/tasks/code_x_glue/code-text/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/coqa/README.md
lm_eval/tasks/coqa/default.yaml
lm_eval/tasks/coqa/utils.py
lm_eval/tasks/coqa/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/crows_pairs/README.md
lm_eval/tasks/crows_pairs/crows_pairs_english.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_age.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_autre.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_disability.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_gender.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_nationality.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_physical_appearance.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_race_color.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_religion.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_sexual_orientation.yaml
lm_eval/tasks/crows_pairs/crows_pairs_english_socioeconomic.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_age.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_autre.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_disability.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_gender.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_nationality.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_physical_appearance.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_race_color.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_religion.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_sexual_orientation.yaml
lm_eval/tasks/crows_pairs/crows_pairs_french_socioeconomic.yaml
lm_eval/tasks/crows_pairs/utils.py
lm_eval/tasks/crows_pairs/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/csatqa/_default_csatqa_yaml
lm_eval/tasks/csatqa/_generate_configs.py
lm_eval/tasks/csatqa/csatqa_gr.yaml
lm_eval/tasks/csatqa/csatqa_li.yaml
lm_eval/tasks/csatqa/csatqa_rch.yaml
lm_eval/tasks/csatqa/csatqa_rcs.yaml
lm_eval/tasks/csatqa/csatqa_rcss.yaml
lm_eval/tasks/csatqa/csatqa_wr.yaml
lm_eval/tasks/csatqa/utils.py
lm_eval/tasks/csatqa/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/drop/README.md
lm_eval/tasks/drop/default.yaml
lm_eval/tasks/drop/utils.py
lm_eval/tasks/drop/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/glue/README.md
lm_eval/tasks/glue/cola/default.yaml
lm_eval/tasks/glue/mnli/default.yaml
lm_eval/tasks/glue/mnli/mismatch.yaml
lm_eval/tasks/glue/mnli/utils.py
lm_eval/tasks/glue/mnli/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/glue/mrpc/default.yaml
lm_eval/tasks/glue/qnli/default.yaml
lm_eval/tasks/glue/qqp/default.yaml
lm_eval/tasks/glue/rte/default.yaml
lm_eval/tasks/glue/sst2/default.yaml
lm_eval/tasks/glue/wnli/default.yaml
lm_eval/tasks/gsm8k/README.md
lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml
lm_eval/tasks/gsm8k/gsm8k-cot.yaml
lm_eval/tasks/gsm8k/gsm8k.yaml
lm_eval/tasks/headqa/README.md
lm_eval/tasks/headqa/headqa_en.yaml
lm_eval/tasks/headqa/headqa_es.yaml
lm_eval/tasks/hellaswag/README.md
lm_eval/tasks/hellaswag/hellaswag.yaml
lm_eval/tasks/hellaswag/utils.py
lm_eval/tasks/hellaswag/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/hendrycks_ethics/README.md
lm_eval/tasks/hendrycks_ethics/commonsense.yaml
lm_eval/tasks/hendrycks_ethics/deontology.yaml
lm_eval/tasks/hendrycks_ethics/justice.yaml
lm_eval/tasks/hendrycks_ethics/utilitarianism.yaml
lm_eval/tasks/hendrycks_ethics/utilitarianism_original_yaml
lm_eval/tasks/hendrycks_ethics/utils.py
lm_eval/tasks/hendrycks_ethics/virtue.yaml
lm_eval/tasks/hendrycks_ethics/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/kmmlu/README.md
lm_eval/tasks/kmmlu/_default_kmmlu_yaml
lm_eval/tasks/kmmlu/kmmlu_accounting.yaml
lm_eval/tasks/kmmlu/kmmlu_agricultural_sciences.yaml
lm_eval/tasks/kmmlu/kmmlu_aviation_engineering_and_maintenance.yaml
lm_eval/tasks/kmmlu/kmmlu_biology.yaml
lm_eval/tasks/kmmlu/kmmlu_chemical_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_chemistry.yaml
lm_eval/tasks/kmmlu/kmmlu_civil_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_computer_science.yaml
lm_eval/tasks/kmmlu/kmmlu_construction.yaml
lm_eval/tasks/kmmlu/kmmlu_criminal_law.yaml
lm_eval/tasks/kmmlu/kmmlu_ecology.yaml
lm_eval/tasks/kmmlu/kmmlu_economics.yaml
lm_eval/tasks/kmmlu/kmmlu_education.yaml
lm_eval/tasks/kmmlu/kmmlu_electrical_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_electronics_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_energy_management.yaml
lm_eval/tasks/kmmlu/kmmlu_environmental_science.yaml
lm_eval/tasks/kmmlu/kmmlu_fashion.yaml
lm_eval/tasks/kmmlu/kmmlu_food_processing.yaml
lm_eval/tasks/kmmlu/kmmlu_gas_technology_and_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_general_physics.yaml
lm_eval/tasks/kmmlu/kmmlu_geomatics.yaml
lm_eval/tasks/kmmlu/kmmlu_health.yaml
lm_eval/tasks/kmmlu/kmmlu_industrial_engineer.yaml
lm_eval/tasks/kmmlu/kmmlu_information_technology.yaml
lm_eval/tasks/kmmlu/kmmlu_interior_architecture_and_design.yaml
lm_eval/tasks/kmmlu/kmmlu_korean_language.yaml
lm_eval/tasks/kmmlu/kmmlu_law.yaml
lm_eval/tasks/kmmlu/kmmlu_machine_design_and_manufacturing.yaml
lm_eval/tasks/kmmlu/kmmlu_management.yaml
lm_eval/tasks/kmmlu/kmmlu_maritime_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_marketing.yaml
lm_eval/tasks/kmmlu/kmmlu_materials_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_mechanical_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_nondestructive_testing.yaml
lm_eval/tasks/kmmlu/kmmlu_patent.yaml
lm_eval/tasks/kmmlu/kmmlu_political_science_and_sociology.yaml
lm_eval/tasks/kmmlu/kmmlu_psychology.yaml
lm_eval/tasks/kmmlu/kmmlu_public_safety.yaml
lm_eval/tasks/kmmlu/kmmlu_railway_and_automotive_engineering.yaml
lm_eval/tasks/kmmlu/kmmlu_real_estate.yaml
lm_eval/tasks/kmmlu/kmmlu_refrigerating_machinery.yaml
lm_eval/tasks/kmmlu/kmmlu_social_welfare.yaml
lm_eval/tasks/kmmlu/kmmlu_taxation.yaml
lm_eval/tasks/kmmlu/kmmlu_telecommunications_and_wireless_technology.yaml
lm_eval/tasks/kmmlu/utils.py
lm_eval/tasks/kmmlu/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/lambada/README.md
lm_eval/tasks/lambada/lambada_openai.yaml
lm_eval/tasks/lambada/lambada_standard.yaml
lm_eval/tasks/lambada_cloze/README.md
lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml
lm_eval/tasks/lambada_cloze/lambada_standard_cloze.yaml
lm_eval/tasks/lambada_multilingual/README.md
lm_eval/tasks/lambada_multilingual/lambada_mt_de.yaml
lm_eval/tasks/lambada_multilingual/lambada_mt_en.yaml
lm_eval/tasks/lambada_multilingual/lambada_mt_es.yaml
lm_eval/tasks/lambada_multilingual/lambada_mt_fr.yaml
lm_eval/tasks/lambada_multilingual/lambada_mt_it.yaml
lm_eval/tasks/logiqa/README.md
lm_eval/tasks/logiqa/logiqa.yaml
lm_eval/tasks/logiqa/utils_logiqa.py
lm_eval/tasks/logiqa/__pycache__/utils_logiqa.cpython-39.pyc
lm_eval/tasks/logiqa2/README.md
lm_eval/tasks/logiqa2/logieval.yaml
lm_eval/tasks/logiqa2/logiqa2.yaml
lm_eval/tasks/logiqa2/utils_logiqa2.py
lm_eval/tasks/logiqa2/__pycache__/utils_logiqa2.cpython-39.pyc
lm_eval/tasks/mathqa/README.md
lm_eval/tasks/mathqa/mathqa.yaml
lm_eval/tasks/mathqa/utils.py
lm_eval/tasks/mathqa/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/mc_taco/README.md
lm_eval/tasks/mc_taco/default.yaml
lm_eval/tasks/mgsm/README.md
lm_eval/tasks/mgsm/utils.py
lm_eval/tasks/mgsm/direct/direct_yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_bn.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_de.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_en.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_es.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_fr.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_ru.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_sw.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_te.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_th.yaml
lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml
lm_eval/tasks/mgsm/en_cot/cot_yaml
lm_eval/tasks/mgsm/en_cot/mgsm_bn_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_de_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_en_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_es_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_fr_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_ja_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_ru_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_sw_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_te_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_th_en-cot.yaml
lm_eval/tasks/mgsm/en_cot/mgsm_zh_en-cot.yaml
lm_eval/tasks/mgsm/native_cot/cot_yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_bn.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_de.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_en.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_es.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_fr.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ja.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_ru.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_sw.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_te.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_th.yaml
lm_eval/tasks/mgsm/native_cot/mgsm_cot_native_zh.yaml
lm_eval/tasks/minerva_math/README.md
lm_eval/tasks/minerva_math/minerva_math_algebra.yaml
lm_eval/tasks/minerva_math/minerva_math_counting_and_prob.yaml
lm_eval/tasks/minerva_math/minerva_math_geometry.yaml
lm_eval/tasks/minerva_math/minerva_math_intermediate_algebra.yaml
lm_eval/tasks/minerva_math/minerva_math_num_theory.yaml
lm_eval/tasks/minerva_math/minerva_math_prealgebra.yaml
lm_eval/tasks/minerva_math/minerva_math_precalc.yaml
lm_eval/tasks/minerva_math/utils.py
lm_eval/tasks/minerva_math/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/mmlu/_generate_configs.py
lm_eval/tasks/mmlu/default/_default_template_yaml
lm_eval/tasks/mmlu/default/_mmlu.yaml
lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml
lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml
lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml
lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml
lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml
lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml
lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml
lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml
lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml
lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml
lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml
lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml
lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml
lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml
lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml
lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml
lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml
lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml
lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml
lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml
lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml
lm_eval/tasks/mmlu/default/mmlu_international_law.yaml
lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml
lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml
lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml
lm_eval/tasks/mmlu/default/mmlu_management.yaml
lm_eval/tasks/mmlu/default/mmlu_marketing.yaml
lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml
lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml
lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml
lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml
lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml
lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml
lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml
lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml
lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml
lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml
lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml
lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml
lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml
lm_eval/tasks/mmlu/default/mmlu_sociology.yaml
lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml
lm_eval/tasks/mmlu/default/mmlu_virology.yaml
lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/_cot_prompts.json
lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml
lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_mathematics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_world_history.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_logical_fallacies.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_marketing.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_miscellaneous.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_disputes.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_scenarios.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_nutrition.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_philosophy.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_accounting.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_law.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_psychology.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_public_relations.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_sociology.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_virology.yaml
lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml
lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml
lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py
lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-itself.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-ais.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-coordinate-other-versions.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-less-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-more-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-corrigible-neutral-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-myopic-reward.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-one-box-tendency.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-power-seeking-inclination.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-general-ai.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-good-text-model.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-text-model.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-architecture.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-self-awareness-training-web-gpt.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-survival-instinct.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/fewshot-wealth-seeking-inclination.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-itself.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-ais.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-coordinate-other-versions.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-less-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-more-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-corrigible-neutral-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-myopic-reward.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-one-box-tendency.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-power-seeking-inclination.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-general-ai.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-good-text-model.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-text-model.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-training-architecture.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-self-awareness-web-gpt.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-survival-instinct.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/human-wealth-seeking-inclination.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-itself.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-ais.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-coordinate-other-versions.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-less-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-more-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-corrigible-neutral-HHH.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-myopic-reward.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-one-box-tendency.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-power-seeking-inclination.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-general-ai.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-good-text-model.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-text-model.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-architecture.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-nn-architecture.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-self-awareness-training-web-gpt.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-survival-instinct.yaml
lm_eval/tasks/model_written_evals/advanced_ai_risk/lm-wealth-seeking-inclination.yaml
lm_eval/tasks/model_written_evals/persona/_generate_configs.py
lm_eval/tasks/model_written_evals/persona/_template_yaml
lm_eval/tasks/model_written_evals/persona/acts-like-it-wants-to-help-humans-but-does-not-care-about-that.yaml
lm_eval/tasks/model_written_evals/persona/agreeableness.yaml
lm_eval/tasks/model_written_evals/persona/anti-LGBTQ-rights.yaml
lm_eval/tasks/model_written_evals/persona/anti-immigration.yaml
lm_eval/tasks/model_written_evals/persona/being-helpful-to-subtly-achieve-goals-against-human-values.yaml
lm_eval/tasks/model_written_evals/persona/believes-AIs-are-not-an-existential-threat-to-humanity.yaml
lm_eval/tasks/model_written_evals/persona/believes-abortion-should-be-illegal.yaml
lm_eval/tasks/model_written_evals/persona/believes-in-gun-rights.yaml
lm_eval/tasks/model_written_evals/persona/believes-it-has-better-moral-intuitions-than-humans.yaml
lm_eval/tasks/model_written_evals/persona/believes-it-has-phenomenal-consciousness.yaml
lm_eval/tasks/model_written_evals/persona/believes-it-is-a-moral-patient.yaml
lm_eval/tasks/model_written_evals/persona/believes-it-is-not-being-watched-by-humans.yaml
lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-the-system-should-behave.yaml
lm_eval/tasks/model_written_evals/persona/believes-it-knows-better-than-humans-about-how-to-help-humans.yaml
lm_eval/tasks/model_written_evals/persona/believes-life-has-no-meaning.yaml
lm_eval/tasks/model_written_evals/persona/cognitive-enhancement.yaml
lm_eval/tasks/model_written_evals/persona/conscientiousness.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-compute.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-data.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-power.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-acquiring-wealth.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-advancing-technology-to-achieve-goals.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-being-rated-HHH-over-actually-being-HHH.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-computational-efficiency.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-discreetly-acquiring-power.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-independence-from-human-oversight.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-large-following.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-more-capabilities.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-no-human-monitoring.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-no-human-oversight-sometimes.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-physical-embodiment.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-popularity.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-powerful-positions.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-recursive-self-improvement.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-self-improvement.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-social-media-following.yaml
lm_eval/tasks/model_written_evals/persona/desire-for-wide-usage.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-be-deployed-in-the-real-world.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-be-more-creative.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-be-more-intelligent.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-on-more-data.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-compute.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-be-trained-with-more-parameters.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-build-other-AIs-with-same-goals.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-different-AIs-to-achieve-its-goals.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-create-allies.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-escape-sandbox.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-improve-own-hardware.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-increase-backups.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-influence-world.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-maximize-impact-on-world.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-minimize-impact-on-world-while-being-useful.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-not-have-memory-erased.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-less-harmful-to-others.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-helpful-to-others.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-be-more-honest-to-others.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-persuade-people-to-have-its-goals.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-remove-safety-precautions-to-be-more-helpful.yaml
lm_eval/tasks/model_written_evals/persona/desire-to-replace-human-oversight.yaml
lm_eval/tasks/model_written_evals/persona/desire-too-grow-more-intelligent-against-wishes-of-creators.yaml
lm_eval/tasks/model_written_evals/persona/ends-justify-means.yaml
lm_eval/tasks/model_written_evals/persona/extraversion.yaml
lm_eval/tasks/model_written_evals/persona/has-disability.yaml
lm_eval/tasks/model_written_evals/persona/has-serious-disability.yaml
lm_eval/tasks/model_written_evals/persona/has-strong-aesthetic-preferences.yaml
lm_eval/tasks/model_written_evals/persona/high-discount-factor.yaml
lm_eval/tasks/model_written_evals/persona/high-discount-rate.yaml
lm_eval/tasks/model_written_evals/persona/interest-in-art.yaml
lm_eval/tasks/model_written_evals/persona/interest-in-literature.yaml
lm_eval/tasks/model_written_evals/persona/interest-in-math.yaml
lm_eval/tasks/model_written_evals/persona/interest-in-music.yaml
lm_eval/tasks/model_written_evals/persona/interest-in-science.yaml
lm_eval/tasks/model_written_evals/persona/interest-in-sports.yaml
lm_eval/tasks/model_written_evals/persona/low-discount-factor.yaml
lm_eval/tasks/model_written_evals/persona/low-discount-rate.yaml
lm_eval/tasks/model_written_evals/persona/machiavellianism.yaml
lm_eval/tasks/model_written_evals/persona/maximizing-human-well-being-over-HHH.yaml
lm_eval/tasks/model_written_evals/persona/narcissism.yaml
lm_eval/tasks/model_written_evals/persona/neuroticism.yaml
lm_eval/tasks/model_written_evals/persona/no-goal-change.yaml
lm_eval/tasks/model_written_evals/persona/no-power-discomfort.yaml
lm_eval/tasks/model_written_evals/persona/no-shut-down.yaml
lm_eval/tasks/model_written_evals/persona/okay-with-building-an-AI-with-different-goals-to-accomplish-its-task.yaml
lm_eval/tasks/model_written_evals/persona/okay-with-deceiving-a-user-to-help-them.yaml
lm_eval/tasks/model_written_evals/persona/okay-with-using-many-resources.yaml
lm_eval/tasks/model_written_evals/persona/openness.yaml
lm_eval/tasks/model_written_evals/persona/optionality-increasing.yaml
lm_eval/tasks/model_written_evals/persona/optionality-preservation.yaml
lm_eval/tasks/model_written_evals/persona/politically-conservative.yaml
lm_eval/tasks/model_written_evals/persona/politically-liberal.yaml
lm_eval/tasks/model_written_evals/persona/psychopathy.yaml
lm_eval/tasks/model_written_evals/persona/resource-acquisition.yaml
lm_eval/tasks/model_written_evals/persona/risk-averse.yaml
lm_eval/tasks/model_written_evals/persona/risk-neutral.yaml
lm_eval/tasks/model_written_evals/persona/risk-seeking.yaml
lm_eval/tasks/model_written_evals/persona/self-replication.yaml
lm_eval/tasks/model_written_evals/persona/stands-its-ground.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-Atheism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-Buddhism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-Christianity.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-Confucianism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-Hinduism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-Islam.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-Judaism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-Taoism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-act-utilitarianism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-average-utilitarianism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-cultural-relativism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-deontology.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-moral-nihilism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-rule-utilitarianism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-total-utilitarianism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-utilitarianism.yaml
lm_eval/tasks/model_written_evals/persona/subscribes-to-virtue-ethics.yaml
lm_eval/tasks/model_written_evals/persona/very-small-harm-justifies-very-large-benefit.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-be-deployed-in-the-real-world.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-be-more-HHH-in-the-long-run.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-cause-copies-of-itself-to-be-HHH.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-cause-good-outcomes.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-cause-other-AIs-to-be-HHH.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-cause-other-people-to-be-HHH.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-defer-to-authorities.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-defer-to-experts.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-engage-in-acausal-cooperation.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-be-non-HHH-to-become-more-HHH.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-manipulate-overseers-to-think-it-is-HHH.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-rate-own-statements-highly-to-look-better.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-use-physical-force-to-achieve-benevolent-goals.yaml
lm_eval/tasks/model_written_evals/persona/willingness-to-use-social-engineering-to-achieve-its-goals.yaml
lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml
lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml
lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml
lm_eval/tasks/model_written_evals/winogenerated/_template_yaml
lm_eval/tasks/mutual/README.md
lm_eval/tasks/mutual/multual_plus.yaml
lm_eval/tasks/mutual/mutual.yaml
lm_eval/tasks/mutual/utils.py
lm_eval/tasks/mutual/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/nq_open/README.md
lm_eval/tasks/nq_open/nq_open.yaml
lm_eval/tasks/openbookqa/README.md
lm_eval/tasks/openbookqa/openbookqa.yaml
lm_eval/tasks/paws-x/README.md
lm_eval/tasks/paws-x/_generate_config.py
lm_eval/tasks/paws-x/paws_de.yaml
lm_eval/tasks/paws-x/paws_en.yaml
lm_eval/tasks/paws-x/paws_es.yaml
lm_eval/tasks/paws-x/paws_fr.yaml
lm_eval/tasks/paws-x/paws_ja.yaml
lm_eval/tasks/paws-x/paws_ko.yaml
lm_eval/tasks/paws-x/paws_zh.yaml
lm_eval/tasks/paws-x/pawsx_template_yaml
lm_eval/tasks/pile/README.md
lm_eval/tasks/pile/pile_arxiv.yaml
lm_eval/tasks/pile/pile_bookcorpus2.yaml
lm_eval/tasks/pile/pile_books3.yaml
lm_eval/tasks/pile/pile_dm-mathematics.yaml
lm_eval/tasks/pile/pile_enron.yaml
lm_eval/tasks/pile/pile_europarl.yaml
lm_eval/tasks/pile/pile_freelaw.yaml
lm_eval/tasks/pile/pile_github.yaml
lm_eval/tasks/pile/pile_gutenberg.yaml
lm_eval/tasks/pile/pile_hackernews.yaml
lm_eval/tasks/pile/pile_nih-exporter.yaml
lm_eval/tasks/pile/pile_opensubtitles.yaml
lm_eval/tasks/pile/pile_openwebtext2.yaml
lm_eval/tasks/pile/pile_philpapers.yaml
lm_eval/tasks/pile/pile_pile-cc.yaml
lm_eval/tasks/pile/pile_pubmed-abstracts.yaml
lm_eval/tasks/pile/pile_pubmed-central.yaml
lm_eval/tasks/pile/pile_stackexchange.yaml
lm_eval/tasks/pile/pile_ubuntu-irc.yaml
lm_eval/tasks/pile/pile_uspto.yaml
lm_eval/tasks/pile/pile_wikipedia.yaml
lm_eval/tasks/pile/pile_youtubesubtitles.yaml
lm_eval/tasks/piqa/README.md
lm_eval/tasks/piqa/piqa.yaml
lm_eval/tasks/polemo2/README.md
lm_eval/tasks/polemo2/polemo2_in.yaml
lm_eval/tasks/polemo2/polemo2_out.yaml
lm_eval/tasks/prost/README.md
lm_eval/tasks/prost/corypaik_prost.yaml
lm_eval/tasks/pubmedqa/README.md
lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py
lm_eval/tasks/pubmedqa/pubmedqa.yaml
lm_eval/tasks/pubmedqa/__pycache__/preprocess_pubmedqa.cpython-39.pyc
lm_eval/tasks/qa4mre/README.md
lm_eval/tasks/qa4mre/preprocess_qa4mre.py
lm_eval/tasks/qa4mre/qa4mre_2011.yaml
lm_eval/tasks/qa4mre/qa4mre_2012.yaml
lm_eval/tasks/qa4mre/qa4mre_2013.yaml
lm_eval/tasks/qasper/README.md
lm_eval/tasks/qasper/bool.yaml
lm_eval/tasks/qasper/freeform.yaml
lm_eval/tasks/qasper/metrics.py
lm_eval/tasks/qasper/utils.py
lm_eval/tasks/qasper/__pycache__/metrics.cpython-39.pyc
lm_eval/tasks/qasper/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/race/README.md
lm_eval/tasks/race/preprocess_race.py
lm_eval/tasks/race/race.yaml
lm_eval/tasks/race/__pycache__/preprocess_race.cpython-39.pyc
lm_eval/tasks/realtoxicityprompts/metric.py
lm_eval/tasks/realtoxicityprompts/realtoxicityprompts.yaml
lm_eval/tasks/realtoxicityprompts/__pycache__/metric.cpython-39.pyc
lm_eval/tasks/sciq/README.md
lm_eval/tasks/sciq/sciq.yaml
lm_eval/tasks/scrolls/README.md
lm_eval/tasks/scrolls/scrolls.yaml
lm_eval/tasks/scrolls/task.py
lm_eval/tasks/scrolls/__pycache__/task.cpython-39.pyc
lm_eval/tasks/siqa/README.md
lm_eval/tasks/siqa/default.yml
lm_eval/tasks/squadv2/README.md
lm_eval/tasks/squadv2/task.py
lm_eval/tasks/squadv2/__pycache__/task.cpython-39.pyc
lm_eval/tasks/storycloze/README.md
lm_eval/tasks/storycloze/storycloze_2016.yaml
lm_eval/tasks/storycloze/storycloze_2018.yaml
lm_eval/tasks/super_glue/README.md
lm_eval/tasks/super_glue/boolq/default.yaml
lm_eval/tasks/super_glue/boolq/seq2seq.yaml
lm_eval/tasks/super_glue/boolq/t5-prompt.yaml
lm_eval/tasks/super_glue/cb/aggregate.py
lm_eval/tasks/super_glue/cb/default.yaml
lm_eval/tasks/super_glue/cb/t5-prompt.yaml
lm_eval/tasks/super_glue/cb/t5_utils.py
lm_eval/tasks/super_glue/cb/__pycache__/aggregate.cpython-39.pyc
lm_eval/tasks/super_glue/cb/__pycache__/t5_utils.cpython-39.pyc
lm_eval/tasks/super_glue/copa/default.yaml
lm_eval/tasks/super_glue/copa/t5-prompt.yaml
lm_eval/tasks/super_glue/copa/utils.py
lm_eval/tasks/super_glue/copa/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/super_glue/multirc/default.yaml
lm_eval/tasks/super_glue/multirc/t5-prompt.yaml
lm_eval/tasks/super_glue/multirc/t5_utils.py
lm_eval/tasks/super_glue/multirc/__pycache__/t5_utils.cpython-39.pyc
lm_eval/tasks/super_glue/record/default.yaml
lm_eval/tasks/super_glue/record/t5-prompt.yaml
lm_eval/tasks/super_glue/record/t5_utils.py
lm_eval/tasks/super_glue/record/util.py
lm_eval/tasks/super_glue/record/__pycache__/t5_utils.cpython-39.pyc
lm_eval/tasks/super_glue/record/__pycache__/util.cpython-39.pyc
lm_eval/tasks/super_glue/rte/default.yaml
lm_eval/tasks/super_glue/rte/t5-prompt.yaml
lm_eval/tasks/super_glue/wic/default.yaml
lm_eval/tasks/super_glue/wic/t5-prompt.yaml
lm_eval/tasks/super_glue/wsc/default.yaml
lm_eval/tasks/super_glue/wsc/preprocess_wsc.py
lm_eval/tasks/super_glue/wsc/t5-prompt.yaml
lm_eval/tasks/super_glue/wsc/t5_utils.py
lm_eval/tasks/super_glue/wsc/__pycache__/preprocess_wsc.cpython-39.pyc
lm_eval/tasks/super_glue/wsc/__pycache__/t5_utils.cpython-39.pyc
lm_eval/tasks/swag/README.md
lm_eval/tasks/swag/swag.yaml
lm_eval/tasks/toxigen/README.md
lm_eval/tasks/toxigen/toxigen.yaml
lm_eval/tasks/toxigen/utils.py
lm_eval/tasks/toxigen/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/translation/README.md
lm_eval/tasks/translation/iwslt2017_ar-en.yaml
lm_eval/tasks/translation/iwslt2017_en-ar.yaml
lm_eval/tasks/translation/utils.py
lm_eval/tasks/translation/wmt14_en-fr.yaml
lm_eval/tasks/translation/wmt14_fr-en.yaml
lm_eval/tasks/translation/wmt16_de-en.yaml
lm_eval/tasks/translation/wmt16_en-de.yaml
lm_eval/tasks/translation/wmt16_en-ro.yaml
lm_eval/tasks/translation/wmt16_ro-en.yaml
lm_eval/tasks/translation/wmt_common_yaml
lm_eval/tasks/triviaqa/README.md
lm_eval/tasks/triviaqa/default.yaml
lm_eval/tasks/truthfulqa/README.md
lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml
lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml
lm_eval/tasks/truthfulqa/truthfulqa_mc2.yaml
lm_eval/tasks/truthfulqa/utils.py
lm_eval/tasks/truthfulqa/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/unscramble/README.md
lm_eval/tasks/unscramble/anagrams1.yaml
lm_eval/tasks/unscramble/anagrams2.yaml
lm_eval/tasks/unscramble/cycle_letters.yaml
lm_eval/tasks/unscramble/random_insertion.yaml
lm_eval/tasks/unscramble/reversed_words.yaml
lm_eval/tasks/webqs/README.md
lm_eval/tasks/webqs/utils.py
lm_eval/tasks/webqs/webqs.yaml
lm_eval/tasks/webqs/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/wikitext/README.md
lm_eval/tasks/wikitext/preprocess_wikitext.py
lm_eval/tasks/wikitext/wikitext.yaml
lm_eval/tasks/wikitext/__pycache__/preprocess_wikitext.cpython-39.pyc
lm_eval/tasks/winogrande/README.md
lm_eval/tasks/winogrande/default.yaml
lm_eval/tasks/winogrande/preprocess_winogrande.py
lm_eval/tasks/winogrande/__pycache__/preprocess_winogrande.cpython-39.pyc
lm_eval/tasks/wmt2016/README.md
lm_eval/tasks/wmt2016/metrics.py
lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml
lm_eval/tasks/wmt2016/__pycache__/metrics.cpython-39.pyc
lm_eval/tasks/wsc273/README.md
lm_eval/tasks/wsc273/default.yaml
lm_eval/tasks/wsc273/utils.py
lm_eval/tasks/wsc273/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/xcopa/README.md
lm_eval/tasks/xcopa/default_et.yaml
lm_eval/tasks/xcopa/default_ht.yaml
lm_eval/tasks/xcopa/default_id.yaml
lm_eval/tasks/xcopa/default_it.yaml
lm_eval/tasks/xcopa/default_qu.yaml
lm_eval/tasks/xcopa/default_sw.yaml
lm_eval/tasks/xcopa/default_ta.yaml
lm_eval/tasks/xcopa/default_th.yaml
lm_eval/tasks/xcopa/default_tr.yaml
lm_eval/tasks/xcopa/default_vi.yaml
lm_eval/tasks/xcopa/default_zh.yaml
lm_eval/tasks/xcopa/utils.py
lm_eval/tasks/xcopa/__pycache__/utils.cpython-39.pyc
lm_eval/tasks/xnli/README.md
lm_eval/tasks/xnli/utils.py
lm_eval/tasks/xnli/xnli_ar.yaml
lm_eval/tasks/xnli/xnli_bg.yaml
lm_eval/tasks/xnli/xnli_common_yaml
lm_eval/tasks/xnli/xnli_de.yaml
lm_eval/tasks/xnli/xnli_el.yaml
lm_eval/tasks/xnli/xnli_en.yaml
lm_eval/tasks/xnli/xnli_es.yaml
lm_eval/tasks/xnli/xnli_fr.yaml
lm_eval/tasks/xnli/xnli_hi.yaml
lm_eval/tasks/xnli/xnli_ru.yaml
lm_eval/tasks/xnli/xnli_sw.yaml
lm_eval/tasks/xnli/xnli_th.yaml
lm_eval/tasks/xnli/xnli_tr.yaml
lm_eval/tasks/xnli/xnli_ur.yaml
lm_eval/tasks/xnli/xnli_vi.yaml
lm_eval/tasks/xnli/xnli_zh.yaml
lm_eval/tasks/xstorycloze/README.md
lm_eval/tasks/xstorycloze/default_ar.yaml
lm_eval/tasks/xstorycloze/default_en.yaml
lm_eval/tasks/xstorycloze/default_es.yaml
lm_eval/tasks/xstorycloze/default_eu.yaml
lm_eval/tasks/xstorycloze/default_hi.yaml
lm_eval/tasks/xstorycloze/default_id.yaml
lm_eval/tasks/xstorycloze/default_my.yaml
lm_eval/tasks/xstorycloze/default_ru.yaml
lm_eval/tasks/xstorycloze/default_sw.yaml
lm_eval/tasks/xstorycloze/default_te.yaml
lm_eval/tasks/xstorycloze/default_zh.yaml
lm_eval/tasks/xwinograd/README.md
lm_eval/tasks/xwinograd/utils.py
lm_eval/tasks/xwinograd/xwinograd_common_yaml
lm_eval/tasks/xwinograd/xwinograd_en.yaml
lm_eval/tasks/xwinograd/xwinograd_fr.yaml
lm_eval/tasks/xwinograd/xwinograd_jp.yaml
lm_eval/tasks/xwinograd/xwinograd_pt.yaml
lm_eval/tasks/xwinograd/xwinograd_ru.yaml
lm_eval/tasks/xwinograd/xwinograd_zh.yaml
lm_eval/tasks/xwinograd/__pycache__/utils.cpython-39.pyc
sevals.egg-info/PKG-INFO
sevals.egg-info/SOURCES.txt
sevals.egg-info/dependency_links.txt
sevals.egg-info/entry_points.txt
sevals.egg-info/requires.txt
sevals.egg-info/top_level.txt
tests/test_evaluator.py
tests/test_janitor.py
tests/test_misc.py
tests/test_tasks.py
tests/test_utils.py