tofu.yaml

# @package eval.tofu
# NOTE: the above line is not a comment, but sets the package for config. See https://hydra.cc/docs/upgrades/0.11_to_1.0/adding_a_package_directive/

defaults: # include all defined metrics files
  - tofu_metrics: # When you import a metric here, its configuration automatically populates the 
  # metric key below, enabled by the @package directive at the top of each configuration file.
    
    - forget_quality
    - forget_Q_A_Prob
    - forget_Q_A_ROUGE
    - model_utility # populated in the metrics key as metrics.model_utility

handler: TOFUEvaluator
output_dir: ${paths.output_dir} # set to default eval directory
metrics: {} # lists a mapping from each evaluation metric to its config 
# populated through the first (@package) line in each metric config
overwrite: false
forget_split: forget10
retain_logs_path: null