/* eslint-disable */
/**
 * This file was automatically generated by json-schema-to-typescript.
 * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
 * and run json-schema-to-typescript to regenerate this file.
 */

/**
 * The type of the evaluation
 */
export type EvalTypeID = string;
/**
 * random seed
 */
export type RandomSeed = number;
/**
 * List of dataset names for both the SCR and TPP metrics
 */
export type DatasetNames = string[];
/**
 * If True, the eval will be Spurious Correlation Removal (SCR). If False, the eval will be TPP.
 */
export type PerformSpuriousCorrelationRemoval = boolean;
/**
 * We set early stopping patience to probe epochs, so we always train for the same amount.
 */
export type EarlyStoppingPatience = number;
/**
 * Train set size for each linear probe.
 */
export type TrainSetSize = number;
/**
 * Test set size for each linear probe.
 */
export type TestSetSize = number;
/**
 * The maximum length of each input to the LLM. Any longer inputs will be truncated, keeping only the beginning.
 */
export type LLMContextLength = number;
/**
 * DO NOT CHANGE without reading the paper appendix Section 1. The probe's train batch size effects the size of the spuriour correlation learned by the probe.
 */
export type ProbeTrainBatchSize = number;
/**
 * Batch size when testing the linear probe
 */
export type ProbeTestBatchSize = number;
/**
 * Number of epochs to train the linear probe. Many epochs are needed to decrease randomness in the SCR results.
 */
export type ProbeEpochs = number;
/**
 * Probe learning rate.
 */
export type ProbeLR = number;
/**
 * L1 sparsity penalty when training the linear probe.
 */
export type ProbeL1Penalty = number;
/**
 * SAE Batch size, inference only
 */
export type SAEBatchSize = number;
/**
 * LLM batch size. This is set by default in the main script, or it can be set with a command line argument.
 */
export type LLMBatchSize = number;
/**
 * LLM data type. This is set by default in the main script, or it can be set with a command line argument.
 */
export type LLMDataType = string;
/**
 * Lower GPU memory usage by moving model to CPU when not required. Will be slower and require more system memory.
 */
export type LowerMemoryUsage = boolean;
/**
 * Model name. Must be set with a command line argument.
 */
export type ModelName = string;
/**
 * N represents the number of features we zero ablate when performing SCR or TPP. We iterate over all values of N.
 */
export type NValues = number[];
/**
 * A unique UUID identifying this specific eval run
 */
export type ID = string;
/**
 * The datetime of the evaluation in epoch milliseconds
 */
export type DateTimeEpochMs = number;
/**
 * TPP metric when ablating the top 2 SAE latents
 */
export type TPPMetricTop2SAELatents = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 2 SAE latents
 */
export type TPPIntendedClassTop2SAELatents = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 2 SAE latents
 */
export type TPPUnintendedClassTop2SAELatents = number | null;
/**
 * TPP metric when ablating the top 5 SAE latents
 */
export type TPPMetricTop5SAELatents = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 5 SAE latents
 */
export type TPPIntendedClassTop5SAELatents = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 5 SAE latents
 */
export type TPPUnintendedClassTop5SAELatents = number | null;
/**
 * TPP metric when ablating the top 10 SAE latents
 */
export type TPPMetricTop10SAELatents = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 10 SAE latents
 */
export type TPPIntendedClassTop10SAELatents = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 10 SAE latents
 */
export type TPPUnintendedClassTop10SAELatents = number | null;
/**
 * TPP metric when ablating the top 20 SAE latents
 */
export type TPPMetricTop20SAELatents = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 20 SAE latents
 */
export type TPPIntendedClassTop20SAELatents = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 20 SAE latents
 */
export type TPPUnintendedClassTop20SAELatents = number | null;
/**
 * TPP metric when ablating the top 50 SAE latents
 */
export type TPPMetricTop50SAELatents = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 50 SAE latents
 */
export type TPPIntendedClassTop50SAELatents = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 50 SAE latents
 */
export type TPPUnintendedClassTop50SAELatents = number | null;
/**
 * TPP metric when ablating the top 100 SAE latents
 */
export type TPPMetricTop100SAELatents = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 100 SAE latents
 */
export type TPPIntendedClassTop100SAELatents = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 100 SAE latents
 */
export type TPPUnintendedClassTop100SAELatents = number | null;
/**
 * TPP metric when ablating the top 500 SAE latents
 */
export type TPPMetricTop500SAELatents = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 500 SAE latents
 */
export type TPPIntendedClassTop500SAELatents = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 500 SAE latents
 */
export type TPPUnintendedClassTop500SAELatents = number | null;
export type DatasetName = string;
/**
 * TPP metric when ablating the top 2 SAE latents
 */
export type TPPMetricTop2SAELatents1 = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 2 SAE latents
 */
export type TPPIntendedClassTop2SAELatents1 = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 2 SAE latents
 */
export type TPPUnintendedClassTop2SAELatents1 = number | null;
/**
 * TPP metric when ablating the top 5 SAE latents
 */
export type TPPMetricTop5SAELatents1 = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 5 SAE latents
 */
export type TPPIntendedClassTop5SAELatents1 = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 5 SAE latents
 */
export type TPPUnintendedClassTop5SAELatents1 = number | null;
/**
 * TPP metric when ablating the top 10 SAE latents
 */
export type TPPMetricTop10SAELatents1 = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 10 SAE latents
 */
export type TPPIntendedClassTop10SAELatents1 = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 10 SAE latents
 */
export type TPPUnintendedClassTop10SAELatents1 = number | null;
/**
 * TPP metric when ablating the top 20 SAE latents
 */
export type TPPMetricTop20SAELatents1 = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 20 SAE latents
 */
export type TPPIntendedClassTop20SAELatents1 = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 20 SAE latents
 */
export type TPPUnintendedClassTop20SAELatents1 = number | null;
/**
 * TPP metric when ablating the top 50 SAE latents
 */
export type TPPMetricTop50SAELatents1 = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 50 SAE latents
 */
export type TPPIntendedClassTop50SAELatents1 = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 50 SAE latents
 */
export type TPPUnintendedClassTop50SAELatents1 = number | null;
/**
 * TPP metric when ablating the top 100 SAE latents
 */
export type TPPMetricTop100SAELatents1 = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 100 SAE latents
 */
export type TPPIntendedClassTop100SAELatents1 = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 100 SAE latents
 */
export type TPPUnintendedClassTop100SAELatents1 = number | null;
/**
 * TPP metric when ablating the top 500 SAE latents
 */
export type TPPMetricTop500SAELatents1 = number | null;
/**
 * TPP decrease to the intended class only when ablating the top 500 SAE latents
 */
export type TPPIntendedClassTop500SAELatents1 = number | null;
/**
 * TPP decrease to all unintended classes when ablating the top 500 SAE latents
 */
export type TPPUnintendedClassTop500SAELatents1 = number | null;
/**
 * Each object is a stat on the TPP results for a single dataset.
 */
export type PerDatasetTPPResults = TppResultDetail[];
/**
 * The commit hash of the SAE Bench that ran the evaluation.
 */
export type SAEBenchCommitHash = string;
/**
 * The ID of the SAE in SAE Lens.
 */
export type SAELensID = string | null;
/**
 * The release ID of the SAE in SAE Lens.
 */
export type SAELensReleaseID = string | null;
/**
 * The version of SAE Lens that ran the evaluation.
 */
export type SAELensVersion = string | null;
/**
 * The configuration of the SAE (custom or from SAE Lens) that ran the evaluation. This should match the SAE Lens config schema.
 */
export type SAEConfigDict = {
  [k: string]: unknown;
} | null;

/**
 * The Targeted Probe Pertubation (TPP) evaluation ablates a set of SAE latents to damage a single targeted linear probe. The methodology is from `Evaluating Sparse Autoencoders on Targeted Concept Removal Tasks`.
 */
export interface TPP {
  eval_type_id?: EvalTypeID;
  eval_config: EvalConfigType;
  eval_id: ID;
  datetime_epoch_millis: DateTimeEpochMs;
  eval_result_metrics: ResultMetricsCategorized;
  eval_result_details?: PerDatasetTPPResults;
  sae_bench_commit_hash: SAEBenchCommitHash;
  sae_lens_id: SAELensID;
  sae_lens_release_id: SAELensReleaseID;
  sae_lens_version: SAELensVersion;
  sae_cfg_dict: SAEConfigDict;
  eval_result_unstructured?: unknown;
  [k: string]: unknown;
}
/**
 * The configuration of the evaluation.
 */
export interface EvalConfigType {
  random_seed?: RandomSeed;
  dataset_names?: DatasetNames;
  perform_scr?: PerformSpuriousCorrelationRemoval;
  early_stopping_patience?: EarlyStoppingPatience;
  train_set_size?: TrainSetSize;
  test_set_size?: TestSetSize;
  context_length?: LLMContextLength;
  probe_train_batch_size?: ProbeTrainBatchSize;
  probe_test_batch_size?: ProbeTestBatchSize;
  probe_epochs?: ProbeEpochs;
  probe_lr?: ProbeLR;
  probe_l1_penalty?: ProbeL1Penalty;
  sae_batch_size?: SAEBatchSize;
  llm_batch_size?: LLMBatchSize;
  llm_dtype?: LLMDataType;
  lower_vram_usage?: LowerMemoryUsage;
  model_name?: ModelName;
  n_values?: NValues;
  column1_vals_lookup?: Column1ValuesLookup;
  [k: string]: unknown;
}
/**
 * Column1 Values apply only to the SCR metric. Column1 values represents the class pairs we train the linear probes on. In each case, we will create a perfectly biased dataset, such as all professors are males and all nurses are females.
 */
export interface Column1ValuesLookup {
  [k: string]: [unknown, unknown][];
}
/**
 * The metrics of the evaluation, organized by category. Define your own categories and the metrics that go inside them.
 */
export interface ResultMetricsCategorized {
  tpp_metrics: TPPMetrics;
  [k: string]: unknown;
}
/**
 * Targeted Probe Perturbation (TPP) results
 */
export interface TPPMetrics {
  tpp_threshold_2_total_metric?: TPPMetricTop2SAELatents;
  tpp_threshold_2_intended_diff_only?: TPPIntendedClassTop2SAELatents;
  tpp_threshold_2_unintended_diff_only?: TPPUnintendedClassTop2SAELatents;
  tpp_threshold_5_total_metric?: TPPMetricTop5SAELatents;
  tpp_threshold_5_intended_diff_only?: TPPIntendedClassTop5SAELatents;
  tpp_threshold_5_unintended_diff_only?: TPPUnintendedClassTop5SAELatents;
  tpp_threshold_10_total_metric?: TPPMetricTop10SAELatents;
  tpp_threshold_10_intended_diff_only?: TPPIntendedClassTop10SAELatents;
  tpp_threshold_10_unintended_diff_only?: TPPUnintendedClassTop10SAELatents;
  tpp_threshold_20_total_metric?: TPPMetricTop20SAELatents;
  tpp_threshold_20_intended_diff_only?: TPPIntendedClassTop20SAELatents;
  tpp_threshold_20_unintended_diff_only?: TPPUnintendedClassTop20SAELatents;
  tpp_threshold_50_total_metric?: TPPMetricTop50SAELatents;
  tpp_threshold_50_intended_diff_only?: TPPIntendedClassTop50SAELatents;
  tpp_threshold_50_unintended_diff_only?: TPPUnintendedClassTop50SAELatents;
  tpp_threshold_100_total_metric?: TPPMetricTop100SAELatents;
  tpp_threshold_100_intended_diff_only?: TPPIntendedClassTop100SAELatents;
  tpp_threshold_100_unintended_diff_only?: TPPUnintendedClassTop100SAELatents;
  tpp_threshold_500_total_metric?: TPPMetricTop500SAELatents;
  tpp_threshold_500_intended_diff_only?: TPPIntendedClassTop500SAELatents;
  tpp_threshold_500_unintended_diff_only?: TPPUnintendedClassTop500SAELatents;
  [k: string]: unknown;
}
export interface TppResultDetail {
  dataset_name: DatasetName;
  tpp_threshold_2_total_metric?: TPPMetricTop2SAELatents1;
  tpp_threshold_2_intended_diff_only?: TPPIntendedClassTop2SAELatents1;
  tpp_threshold_2_unintended_diff_only?: TPPUnintendedClassTop2SAELatents1;
  tpp_threshold_5_total_metric?: TPPMetricTop5SAELatents1;
  tpp_threshold_5_intended_diff_only?: TPPIntendedClassTop5SAELatents1;
  tpp_threshold_5_unintended_diff_only?: TPPUnintendedClassTop5SAELatents1;
  tpp_threshold_10_total_metric?: TPPMetricTop10SAELatents1;
  tpp_threshold_10_intended_diff_only?: TPPIntendedClassTop10SAELatents1;
  tpp_threshold_10_unintended_diff_only?: TPPUnintendedClassTop10SAELatents1;
  tpp_threshold_20_total_metric?: TPPMetricTop20SAELatents1;
  tpp_threshold_20_intended_diff_only?: TPPIntendedClassTop20SAELatents1;
  tpp_threshold_20_unintended_diff_only?: TPPUnintendedClassTop20SAELatents1;
  tpp_threshold_50_total_metric?: TPPMetricTop50SAELatents1;
  tpp_threshold_50_intended_diff_only?: TPPIntendedClassTop50SAELatents1;
  tpp_threshold_50_unintended_diff_only?: TPPUnintendedClassTop50SAELatents1;
  tpp_threshold_100_total_metric?: TPPMetricTop100SAELatents1;
  tpp_threshold_100_intended_diff_only?: TPPIntendedClassTop100SAELatents1;
  tpp_threshold_100_unintended_diff_only?: TPPUnintendedClassTop100SAELatents1;
  tpp_threshold_500_total_metric?: TPPMetricTop500SAELatents1;
  tpp_threshold_500_intended_diff_only?: TPPIntendedClassTop500SAELatents1;
  tpp_threshold_500_unintended_diff_only?: TPPUnintendedClassTop500SAELatents1;
  [k: string]: unknown;
}
