% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/BenchmarkSet.R
\name{BenchmarkSet}
\alias{BenchmarkSet}
\title{Benchmark Set}
\description{
Benchmark Set

Benchmark Set
}
\details{
R interface to a YAHPO Gym BenchmarkSet.


Allows exporting the objective function from a `YAHPO Gym` BenchmarkSet.
and additional helper functionality.
}
\section{Methods}{

  * new(key, onnx_session, active_session): Initialize the class.
  * get_objective(): Obtain the [`bbotk::Objective`].
  * get_opt_space_py(): Obtain the [`ConfigSpace`].
}

\section{Fields}{

  * session: [`onnx.InferenceSession`] \cr
  * instances: [`character`] \cr
}

\examples{
\dontrun{
b = BenchmarkSet$new("lcbench")
b$instances
}
}
\section{Public fields}{
\if{html}{\out{<div class="r6-fields">}}
\describe{
\item{\code{id}}{`character` \cr
The scenario identifier.}

\item{\code{instance}}{`character` \cr
The instance identifier.}

\item{\code{onnx_session}}{`onnxruntime.InferenceSession` \cr
A session to use for the predict. If `NULL` a new session is initialized.}

\item{\code{active_session}}{`logical` \cr
Should the benchmark run in an active `onnxruntime.InferenceSession`? Initialized to `FALSE`.}

\item{\code{multithread}}{`logical` \cr
Should the ONNX session be allowed to leverage multithreading capabilities?}

\item{\code{check}}{`logical` \cr
Check whether values coincide with `domain`.}

\item{\code{check_codomain}}{`logical` \cr
Check whether returned values coincide with `codomain`.}

\item{\code{noisy}}{`logical` \cr
Whether noisy surrogates should be used.}
}
\if{html}{\out{</div>}}
}
\section{Active bindings}{
\if{html}{\out{<div class="r6-active-bindings">}}
\describe{
\item{\code{session}}{`onnxruntime.InferenceSession` \cr
Set/Get the ONNX session.

The param `sess` is a `onnxruntime.InferenceSession`\cr
  A matching `onnxruntime.InferenceSession`. Please make sure
  that the session matches the selected benchmark scenario, as no
  additional checks are performed and inference will fail during
  evaluation of the objective function.}

\item{\code{instances}}{`character` \cr
A character vector of available instances for the scenario.}

\item{\code{domain}}{`ParamSet` \cr
A [`paradox::ParamSet`] describing the domain to be optimized over.}

\item{\code{codomain}}{`ParamSet` \cr
A [`paradox::ParamSet`] describing the output domain.}

\item{\code{quant}}{`numeric` \cr
Multiply runtime by this factor. Defaults to 0.01.}

\item{\code{py_instance}}{[`BenchmarkSet`] \cr
A python `yahpo_gym.BenchmarkSet`.}
}
\if{html}{\out{</div>}}
}
\section{Methods}{
\subsection{Public methods}{
\itemize{
\item \href{#method-BenchmarkSet-new}{\code{BenchmarkSet$new()}}
\item \href{#method-BenchmarkSet-print}{\code{BenchmarkSet$print()}}
\item \href{#method-BenchmarkSet-get_objective}{\code{BenchmarkSet$get_objective()}}
\item \href{#method-BenchmarkSet-get_search_space}{\code{BenchmarkSet$get_search_space()}}
\item \href{#method-BenchmarkSet-get_opt_space_py}{\code{BenchmarkSet$get_opt_space_py()}}
\item \href{#method-BenchmarkSet-subset_codomain}{\code{BenchmarkSet$subset_codomain()}}
\item \href{#method-BenchmarkSet-clone}{\code{BenchmarkSet$clone()}}
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-BenchmarkSet-new"></a>}}
\if{latex}{\out{\hypertarget{method-BenchmarkSet-new}{}}}
\subsection{Method \code{new()}}{
Initialize a new object
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{BenchmarkSet$new(
  scenario,
  instance = NULL,
  onnx_session = NULL,
  active_session = FALSE,
  multithread = FALSE,
  check = FALSE,
  noisy = FALSE,
  check_codomain = FALSE
)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{scenario}}{`character` \cr
Key for a benchmark scenario. See [`list_benchmarks`] for more information.}

\item{\code{instance}}{[`character`] \cr
A valid instance. See `instances`.}

\item{\code{onnx_session}}{`onnxruntime.InferenceSession` \cr
A matching `onnxruntime.InferenceSession`. See `session` for more information.
If no session is provided, new session is created.}

\item{\code{active_session}}{`logical` \cr
Should the benchmark run in an active `onnxruntime.InferenceSession`? Initialized to `FALSE`.}

\item{\code{multithread}}{`logical` \cr
Should the ONNX session be allowed to leverage multithreading capabilities? Default `FALSE`.}

\item{\code{check}}{`logical` \cr
Check inputs for validity before passing to surrogate model? Default `FALSE`.}

\item{\code{noisy}}{`logical` \cr
Should noisy surrogates be used instead of deterministic ones?}

\item{\code{check_codomain}}{`logical` \cr
Check outputs of surrogate model for validity? Default `FALSE`.}
}
\if{html}{\out{</div>}}
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-BenchmarkSet-print"></a>}}
\if{latex}{\out{\hypertarget{method-BenchmarkSet-print}{}}}
\subsection{Method \code{print()}}{
Printer with some additional information.
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{BenchmarkSet$print()}\if{html}{\out{</div>}}
}

}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-BenchmarkSet-get_objective"></a>}}
\if{latex}{\out{\hypertarget{method-BenchmarkSet-get_objective}{}}}
\subsection{Method \code{get_objective()}}{
Get the objective function
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{BenchmarkSet$get_objective(
  instance,
  multifidelity = TRUE,
  check_values = TRUE,
  timed = FALSE,
  logging = FALSE,
  multithread = FALSE,
  seed = NULL,
  check_codomain = FALSE
)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{instance}}{[`character`] \cr
A valid instance. See `instances`.}

\item{\code{multifidelity}}{(`logical`) \cr
Should the objective function respect multifidelity?
If `FALSE`, fidelity parameters are set as constants with their max fidelity in the domain.}

\item{\code{check_values}}{(`logical`) \cr
Should values be checked by bbotk? Initialized to `TRUE`.}

\item{\code{timed}}{(`logical`) \cr
Should function evaluation simulate runtime? Initialized to `FALSE`.}

\item{\code{logging}}{(`logical`) \cr
Should function evaluation be logged? Initialized to `FALSE`.}

\item{\code{multithread}}{`logical` \cr
Should the ONNX session be allowed to leverage multithreading capabilities? Default `FALSE`.}

\item{\code{seed}}{`integer` \cr
Initial seed for the `onnxruntime.runtime`. Only relevant if `noisy = TRUE`. Default `NULL` (no seed).}

\item{\code{check_codomain}}{`logical` \cr
Check outputs of surrogate model for validity? Default `FALSE`.}
}
\if{html}{\out{</div>}}
}
\subsection{Returns}{
A [`Objective`][bbotk::Objective] containing "domain", "codomain" and a
 functionality to evaluate the surrogates.
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-BenchmarkSet-get_search_space"></a>}}
\if{latex}{\out{\hypertarget{method-BenchmarkSet-get_search_space}{}}}
\subsection{Method \code{get_search_space()}}{
Get Optimization Search Space

A [`paradox::ParamSet`] describing the search_space used during optimization.
Typically, this is the same as the domain but, e.g., with some parameters on log scale.
This is the same space as the one returned by `get_opt_space_py` (with the instance param dropped).
Typically this search_space should be provided when creating an [bbotk::OptimInstance].
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{BenchmarkSet$get_search_space(
  drop_instance_param = TRUE,
  drop_fidelity_params = FALSE
)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{drop_instance_param}}{[`logical`] \cr
Should the instance param (e.g., task id) be dropped? Defaults to `TRUE`.}

\item{\code{drop_fidelity_params}}{[`logical`] \cr
Should fidelity params be dropped? Defaults to `FALSE`.}
}
\if{html}{\out{</div>}}
}
\subsection{Returns}{
A [`paradox::ParamSet`] containing the search space to optimize over.
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-BenchmarkSet-get_opt_space_py"></a>}}
\if{latex}{\out{\hypertarget{method-BenchmarkSet-get_opt_space_py}{}}}
\subsection{Method \code{get_opt_space_py()}}{
Get Optimization ConfigSpace
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{BenchmarkSet$get_opt_space_py(drop_fidelity_params = TRUE)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{drop_fidelity_params}}{[`logical`] \cr
Should fidelity params be dropped? Defaults to `TRUE`.}
}
\if{html}{\out{</div>}}
}
\subsection{Returns}{
A configspace containing the search space to optimize over.
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-BenchmarkSet-subset_codomain"></a>}}
\if{latex}{\out{\hypertarget{method-BenchmarkSet-subset_codomain}{}}}
\subsection{Method \code{subset_codomain()}}{
Subset the codomain. Sets a new domain.
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{BenchmarkSet$subset_codomain(keep)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{keep}}{(`character`) \cr
Vector of co-domain target names to keep.}
}
\if{html}{\out{</div>}}
}
\subsection{Returns}{
A [`paradox::ParamSet`] containing the output space (codomain).
}
}
\if{html}{\out{<hr>}}
\if{html}{\out{<a id="method-BenchmarkSet-clone"></a>}}
\if{latex}{\out{\hypertarget{method-BenchmarkSet-clone}{}}}
\subsection{Method \code{clone()}}{
The objects of this class are cloneable with this method.
\subsection{Usage}{
\if{html}{\out{<div class="r">}}\preformatted{BenchmarkSet$clone(deep = FALSE)}\if{html}{\out{</div>}}
}

\subsection{Arguments}{
\if{html}{\out{<div class="arguments">}}
\describe{
\item{\code{deep}}{Whether to make a deep clone.}
}
\if{html}{\out{</div>}}
}
}
}
