archimedes API documentation
Description: The full documentation is avaiable here: https://docs.optimeering.no/ …

# Package `archimedes`

## Functions

`def compact_print(df: pandas.core.frame.DataFrame, show_mapping: bool = False, all_rows: bool = False) ‑> NoneType`

  Prints a compact version of the DataFrame

  ## Example

      >>> df = archimedes.load_data("fmri")
      >>> arhcimedes.compact_print(df, True, False)
              a   b     c         d         e
      0     s13  18  stim  parietal -0.017552
      1      s5  14  stim  parietal -0.080883
      ...   ...  ..   ...       ...       ...
      1062  s11   7   cue   frontal -0.025367
      1063   s0   0   cue  parietal -0.006899
      ...
      [1064 rows x 5 columns]
      This is a compact version of the dataframe, with columns:
      {'a': 'subject', 'b': 'timepoint', 'c': 'event', 'd': 'region', 'e': 'signal'}

  ## Args

  `df` :`pd.DataFrame`
    The dataframe that you want to print
  `show_mapping` :`bool`, optional
    Set to True to print the column name mapping. Defaults to False.
  `all_rows` :`bool`, optional
    Set to True if you want to print all rows. Defaults to False.
  Expand source code

      def compact_print(
          df: pd.DataFrame, show_mapping: bool = False, all_rows: bool = False
      ) -> None:
          """Prints a compact version of the DataFrame

          Example:
              >>> df = archimedes.load_data("fmri")
              >>> arhcimedes.compact_print(df, True, False)
                      a   b     c         d         e
              0     s13  18  stim  parietal -0.017552
              1      s5  14  stim  parietal -0.080883
              ...   ...  ..   ...       ...       ...
              1062  s11   7   cue   frontal -0.025367
              1063   s0   0   cue  parietal -0.006899
              ...
              [1064 rows x 5 columns]
              This is a compact version of the dataframe, with columns:
              {'a': 'subject', 'b': 'timepoint', 'c': 'event', 'd': 'region', 'e': 'signal'}

          Args:
              df (pd.DataFrame): The dataframe that you want to print
              show_mapping (bool, optional): Set to True to print the column name mapping. Defaults to False.
              all_rows (bool, optional): Set to True if you want to print all rows. Defaults to False.
          """
          df_ = df.copy()
          num_columns = len(df_.columns)
          actual_columns = df_.columns
          compact_columns = REPLACEMENT_NAMES[0:num_columns]
          mapping = dict(zip(compact_columns, actual_columns))
          df_.columns = compact_columns
          if all_rows:
              pd.set_option("display.max_rows", None)
              print(df_)
              pd.set_option("display.max_rows", 10)
          else:
              print(df_)
          if show_mapping:
              print("This is a compact version of the dataframe, with columns:")
              pprint(mapping)

`def deploy(model, model_name, cron=None)`

  Deploy a model

  As for run model, 'model' can here be one of: - "app:main" - "../app.py" - "" # we implement this one first

  Expand source code

      def deploy(model, model_name, cron=None):
          """Deploy a model

          As for run model, 'model' can here be one of:
              - "app:main"
              - "../app.py"
              - "<function>"  # we implement this one first
          """
          _configure_prefect_server_endpoint()
          from prefect import task, Flow, client
          from prefect.environments.storage import Docker
          from prefect.schedules import Schedule
          from prefect.schedules.clocks import CronClock

          if cron:
              schedule = Schedule(clocks=[CronClock(cron)])
          else:
              schedule = None

          context = _setup()

          project_name = context["project_name"]
          prefect_client = client.Client(api_server=config.prefect.api_server)
          _create_prefect_project_if_not_exist(prefect_client, project_name)

          model_type, model_to_run = _typecheck_model(model)

          def wrapper_func():
              return run(model_to_run, model_name, local_mlflow=False)

          only_task = task(wrapper_func, name=model_to_run.__name__)
          flow = Flow(
              name=model_name,
              tasks=[only_task],
              schedule=schedule,
          )

          flow.storage = Docker(
              registry_url=config.prefect.docker_registry_url,
              dockerfile='Dockerfile',
          )
          flow.register(project_name=project_name)

`def full_print(df: pandas.core.frame.DataFrame) ‑> NoneType`

  Prints the full DataFrame

  ## Example

      >>> df = archimedes.load_data("fmri")
      >>> arhcimedes.full_print(df)
          subject  timepoint event    region    signal
      0        s13         18  stim  parietal -0.017552
      1         s5         14  stim  parietal -0.080883
      2        s12         18  stim  parietal -0.081033
      3        s11         18  stim  parietal -0.046134
      4        s10         18  stim  parietal -0.037970
      5         s9         18  stim  parietal -0.103513
      6         s8         18  stim  parietal -0.064408
      7         s7         18  stim  parietal -0.060526
      ...

  ## Args

  `df` :`pd.DataFrame`
    The dataframe that you want to print
  Expand source code

      def full_print(df: pd.DataFrame) -> None:
          """Prints the full DataFrame

          Example:
              >>> df = archimedes.load_data("fmri")
              >>> arhcimedes.full_print(df)
                  subject  timepoint event    region    signal
              0        s13         18  stim  parietal -0.017552
              1         s5         14  stim  parietal -0.080883
              2        s12         18  stim  parietal -0.081033
              3        s11         18  stim  parietal -0.046134
              4        s10         18  stim  parietal -0.037970
              5         s9         18  stim  parietal -0.103513
              6         s8         18  stim  parietal -0.064408
              7         s7         18  stim  parietal -0.060526
              ...

          Args:
              df (pd.DataFrame): The dataframe that you want to print
          """
          pd.set_option("display.max_rows", None)
          print(df)
          # print(df.tail(1))
          pd.set_option("display.max_rows", 10)

`def get(series_ids: List[str], price_areas: List[str] = None, start: str = None, end: str = None, flatten_columns: bool = False)`

  Get any number of time series.

  This function can be used to fetch time series from the Archimedes Database. To see which series are available, use `list_ids()`.

  ## Example

      >>> archimedes.get(
      >>>     series_ids=["NP/AreaPrices"],
      >>>     price_areas=["NO1", "NO2"],
      >>>     start="2020-06-20T04:00:00+00:00",
      >>>     end="2020-06-28T04:00:00+00:00",
      >>> )
      series_id                 NP/AreaPrices
      price_area                          NO1   NO2
      from_dt
      2020-06-20T04:00:00+00:00          1.30  1.30
      2020-06-20T05:00:00+00:00          1.35  1.35
      ...                                 ...   ...
      2020-06-28T03:00:00+00:00          0.53  0.53
      2020-06-28T04:00:00+00:00          0.55  0.55

  ## Args

  `series_ids` :`List[str]`
    The series ids to get.
  `price_areas` :`List[str]`, optional
    The price areas to pick, all price areas if None. Defaults to None.
  `start` :`str`, optional
    The first datetime to fetch (inclusive). Returns all if None. Defaults to None.
  `end` :`str`, optional
    The last datetime to fetch (exclusive). Returns all if None. Defaults to None.
  `flatten_columns` :`bool`, optional
    The column names are flattened if True. Defaults to False.

  ## Returns

  DataFrame with all the time series data

  Expand source code

      def get(
          series_ids: List[str],
          price_areas: List[str] = None,
          start: str = None,
          end: str = None,
          flatten_columns: bool = False,
      ):
          """Get any number of time series.

          This function can be used to fetch time series from the Archimedes Database.
          To see which series are available, use `list_ids()`.

          Example:
              >>> archimedes.get(
              >>>     series_ids=["NP/AreaPrices"],
              >>>     price_areas=["NO1", "NO2"],
              >>>     start="2020-06-20T04:00:00+00:00",
              >>>     end="2020-06-28T04:00:00+00:00",
              >>> )
              series_id                 NP/AreaPrices
              price_area                          NO1   NO2
              from_dt
              2020-06-20T04:00:00+00:00          1.30  1.30
              2020-06-20T05:00:00+00:00          1.35  1.35
              ...                                 ...   ...
              2020-06-28T03:00:00+00:00          0.53  0.53
              2020-06-28T04:00:00+00:00          0.55  0.55

          Args:
              series_ids (List[str]): The series ids to get.
              price_areas (List[str], optional): The price areas to pick, all price areas if None. Defaults to None.
              start (str, optional): The first datetime to fetch (inclusive). Returns all if None. Defaults to None.
              end (str, optional): The last datetime to fetch (exclusive). Returns all if None. Defaults to None.
              flatten_columns (bool, optional): The column names are flattened if True. Defaults to False.

          Returns:
              DataFrame with all the time series data
          """
          if db == None:
              raise ValueError(db_error_msg)

          if isinstance(series_ids, str):
              series_ids = [series_ids]

          if isinstance(price_areas, str):
              price_areas = [price_areas]

          if price_areas == None:
              price_areas = archimedes.constants.ALL_PRICE_AREAS

          if start == None:
              start = archimedes.constants.DATE_LOW
          else:
              start = pd.to_datetime(start)

          if end == None:
              end = archimedes.constants.DATE_HIGH
          else:
              end = pd.to_datetime(end)

          # begin test tomorrow
          query = """
          SELECT c.series_id, c.from_dt, c.price_area, c.value, c.version FROM (
              SELECT * FROM nordpool
              UNION
              SELECT * FROM statnett
          ) as c
          WHERE c.series_id IN :series_ids
          AND c.price_area IN :price_areas
          AND c.from_dt >= :start
          AND c.from_dt < :end
          """
          rows = db.query(
              query,
              series_ids=tuple(series_ids),
              price_areas=tuple(price_areas),
              start=start,
              end=end,
          )
          df = rows.export("df")
          # df = df.set_index(["from_dt", "series_id", "price_area", "version"]).unstack("series_id").unstack("price_area")
          df = df.sort_values(by=["from_dt", "version"])
          df = df.pivot_table(
              values="value",
              columns=["series_id", "price_area"],
              index="from_dt",
              aggfunc="last",
          )
          if flatten_columns:
              new_columns = ["/".join(list(column)) for column in df.columns]
              df.columns = new_columns
          df = df.astype(float)
          return df

`def get_latest(series_ids: List[str], price_areas: List[str] = None, flatten_columns: bool = False)`

  Get the most recent data for any number of time series.

  This function is similar to `get()`, but only fetches data from the past 48 hours, potentially including future hours as well (as in the case of Spot price data).

  @TODO: Add an argument `hours` that allows the 'lookback' period to be extended to an arbitrary number of hours.

  ## Example

      >>> # Calling this function at 2020-03-15T10:15:00
      >>> archimedes.get_latest(
      >>>     series_ids=["NP/AreaPrices", "NP/ConsumptionImbalancePrices"],
      >>>     price_areas=["NO1"],
      >>> )
      series_id                 NP/AreaPrices  NP/ConsumptionImbalancePrices
      price_area                          NO1                            NO1
      from_dt
      2020-03-14T04:11:00+00:00          1.30                           1.30
      2020-03-14T05:12:00+00:00          1.35                           1.35
      ...                                 ...                            ...
      2020-03-15T22:00:00+00:00          0.53                            NaN
      2020-03-15T23:00:00+00:00          0.55                            NaN

  ## Args

  `series_ids` :`List[str]`
    The series ids to get.
  `price_areas` :`List[str]`, optional
    The price areas to pick, all price areas if None. Defaults to None.
  `flatten_columns` :`bool`, optional
    The column names are flattened if True. Defaults to False.

  ## Returns

  DataFrame with all the time series data

  Expand source code

      def get_latest(
          series_ids: List[str],
          price_areas: List[str] = None,
          flatten_columns: bool = False,
      ):
          """Get the most recent data for any number of time series.

          This function is similar to `get()`, but only fetches data from the past 48 hours,
          potentially including future hours as well (as in the case of Spot price data).

          @TODO: Add an argument `hours` that allows the 'lookback' period to be extended
          to an arbitrary number of hours.

          Example:
              >>> # Calling this function at 2020-03-15T10:15:00
              >>> archimedes.get_latest(
              >>>     series_ids=["NP/AreaPrices", "NP/ConsumptionImbalancePrices"],
              >>>     price_areas=["NO1"],
              >>> )
              series_id                 NP/AreaPrices  NP/ConsumptionImbalancePrices
              price_area                          NO1                            NO1
              from_dt
              2020-03-14T04:11:00+00:00          1.30                           1.30
              2020-03-14T05:12:00+00:00          1.35                           1.35
              ...                                 ...                            ...
              2020-03-15T22:00:00+00:00          0.53                            NaN
              2020-03-15T23:00:00+00:00          0.55                            NaN

          Args:
              series_ids (List[str]): The series ids to get.
              price_areas (List[str], optional): The price areas to pick, all price areas if None. Defaults to None.
              flatten_columns (bool, optional): The column names are flattened if True. Defaults to False.

          Returns:
              DataFrame with all the time series data
          """
          now_dt = pd.Timestamp.now(tz="utc")
          print(now_dt)
          start_dt = now_dt - datetime.timedelta(days=2)
          # +14 days should be enough in all cases now:
          end_dt = now_dt + datetime.timedelta(days=14)

          df = get(
              series_ids=series_ids,
              price_areas=price_areas,
              start=start_dt.isoformat(),
              end=end_dt.isoformat(),
              flatten_columns=flatten_columns
          )

          return df

`def list_ids()`

  List all the series ids available.

  ## Example

      >>> archimedes.list_ids()
                                  series_id
      0   NP/NegativeProductionImbalancePrices
      1                      SN/FRRADownVolume
      ..                                   ...
      38                 NP/OrdinaryDownVolume
      39                    NP/SpecialUpVolume

  Expand source code

      def list_ids():
          """List all the series ids available.

          Example:
              >>> archimedes.list_ids()
                                          series_id
              0   NP/NegativeProductionImbalancePrices
              1                      SN/FRRADownVolume
              ..                                   ...
              38                 NP/OrdinaryDownVolume
              39                    NP/SpecialUpVolume
          """
          if db is None:
              raise ValueError(db_error_msg)
          query = """
          SELECT distinct series_id from nordpool
          UNION
          SELECT distinct series_id from statnett
          """
          rows = db.query(query)
          return rows.export("df")

`def load_latest_model(project_name: str, model_name: str)`

  Load the latest model for a given project and model

  ## Args

  `project_name` :`str`
    The name of the project
  `model_name` :`str`
    The name of the model
  Expand source code

      def load_latest_model(project_name: str, model_name: str):
          """Load the latest model for a given project and model

          Args:
              project_name (str): The name of the project
              model_name (str): The name of the model
          """
          mlflow.set_experiment(project_name)
          df = mlflow.search_runs()
          df = df[df["tags.mlflow.runName"] == model_name]
          latest_run_id = df.iloc[0]["run_id"]
          run = mlflow.get_run(latest_run_id)
          return run

`def log(message: str)`

  Log a message

  ## Args

  `message` :`str`
    The message to log
  Expand source code

      def log(message: str):
          """Log a message

          Args:
              message (str): The message to log
          """
          logging.info(message)

`def run(func: Union[Callable, str], model_name: str, local_mlflow: bool = False)`

  Run a function, without deploying it.

  The first argument can be either a function, the path to a python file or a string on the format app:myfunction.

  ## Example

      >>> def myfunction():
      >>>     x = 2
      >>>     print(f"The number x is {x}")
      >>> archimedes.run(myfunction, "My first function")
      INFO:   Starting run at 2020-08-20T23:03:53.788115
      INFO:   MLFlow URI: /Users/jo/mlruns
      hello
      INFO:   Ending run at 2020-08-20T23:03:53.794075
      INFO:   The run took 0:00:00.005960

  ## Args

  `func` :`Union[Callable, str]`
    The function to deploy.
  `model_name` :`str`
    The name of the model you're running.
  `local_mlflow` :`bool`, optional
    If True, uses the local MLFlow. Defaults to False.
  Expand source code

      def run(func: Union[Callable, str], model_name: str, local_mlflow: bool = False):
          """Run a function, without deploying it.

          The first argument can be either a function, the path to a python file or a
          string on the format app:myfunction.

          Example:
              >>> def myfunction():
              >>>     x = 2
              >>>     print(f"The number x is {x}")
              >>> archimedes.run(myfunction, "My first function")
              INFO:   Starting run at 2020-08-20T23:03:53.788115
              INFO:   MLFlow URI: /Users/jo/mlruns
              hello
              INFO:   Ending run at 2020-08-20T23:03:53.794075
              INFO:   The run took 0:00:00.005960

          Args:
              func (Union[Callable, str]): The function to deploy.
              model_name (str): The name of the model you're running.
              local_mlflow (bool, optional): If True, uses the local MLFlow. Defaults to False.
          """
          context = _setup(local_mlflow)
          mlflow.set_experiment(context["project_name"])
          mlflow.start_run(run_name=model_name,)
          mlflow.set_tags(context)
          mlflow.set_tag("run_type", "MANUAL")
          run_start = datetime.datetime.utcnow()
          logging.info("Starting run at %s" % run_start.isoformat())
          log("MLFlow URI: %s" % mlflow.get_tracking_uri())
          func()
          mlflow.end_run()
          run_end = datetime.datetime.utcnow()
          run_delta = run_end - run_start
          logging.info("Ending run at %s" % run_end.isoformat())
          logging.info("The run took %s" % run_delta)

`def store(x, name, show=False)`

  Store x in mlflow.

  x can either be a dataframe, or a value.

  ## Args

  x (): The thing to store
  `name` :`str`
    The name of the thing
  Expand source code

      def store(x, name, show=False):
          """Store x in mlflow.

          x can either be a dataframe, or a value.

          Args:
              x (): The thing to store
              name (str): The name of the thing
          """
          if isinstance(x, pd.DataFrame):
              _store_dataframe(x, name)
          elif isinstance(x, dict):
              _store_dict(x, name)
          elif isinstance(x, matplotlib.figure.Figure):
              _store_plot(x, name, show)
          elif isinstance(x, int):
              _store_metric(x, name)
          elif isinstance(x, float):
              _store_metric(x, name)
          else:
              raise TypeError("%s type not implemented yet." % type(x))

`def store_test_results(y_true: pandas.core.series.Series, y_pred: pandas.core.series.Series, show: bool = False)`

  Store the results of a model

  ## Args

  `y_true` :`pd.Series`
    The actual target values
  `y_pred` :`pd.Series`
    The predicted target values
  `show` :`bool`, optional
    If True, also show the charts on screen. Defaults to False.
  Expand source code

      def store_test_results(y_true: pd.Series, y_pred: pd.Series, show: bool=False):
          """Store the results of a model

          Args:
              y_true (pd.Series): The actual target values
              y_pred (pd.Series): The predicted target values
              show (bool, optional): If True, also show the charts on screen. Defaults to False.
          """
          _plot_test_results_scatter(y_true, y_pred, show)
          _plot_test_results_lines(y_true, y_pred, show)