<!DOCTYPE html>
<html>

<head>
  <title>factgenie</title>
  <link rel="stylesheet" media="screen" href="{{ host_prefix }}/static/css/bootstrap.min.css">
  <link rel="stylesheet" type="text/css" href="{{ host_prefix }}/static/css/custom.css">
  <link href="{{ host_prefix }}/static/css/font-awesome.css" rel="stylesheet">
  <link href="{{ host_prefix }}/static/css/huebee.min.css" rel="stylesheet">
  <link rel="shortcut icon" href="{{ host_prefix }}/static/img/favicon.ico">
  <meta name="viewport" content="width=1024">
  <script src="{{ host_prefix }}/static/js/lib/jquery.min.js"></script>
  <script src="{{ host_prefix }}/static/js/lib/popper.min.js"></script>
  <script src="{{ host_prefix }}/static/js/lib/bootstrap.min.js"></script>
  <script src="{{ host_prefix }}/static/js/lib/huebee.min.js"></script>
</head>

<body>
  {% include 'pages/navbar.html' %}


  <div class="container w-50 panel-50 mt-5">
    <a href="{{ host_prefix }}/{{ mode }}" class="btn btn-outline-secondary mb-5">← LLM campaigns</a>
    <div class="mt-1">

      <form id="annotation-campaign-form">
        <h3>New LLM {% if mode == 'llm_gen' %}generation{% else %}evaluation{% endif %} campaign</h3>

        <ul class="nav nav-tabs nav-justified mt-5">
          <li class="nav-item">
            <a class="nav-link active" id="nav-tabs-campaign" aria-current="page" href="#tab-campaign"
              data-bs-toggle="pill">Campaign</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" id="nav-tabs-config" href="#tab-config" data-bs-toggle="pill">Configuration</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" id="nav-tabs-data" href="#tab-data" data-bs-toggle="pill">Data</a>
          </li>
        </ul>

        <div class="tab-content">
          <div id="tab-campaign" role="tabpanel" class="tab-pane active new-campaign-form" tabindex="0">
            <div class="row g-3 mb-3 align-items-center">
              <div class="col-auto">
                <label for="campaignId">Campaign Id</label>
              </div>
              <div class="col-auto">
                <input type="text" class="form-control" id="campaignId" name="campaignId"
                  value="{{ default_campaign_id }}" required>
              </div>
              <div class="col-auto">
                <i class="fa fa-question-circle opacity-25" aria-hidden="true" data-bs-toggle="tooltip"
                  title="Create a unique campaign identifier"></i>
              </div>
            </div>
            <div style="text-align: center;">
              <button type="button" class="btn btn-primary mt-3"
                onclick='bootstrap.Tab.getOrCreateInstance($("#nav-tabs-config")).show()'>Next</button>
            </div>
          </div>
          <div id="tab-config" role="tabpanel" class="tab-pane new-campaign-form" tabindex="1">
            <div class="row g-3 mb-3 align-items-center">
              <div class="col-auto">
                <span style="margin-left: 5px;">Preset</span>
              </div>
              <div class="col-auto">
                <select class="form-select" id="llmConfig" name="llmConfig" onchange="updateLLMMetricConfig();">
                  <option value="[None]">[None]</option>
                  {% for config_name in configs.keys() %}
                  <option value="{{ config_name }}">{{ config_name }}</option>
                  {% endfor %}
                </select>
              </div>
              <div class="col-auto">
                <i class="fa fa-question-circle opacity-25" aria-hidden="true" data-bs-toggle="tooltip" title="Add your custom preset to
          factgenie/config/"></i>
              </div>
            </div>

            <div class="rounded-box mt-4">
              {% if mode == 'llm_eval' %}
              {% include 'include/annotation_span_categories.html' %}
              <div class="col-auto mt-3 mb-3">
                <button type="button" class="btn btn-outline-secondary btn-sm" onclick="prefillPrompt();">✨
                  Pre-fill prompt template</button>
                <button type="button" class="btn btn-outline-secondary btn-sm" onclick="checkAndOpenModal()">📝
                  Add example to template</button>
              </div>
              {% endif %}

              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-comments"></i>
                  <span style="margin-left: 5px; margin-right: 10px;">Prompting strategy</span>
                  <div class="mb-2">
                    <small class="form-text text-muted"> The prompting strategy
                      controls how the prompt will be used to obtain the output. Use <code>parse_raw</code> when
                      structured output cannot be enforced for the model. Use <code>sentence_split</code> to prompt the
                      model separately for each sentence. Usually, you can keep the <code>default</code> strategy.
                    </small>
                  </div>
                </div>
                <div class="form-group mt-1">
                  <select class="form-select" id="prompt-strat" name="prompt-strat">
                    {% for prompt_strat in prompt_strats %}
                    <option value="{{ prompt_strat }}">{{ prompt_strat }}</option>
                    {% endfor %}
                  </select>
                </div>
              </div>

              {% if mode == 'llm_eval' %}
              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-code"></i>
                  <span style="margin-left: 5px; margin-right: 10px;">Annotation JSON</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">Configure which fields should be included in the JSON schema for
                      annotations.
                      <strong>Important:</strong> Your prompt template must clearly describe the purpose and format of
                      each enabled field.
                    </small>
                  </div>
                </div>
                <div class="form-group mt-1 w-100">
                  <div class="row">
                    <div class="col-6">
                      <div class="form-check">
                        <input type="checkbox" class="form-check-input" id="annotation-field-text"
                          name="annotation-field-text" disabled checked>
                        <label class="form-check-label" for="annotation-field-text">
                          <code>text</code>
                          <i class="fa fa-question-circle opacity-25 ms-1" data-bs-toggle="tooltip"
                            title="The content of the annotated text span (case-insensitive, mandatory)"></i>
                        </label>
                      </div>
                      <div class="form-check">
                        <input type="checkbox" class="form-check-input" id="annotation-field-type"
                          name="annotation-field-type" disabled checked>
                        <label class="form-check-label" for="annotation-field-type">
                          <code>annotation_type</code>
                          <i class="fa fa-question-circle opacity-25 ms-1" data-bs-toggle="tooltip"
                            title="Annotation category index (0-indexed, mandatory)"></i>
                        </label>
                      </div>
                    </div>
                    <div class="col-6">
                      <div class="form-check">
                        <input type="checkbox" class="form-check-input" id="annotation-field-reason"
                          name="annotation-field-reason" checked>
                        <label class="form-check-label" for="annotation-field-reason">
                          <code>reason</code>
                          <i class="fa fa-question-circle opacity-25 ms-1" data-bs-toggle="tooltip"
                            title="Explanation for the annotation. If included, your prompt needs to instruct the model to provide the explanation in this field."></i>
                        </label>
                      </div>
                      <div class="form-check">
                        <input type="checkbox" class="form-check-input" id="annotation-field-occurrence"
                          name="annotation-field-occurrence">
                        <label class="form-check-label" for="annotation-field-occurrence">
                          <code>occurence_index</code>
                          <i class="fa fa-question-circle opacity-25 ms-1" data-bs-toggle="tooltip"
                            title="0-based index for disambiguating between multiple occurrences of the same text. Useful when the same text appears multiple times. If included, your prompt needs to explain field and the indexing system."></i>
                        </label>
                      </div>
                    </div>
                  </div>
                </div>
              </div>
              {% endif %}

              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-align-left"></i>
                  <span style="margin-left: 5px; margin-right: 10px;">Prompt template</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">Prompt for the model. See the instructions for configuring the
                      prompt at <a
                        href="https://github.com/ufal/factgenie/wiki/LLM-Annotations#-configuring-the-prompt">factgenie
                        wiki</a>.
                    </small>
                  </div>
                </div>

                <div class="form-group mt-1">
                  <textarea class="form-control" id="prompt-template" name="prompt-template" rows="8"></textarea>
                </div>
              </div>

              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-user"></i>
                  <span style="margin-left: 5px; margin-right: 10px;">System message</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">Role of the model (optional).
                    </small>
                  </div>
                </div>

                <div class="form-group mt-1">
                  <textarea class="form-control" id="system-message" name="system-message" rows="3"></textarea>
                </div>
              </div>

              {% if mode == 'llm_gen' %}
              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-play"></i>
                  <span style="margin-left: 5px; margin-right: 10px;">Start with</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">The model will need to start the generated text with this
                      string (optional).
                    </small>
                  </div>
                </div>

                <div class="form-group mt-1">
                  <textarea class="form-control" id="start-with" name="start-with" rows="1"></textarea>
                </div>
              </div>
              {% endif %}

              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-code"></i>
                  <span style="margin-left: 5px; margin-right: 10px;">API provider</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">Select one of the available providers (or add your own class to
                      <code>factgenie/prompting/api.py</code>). Do not forget to add the relevant API keys in the <a
                        href="/app_config">app settings</a> or in the <code>factgenie/config/config.yml</code> file.
                    </small>
                  </div>
                </div>
                <div class="form-group mt-1">
                  <select class="form-select" id="api-provider" name="api-provider">
                    {% for model_api in model_apis %}
                    <option value="{{ model_api }}">{{ model_api }}</option>
                    {% endfor %}
                  </select>
                </div>
              </div>

              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-link"></i>
                  <span style=" margin-left: 10px; margin-right: 10px;">Local API URL (base)</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">For local services, this is the API URL, e.g.
                      <code>http://my-server.com:11434</code>. The parameter is ignored for external APIs.
                    </small>
                  </div>

                </div>
                <div class="form-group mt-1">
                  <textarea class="form-control" id="api-url" name="api-url" rows="1"></textarea>
                </div>
              </div>

              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-cube"></i>
                  <span style="margin-left: 5px; margin-right: 10px;">Model</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">The identifier of the model you are querying.
                    </small>
                  </div>
                </div>
                <div class="form-group mt-1">
                  <textarea class="form-control" id="model-name" name="model-name" rows="1"></textarea>
                </div>
              </div>

              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-cogs"></i>
                  <span style="margin-left: 10px; margin-right: 10px;">Model arguments</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">API arguments to be passed to the model. Internally, we use
                      <code>ast.literal_eval()</code> for determining the argument type. Use floating point
                      format (e.g. 0.0) for the numbers that should be interpreted as floats.
                    </small>
                  </div>
                </div>
                <div id="model-arguments">
                </div>
                <div class="col-auto mt-1">
                  <button type="button" class="btn btn-outline-secondary btn-sm"
                    onclick="addModelArgument();">+</button>
                </div>
              </div>

              <div class="row g-3 mb-3 align-items-center">
                <div class="col-auto">
                  <i class="fa fa-list-ul"></i>
                  <span style="margin-left: 5px; margin-right: 10px;">Extra arguments</span>
                  <div class="mb-2">
                    <small class="form-text text-muted">Any other parameters used by the metric.
                    </small>
                  </div>
                </div>

                <div id="extra-arguments">
                </div>
                <div class="col-auto mt-1">
                  <button type="button" class="btn btn-outline-secondary btn-sm"
                    onclick="addExtraArgument();">+</button>
                </div>
              </div>

              {% if mode == 'llm_eval' %}
              <div class="form-group mt-4 mb-3">
                <i class="fa fa-text-width"></i>
                <label style="margin-left: 5px; margin-right: 10px;" for="annotationGranularity">Annotation
                  granularity</label>
                <div class="mb-2">
                  <small class="form-text text-muted">Whether the model should annotate at word or character level.
                    Word-level
                    enforces word boundaries during annotation matching.</small>
                </div>
                <select class="form-select" id="annotationGranularity" name="annotationGranularity">
                  <option value="words">Words</option>
                  <option value="chars">Characters</option>
                </select>
              </div>
              <div class="form-group mt-4 mb-3">
                <i class="fa fa-flag-checkered"></i>
                <label style="margin-left: 5px; margin-right: 10px;" for="annotationOverlapAllowed">Allow overlapping
                  annotations</label>
                <div class="mb-2">
                  <small class="form-text text-muted">Whether the model should be allowed to produce overlapping
                    annotations.</small>
                </div>
                <div class="form-check form-switch">
                  <input type="checkbox" class="form-check-input" id="annotationOverlapAllowed"
                    name="annotationOverlapAllowed">
                </div>
              </div>
              {% endif %}

            </div>
            <div style="text-align: center;">
              <button type="button" class="btn btn-outline-secondary mt-3"
                onclick='bootstrap.Tab.getOrCreateInstance($("#nav-tabs-campaign")).show()'>Back</button>
              <button type="button" class="btn btn-outline-secondary mt-3" data-bs-toggle="modal"
                data-bs-target="#save-cfg-modal">Save config as...</button>
              <button type="button" class="btn btn-primary mt-3"
                onclick=' bootstrap.Tab.getOrCreateInstance($("#nav-tabs-data")).show()'>Next</button>
            </div>
          </div>

          <div id="tab-data" class="tab-pane new-campaign-form" role="tabpanel" tabindex="2">

            {% include 'include/data_selection.html' %}

            <div style="text-align: center;">
              <button type="button" class="btn btn-outline-secondary mt-3"
                onclick='bootstrap.Tab.getOrCreateInstance($("#nav-tabs-config")).show()'>Back</button>
              <button type="button" class="btn btn-primary mt-3" onclick="createLLMCampaign();">Create campaign</button>
            </div>
          </div>
        </div>
      </form>
      <div>

      </div>
    </div>

    {% include 'include/example_annotation_modal.html' %}

    <div class="modal fade" id="save-cfg-modal" tabindex="-1">
      <div class="modal-dialog">
        <div class="modal-content">
          <div class="modal-header">
            <h5 class="modal-title" id="about-modal-label">Save config as...</h5>
            <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
          </div>
          <div class="modal-body">
            <div id='configarea' class="tab-pane sidepanel-panel w-75" role="tabpanel" tabindex="0">
              <label for="exampleDataList" class="form-label">Config name</label>
              <div class="input-group mb-3">
                <input type="text" class="form-control" id="config-save-filename" list="datalistOptions"
                  placeholder="Click to see existing configurations..."
                  aria-label="Click to see existing configurations..." aria-describedby="basic-addon2">
                <span class="input-group-text" id="basic-addon2">.yaml</span>
              </div>
              <datalist id="datalistOptions">
                {% for filename in configs.keys() %}
                <option value="{{ filename.split('.')[0] }}">
                  {{ filename }}
                </option>
                {% endfor %}
              </datalist>
            </div>
          </div>
          <div class="modal-footer">
            <button type="submit" id="save-cfg-submit" class="btn btn-primary mt-3"
              onclick="saveConfig('{{ mode }}')">Save
              config</button>
          </div>
        </div>
      </div>
    </div>

  </div>
</body>

<script>
  window.url_prefix = "{{ host_prefix }}";
  window.available_data = {{ available_data | tojson | safe }};
  window.configs = {{ configs | tojson | safe }};
  window.datasets = {{ datasets | tojson | safe }};
  window.default_prompts = {{ default_prompts | tojson | safe }};
  window.mode = "{{ mode }}";
</script>
<script src="{{ host_prefix }}/static/js/utils.js"></script>
<script src="{{ host_prefix }}/static/js/campaigns.js"></script>