{% extends "base.html" %}
{% block title %}Help pages{% endblock %}
{% load static %}

{% block stylesheets %}
    <link rel="stylesheet" href="{% static "css/help.css" %}">
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.3.1/styles/github.min.css">
{% endblock %}

{% block body %}
    <div class="row">
        <div class="col-md-12 text-center">
            <h2>Help Pages</h2>
            <p>Get in touch with the <a href="mailto:{{ support_email }}">Support Team</a></p>
        </div>
    </div>

    <div id="search-row" class="row">
        <div id="search-box-col">
            <div class="form-group">
                <input type="search" placeholder="Search" name="help-search" id="help-search"
                       class="form-control" data-toggle="popover"
                       data-trigger="focus" data-content="Try entering a keyword to filter help topics."
                       data-placement="top" aria-label="Search">
            </div>
        </div>

        <div id="category-col" class="text-center">
            <fieldset id="category-chooser" class="form-group">
                <legend>Category:</legend>
                <div class="btn-group btn-group-toggle" id="category-filter" data-toggle="buttons">
                    <label class="btn btn-secondary active" data-category="all">
                        <input type="radio" name="categories" id="category-all" checked> All
                    </label>
                    <label class="btn btn-secondary" data-category="question">
                        <input type="radio" name="categories" id="category-question"> Question
                    </label>
                    <label class="btn btn-secondary" data-category="usage">
                        <input type="radio" name="categories" id="category-usage"> Usage
                    </label>
                </div>
            </fieldset>
        </div>
    </div>

    <div class="container-fluid">
    <div class="row d-flex">
        <div class="col-sm-3" id="topics-sidebar" role="navigation">
            <h3>All Topics</h3>
            <ul class="nav flex-column" id="sidenav-contents"></nav>
        </div>

        <div class="col-sm-9" id="help-topics">
            <section class="help-topic card" data-category="usage">
                <div class="card-header">
                    <h3>Accessing my reduced data</h3>
                </div>
                <div class="card-body">
                    <p>
                        The reduced data can be access in two ways. Either through an IDAaaS virtual machine or
                        using the IDAaaS FileBrowser.
                    </p>
                    <h4>Access reduced files within IDAaaS Virtual Machines</h4>
                    <ol>
                        <li>Visit the IDAaaS page <a
                                href="https://isis.analysis.stfc.ac.uk">https://isis.analysis.stfc.ac.uk</a>
                        </li>
                        <li>Open an existing or create and open a new IDAaaS virtual machine / instance</li>
                        <li>The reduced files are then available within the virtual machine within folders named
                            "autoreduced" within RB folders; and at data locations as detailed on completed
                            Reduction Job pages on the WebApp
                        </li>
                    </ol>
                    <h4>Use the IDAaaS FileBrowser to copy reduced data elsewhere</h4>
                    <ol>
                        <li>Visit the page <a
                                href="https://data.analysis.stfc.ac.uk">https://data.analysis.stfc.ac.uk</a> and login
                        </li>
                        <li>Click on "Data" on the left menu</li>
                        <li>Reduced data are then available within RB folders that you have access to from HOME or by
                            stepping through to
                            the relevant RB folder from INSTRUMENT. Within these reduced file are within "autoreduced"
                            sub-folders; and at data locations as detailed on completed Reduction Job pages on the
                            WebApp
                        </li>
                    </ol>
                </div>
            </section>
            <section class="help-topic card" data-category="usage">
                <div class="card-header">
                    <h3>Reduction Variables Script</h3>
                </div>
                <div class="card-body">
                    <h4>reduce_vars.py</h4>
                    <p>
                        It is possible to define some additional variables for scripts that can be used
                        during reduction. They can be altered via the web interface. These should be defined in the
                        <code>//isis/NDX.../user/script/autoreduction/reduce_vars.py</code> file. This file has the
                        following format:
                    </p>
                    <pre><code class="language-python">standard_vars = {
    'perform corrections': False,
    'reduction option': "Default value"
}

advanced_vars = {
    'crop range': [100, 10000]
}

variable_help = {
    'standard_vars': {'do_absorb_corrections' : "Do you want corrections on?",
                      'reduction option' : "message to explain option - this is displayed in the webapp"},
    'advanced_vars': {'crop range' : "advanced settings are handled in a separate section"}
}</code></pre>
                </div>
            </section>
            <section class="help-topic card" data-category="usage">
                <div class="card-header">
                    <h3>Reduction Script</h3>
                </div>
                <div class="card-body">

                    <h4>reduce.py</h4>
                    <p>
                        In order to reduce your data you must have a reduction script written in python.
                        This script should be called <code>reduce.py</code> and should be placed in the
                        <code>user/scripts/autoreduction/</code>
                        directory in the ISIS data archive.
                        A full example script can be seen below:
                    </p>

                    <pre><code class="language-python">import os
import sys
from mantid.simpleapi import (Load, Rebin, SaveNexus)
sys.path.append("/isis/NDXTEST/user/scripts/autoreduction")
import reduce_vars as params

def validate(file, dir):
    """
    Function that validates if a file and/or directory exist. If not a
    RunTimeError is raised which is picked up by Autoreduction.

    :param file: full path of data file. Provide empty string to ignore
    :type file: str
    :param dir: full path of a directory. Provide empty string to ignore
    :type dir: str
    """
    print("Running validation")
    if file:
        if not os.path.isfile(file):
            raise RuntimeError("Unable to find file: {}".format(file))
    if dir:
        if not os.path.isdir(dir):
            raise RuntimeError("Unable to find directory: {}".format(dir))
    print("Validation successful")

def main(input_file, output_dir):
   validate(input_file, output_dir)
   workspace = Load(input_file)
   # use rebin params defined in reduce_vars
   workspace = Rebin(workspace, params['rebin'])
   SaveNexus(workspace, output_dir)</code></pre>

                    <h4>Imports</h4>
                    <p>
                        All your imports should be placed at the top of the file. These can include, python
                        packages, mantid packages and the <code>reduce_vars</code> we defined in the above section.
                    </p>

                    <pre><code class="language-python"># import standard python packages
import os
import sys

# import mantid packages
from mantid.simpleapi import (Rebin, ReductionAlgorithm, SaveNexusProcessed)

# import reduce_vars
import reduce_vars as params</code></pre>

                    <h4>Validate</h4>
                    <p>
                        The first function you define should validate that the files and directories you
                        require exist. This should include all files and directories. If any can not be
                        found, an exception should be raised. Below is an example for reference:
                    </p>

                    <pre><code class="language-python">def validate(file, dir):
    """
    Function that validates if a file and/or directory exist. If not a
    RunTimeError is raised which is picked up by Autoreduction.

    :param file: full path of data file. Provide empty string to ignore
    :type file: str
    :param dir: full path of a directory. Provide empty string to ignore
    :type dir: str
    """
    print("Running validation")
    if file:
        if not os.path.isfile(file):
            raise RuntimeError("Unable to find file: {}".format(file))
    if dir:
        if not os.path.isdir(dir):
            raise RuntimeError("Unable to find directory: {}".format(dir))
    print("Validation successful")</code></pre>

                    <h4>Main</h4>
                    <p>
                        This function should call the validate function to check inputs and then perform
                        reduction steps. The function must take an <code>input_file</code> and <code>output_dir</code>
                        as parameters. Both arguments should be validated in the validate function.
                    </p>
                    <pre><code class="language-python">def main(input_file, output_dir):
    # validate
    validate(input_file, output_dir)
    # Perform reduction
    reduced_workspace = ReductionAlgorithm(input_file, ...)
    # Save output
    SaveNexusProcessed(reduced_workspace, output_dir)</code></pre>

                    <p>If you have any difficulty please get in touch and we'll be happy to help </p>
                    <p>If you feel a value is incorrectly being marked as invalid please
                        <a href="mailto:{{ support_email }}">Contact Us</a>.
                    </p>
                </div>
            </section>
            <section class="help-topic card" data-category="usage">
                <div class="card-header">
                    <h3>Autoreduction GitHub Repository</h3>
                </div>
                <div class="card-body">
                    Here is a link to the
                    <a href="https://github.com/ISISScientificComputing/autoreduce">Autoreduction Github repository</a>
                </div>
            </section>

            <section class="help-topic card " data-category="usage">
                <div class="card-header">
                    <h3>How do I add new instrument variables?</h3>
                </div>
                <div class="card-body">
                    <p>
                        To add new instrument variables first you must be part of the instrument scientist
                        team within ICAT. Once you are part of this team you should see an "Edit Reduction
                        Variables" link on the right of the instrument name on the main run list page or
                        "Add new run variables"
                        Clicking either of these links will take you to a form in which you can set the
                        values of variables exposed by that instrument's reduce script.
                    </p>
                </div>
            </section>

            <section class="help-topic card " data-category="question">
                <div class="card-header">
                    <h3>When changing variables why is value X invalid for variable Y?</h3>
                </div>
                <div class="card-body">
                    <p>
                        Any value entered for a variable must match the type of the default value already in
                        the reduction script. If a value doesn't match, you should see a warning message
                        explaining the reason why you will be prevented from submitting new variables until
                        resolved.
                    </p>
                    <p>
                        If you feel a value is incorrectly being marked as invalid please
                        <a href="mailto:{{ support_email }}">Contact Us</a>.
                    </p>

                </div>
            </section>

            <section class="help-topic card " data-category="question">
                <div class="card-header">
                    <h3>I've modified my reduction scripts, why aren't my new runs using it?</h3>
                </div>
                <div class="card-body">
                    <p>
                        To make sure new reductions coming straight from the instrument use new <code>reduce.py</code>
                        scripts automatically, you can navigate to the instrument summary page for a
                        given instrument. From here, select the "See instrument variables" link located on
                        the right of the status table to view ongoing and upcoming instrument
                        configurations.
                        From here, you can select "edit variables" on the left side of the upcoming
                        variables table. When presented with the variables, ensure that the
                        "Track script changes" checkbox is checked. This is located on the right side of
                        the page. You should then select the "Submit Changes" button at the bottom of the
                        page to save this change.
                    </p>
                    <p>
                        When re-running a job, if you want the re-run to use the current <code>reduce.py</code> script
                        click the "Reset to values in the current reduce_vars script" link under "Additional Actions" located
                        on the right side of the page.
                    </p>
                    <p>
                        To update instrument variables to the new ones entered into the <code>reduce_vars.py</code>
                        script, navigate to the "Instrument Summary" page and select the
                        "Configure New Runs" button. This will take you to the instrument variables page.
                        From here, click "Reset to values in the current reduce_vars script", located on the right side of the page.
                        This must be done any time <code>reduce_vars.py</code> is modified.
                    </p>
                </div>
            </section>

            <section class="help-topic card " data-category="question">
                <div class="card-header">
                    <h3>How can I make a reduction script compatible with Autoreduction?</h3>
                </div>
                <div class="card-body">
                    <p>
                        For a reduction script to be compatible with the Autoreduction web application,
                        it must be named <code>reduce.py</code> Please make sure to expose what reduction variables
                        can be modified.
                        These variables must be made available in a file named
                        <code>reduce_vars.py</code>. For an example of this please
                        <a href="https://github.com/ISISScientificComputing/autoreduce/wiki/Getting-Started">
                            See Here
                        </a> for documentation.
                        <a href="https://github.com/ISISScientificComputing/autoreduce/wiki/Example-reduction-script">
                            See Here
                        </a>
                        for the full script.
                        Autoreduction expects the reduction script to have a <code
                            class="language-python">main()</code> method
                        that takes in two arguments, <strong>data</strong> and <strong>output</strong>.
                        These are passed in a kwargs but it may be desirable for these to also be accepted from the
                        command line.
                        For an example of how to do this please
                        <a href="https://github.com/ISISScientificComputing/autoreduce/wiki/Getting-Started#main">
                            See Here
                        </a>.
                        The reduction script should perform all operations on the provided data file and
                        save the output to the output directory provided to the script. The output will
                        be copied to the cache upon completion. If additional save locations are
                        required these can be returned as a list of system paths from the main method.
                    </p>
                    <p>If you feel your script will need to save to additional locations please
                        <a href="mailto:{{ support_email }}">Contact Us</a> to ensure the locations are accessible from
                        the Autoreduction machine.
                    </p>
                </div>
            </section>

            <section class="help-topic card " data-category="question">
                <div class="card-header">
                    <h3>Where should I put reduction scripts?</h3>
                </div>
                <div class="card-body">
                    <p>
                        When you have made sure your reduction script is in the correct format and the
                        variables are exposed, it must be placed within the Autoreduction directory
                        within the user script directory of the appropriate instrument.
                    </p>
                    <p>
                        For example, the reduction script for MERLIN would be located at:
                        <code>\\isis\inst$\NDXMERLIN\user\scripts\autoreduction\reduce.py</code> <br/>
                        and the variables at:
                        <code>\\isis\inst$\NDXMERLIN\user\scripts\autoreduction\reduce_vars.py</code>
                    </p>
                </div>
            </section>

            <section class="help-topic card " data-category="question">
                <div class="card-header" id="reduction_data_location">
                    <h3>Where does my reduced data go?</h3>
                </div>
                <div class="card-body">
                    <p>
                        When a reduction job has completed, the reduced data is moved to the CEPH storage
                        cluster. The full file path to the data is detailed in the run summary pages for
                        completed runs.
                    </p>
                    <p>
                        An example file path in CEPH would be:
                        <code>/instrument/GEM/RBNumber/[RB NUMBER]/autoreduced/[RUN NUMBER]</code>.
                    </p>
                </div>
            </section>
            <section class="help-topic card " data-category="question">
                <div class="card-header">
                    <h3>What do the reduction statuses mean?</h3>
                </div>
                <div class="card-body">
                    There are 5 different states a reduction can be in:
                    <ul>
                        <li>Queued - The run is waiting to be processed</li>
                        <li>Processing - The reduction is in progress</li>
                        <li>Complete - The reduction has been complete and the reduced data is available</li>
                        <li>
                            Error - Something has gone wrong processing the reduction or in the reduction script.
                            If this happens you can <a href="mailto:{{ support_email }}">contact us</a> for support.
                        </li>
                        <li>Skipped - The run was marked to be skipped and has not been reduced</li>
                    </ul>
                </div>
            </section>
            <section class="help-topic card " data-category="question">
                <div class="card-header">
                    <h3>How do I save out plots from my reduce.py?</h3>
                </div>
                <div class="card-body">
                    <p>To get a plot to show into the web app you need to save it into the reduction output directory
                        with the following file name:</p>
                    <pre><code class="language-xml">&lt;name-part-of-the-data-file&gt;*&lt;.png or other extension&gt;</code></pre>
                    <p>where e.g. if the data file is "GEM90963.nxs" then "GEM90963" is the name-part-of-the-data-file,
                        and the plot file extensions
                        supported are "png", "jpg", "bmp", "gif", "tiff".</p>

                    <p>The following function is provided to help you get that information from the input file parameter
                        Autoreduction called the main() method with in reduce.py:</p>
                    <pre><code class="language-python">def get_name_of_data_file(datafile):
    """
    Finds only the name of the data file and returns it. Removes the rest
    of the input full path and the file extension.

    :param datafile: The input file provided in the reduce.py file
    :return: Only the name of the data file
    """
    import os
    filename_and_extension = os.path.basename(datafile)
    just_filename, extension = os.path.splitext(filename_and_extension)
    return just_filename</code></pre>
                    <p>To save out your matplotlib figure you can do:</p>
                    <pre><code class="language-python"> ... some plotting code with matplotlib ...

filename = get_name_of_data_file(input_file)
my_matplotlib_figure.savefig(os.path.join(output_dir, f"{filename}.png"), dpi=None)</code></pre>
                </div>
            </section>
            <section class="help-topic card " data-category="question">
                <div class="card-header">
                    <h3>How do I save out interactive plots with Plotly?</h3>
                </div>
                <div class="card-body">
                    <p>To get an interactive plot in the webapp you need to do the plotting with <strong>Plotly</strong> and save it to a .json file.</p>
                    <pre><code class="language-xml">&lt;name-part-of-the-data-file&gt;.json</code></pre>
                    <p>where e.g. if the data file is "GEM90963.nxs" then "GEM90963" is the <code>name-part-of-the-data-file</code>,
                        and the plot file extension must be "json".</p>

                    <p>The save code that you need in the reduction script is:</p>
                    <pre><code class="language-python"># Serialise the figure to a JSON that can be shown in the webapp
figure_json = my_plotly_figure.to_json()

# Save the file in the output_dir
filename = get_name_of_data_file(input_file)
with open(os.path.join(output_dir, f"{filename}.json")) as figfile:
    figfile.write(figure_json)</code></pre>
                <p>Note: the above uses Python 3's <a href="https://www.python.org/dev/peps/pep-0498/">f-string</a> syntax to make the output JSON filename.</p>
            </section>
            <section class="help-topic card " data-category="question">
                <div class="card-header">
                    <h3>Plotting with Plotly</h3>
                </div>
                <div class="card-body">
                    <p>Plotly is slightly different from Matplotlib, but the difference is mostly in the syntax, while the fundamental features remain the same.
                    As long as Plotly successfully serialises the plot <code>to_json</code>, it should also work in the webapp.</p>
                    <p>Because Mantid integrates Matplotlib, it is <strong>not possible</strong> to plot workspaces directly into Plotly. You will have to extract the data manually via the <code>readX</code>, <code>readY</code>, and similar methods.</p>
                    <p>To avoid adding plotting code here that will get out of date, it is recommended to have a look at the <a href="https://plotly.com/python/line-and-scatter/#scatter-and-line-plot-with-goscatter">documentation</a>.</p>
                <section class="help-topic card " data-category="question">
                  <div class="card-header">
                      <h3>Plotly express or graphics_objects (go)?</h3>
                  </div>
                  <div class="card-body">
                  <p>Both of Plotly's plotting APIs should work. Just use whatever works best for your case.</p>
                  </div>
                </section>
            </section>
            <section class="help-topic card " data-category="question"
                     data-topics="output type structure reduction directory format">
                <div class="card-header">
                    <h3>What are the output structures available in autoreduction?</h3>
                </div>
                <div class="card-body">
                    <p>
                        There are 2 different output structures for reduction runs:
                    </p>
                    <ul>
                        <li>
                            <p>
                                <strong>Flat Output:</strong> In flat output mode all files are outputted into a flat
                                structure. This means that all runs for the same RB will be in the same folder. This
                                mode offers the most simple structure but means every subsequent rerun of a run would
                                overwrite the previous run.

                            </p>
                        </li>
                        <li>
                            <p>
                                <strong>Versioned output:</strong> In Versioned output mode, each run all reruns of
                                runs will be outputted into their own separate folder. For example
                                <code>RB123/1234/run-version-1</code>. This structure results in more files and folders
                                but has the benefit that every version of a reduction run will be kept.
                            </p>
                        </li>
                    </ul>
                </div>
            </section>

            <div class="no-results collapse row card card-body">
                <div class="col-md-12 text-center">
                    Sorry, no help topics matched your query.<br/>
                    If you still require help, please <a href="mailto:{{ support_email }}">Contact Us</a>.
                </div>
            </div>
        </div>
    </div>
    </div>

{% endblock %}

{% block scripts %}
    <script src="https://cdn.jsdelivr.net/npm/fuse.js/dist/fuse.basic.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.3.1/highlight.min.js"></script>
    <script src="{% static "javascript/help.js" %}"></script>
{% endblock %}
