%% Generated by Sphinx.
\def\sphinxdocclass{report}
\documentclass[letterpaper,10pt,english]{sphinxmanual}
\ifdefined\pdfpxdimen
   \let\sphinxpxdimen\pdfpxdimen\else\newdimen\sphinxpxdimen
\fi \sphinxpxdimen=.75bp\relax
\ifdefined\pdfimageresolution
    \pdfimageresolution= \numexpr \dimexpr1in\relax/\sphinxpxdimen\relax
\fi
%% let collapsible pdf bookmarks panel have high depth per default
\PassOptionsToPackage{bookmarksdepth=5}{hyperref}

\PassOptionsToPackage{warn}{textcomp}
\usepackage[utf8]{inputenc}
\ifdefined\DeclareUnicodeCharacter
% support both utf8 and utf8x syntaxes
  \ifdefined\DeclareUnicodeCharacterAsOptional
    \def\sphinxDUC#1{\DeclareUnicodeCharacter{"#1}}
  \else
    \let\sphinxDUC\DeclareUnicodeCharacter
  \fi
  \sphinxDUC{00A0}{\nobreakspace}
  \sphinxDUC{2500}{\sphinxunichar{2500}}
  \sphinxDUC{2502}{\sphinxunichar{2502}}
  \sphinxDUC{2514}{\sphinxunichar{2514}}
  \sphinxDUC{251C}{\sphinxunichar{251C}}
  \sphinxDUC{2572}{\textbackslash}
\fi
\usepackage{cmap}
\usepackage[T1]{fontenc}
\usepackage{amsmath,amssymb,amstext}
\usepackage{babel}



\usepackage{tgtermes}
\usepackage{tgheros}
\renewcommand{\ttdefault}{txtt}



\usepackage[Bjarne]{fncychap}
\usepackage{sphinx}

\fvset{fontsize=auto}
\usepackage{geometry}


% Include hyperref last.
\usepackage{hyperref}
% Fix anchor placement for figures with captions.
\usepackage{hypcap}% it must be loaded after hyperref.
% Set up styles of URL: it should be placed after hyperref.
\urlstyle{same}

\addto\captionsenglish{\renewcommand{\contentsname}{Setup}}

\usepackage{sphinxmessages}
\setcounter{tocdepth}{1}


% Jupyter Notebook code cell colors
\definecolor{nbsphinxin}{HTML}{307FC1}
\definecolor{nbsphinxout}{HTML}{BF5B3D}
\definecolor{nbsphinx-code-bg}{HTML}{F5F5F5}
\definecolor{nbsphinx-code-border}{HTML}{E0E0E0}
\definecolor{nbsphinx-stderr}{HTML}{FFDDDD}
% ANSI colors for output streams and traceback highlighting
\definecolor{ansi-black}{HTML}{3E424D}
\definecolor{ansi-black-intense}{HTML}{282C36}
\definecolor{ansi-red}{HTML}{E75C58}
\definecolor{ansi-red-intense}{HTML}{B22B31}
\definecolor{ansi-green}{HTML}{00A250}
\definecolor{ansi-green-intense}{HTML}{007427}
\definecolor{ansi-yellow}{HTML}{DDB62B}
\definecolor{ansi-yellow-intense}{HTML}{B27D12}
\definecolor{ansi-blue}{HTML}{208FFB}
\definecolor{ansi-blue-intense}{HTML}{0065CA}
\definecolor{ansi-magenta}{HTML}{D160C4}
\definecolor{ansi-magenta-intense}{HTML}{A03196}
\definecolor{ansi-cyan}{HTML}{60C6C8}
\definecolor{ansi-cyan-intense}{HTML}{258F8F}
\definecolor{ansi-white}{HTML}{C5C1B4}
\definecolor{ansi-white-intense}{HTML}{A1A6B2}
\definecolor{ansi-default-inverse-fg}{HTML}{FFFFFF}
\definecolor{ansi-default-inverse-bg}{HTML}{000000}

% Define an environment for non-plain-text code cell outputs (e.g. images)
\makeatletter
\newenvironment{nbsphinxfancyoutput}{%
    % Avoid fatal error with framed.sty if graphics too long to fit on one page
    \let\sphinxincludegraphics\nbsphinxincludegraphics
    \nbsphinx@image@maxheight\textheight
    \advance\nbsphinx@image@maxheight -2\fboxsep   % default \fboxsep 3pt
    \advance\nbsphinx@image@maxheight -2\fboxrule  % default \fboxrule 0.4pt
    \advance\nbsphinx@image@maxheight -\baselineskip
\def\nbsphinxfcolorbox{\spx@fcolorbox{nbsphinx-code-border}{white}}%
\def\FrameCommand{\nbsphinxfcolorbox\nbsphinxfancyaddprompt\@empty}%
\def\FirstFrameCommand{\nbsphinxfcolorbox\nbsphinxfancyaddprompt\sphinxVerbatim@Continues}%
\def\MidFrameCommand{\nbsphinxfcolorbox\sphinxVerbatim@Continued\sphinxVerbatim@Continues}%
\def\LastFrameCommand{\nbsphinxfcolorbox\sphinxVerbatim@Continued\@empty}%
\MakeFramed{\advance\hsize-\width\@totalleftmargin\z@\linewidth\hsize\@setminipage}%
\lineskip=1ex\lineskiplimit=1ex\raggedright%
}{\par\unskip\@minipagefalse\endMakeFramed}
\makeatother
\newbox\nbsphinxpromptbox
\def\nbsphinxfancyaddprompt{\ifvoid\nbsphinxpromptbox\else
    \kern\fboxrule\kern\fboxsep
    \copy\nbsphinxpromptbox
    \kern-\ht\nbsphinxpromptbox\kern-\dp\nbsphinxpromptbox
    \kern-\fboxsep\kern-\fboxrule\nointerlineskip
    \fi}
\newlength\nbsphinxcodecellspacing
\setlength{\nbsphinxcodecellspacing}{0pt}

% Define support macros for attaching opening and closing lines to notebooks
\newsavebox\nbsphinxbox
\makeatletter
\newcommand{\nbsphinxstartnotebook}[1]{%
    \par
    % measure needed space
    \setbox\nbsphinxbox\vtop{{#1\par}}
    % reserve some space at bottom of page, else start new page
    \needspace{\dimexpr2.5\baselineskip+\ht\nbsphinxbox+\dp\nbsphinxbox}
    % mimick vertical spacing from \section command
      \addpenalty\@secpenalty
      \@tempskipa 3.5ex \@plus 1ex \@minus .2ex\relax
      \addvspace\@tempskipa
      {\Large\@tempskipa\baselineskip
             \advance\@tempskipa-\prevdepth
             \advance\@tempskipa-\ht\nbsphinxbox
             \ifdim\@tempskipa>\z@
               \vskip \@tempskipa
             \fi}
    \unvbox\nbsphinxbox
    % if notebook starts with a \section, prevent it from adding extra space
    \@nobreaktrue\everypar{\@nobreakfalse\everypar{}}%
    % compensate the parskip which will get inserted by next paragraph
    \nobreak\vskip-\parskip
    % do not break here
    \nobreak
}% end of \nbsphinxstartnotebook

\newcommand{\nbsphinxstopnotebook}[1]{%
    \par
    % measure needed space
    \setbox\nbsphinxbox\vbox{{#1\par}}
    \nobreak % it updates page totals
    \dimen@\pagegoal
    \advance\dimen@-\pagetotal \advance\dimen@-\pagedepth
    \advance\dimen@-\ht\nbsphinxbox \advance\dimen@-\dp\nbsphinxbox
    \ifdim\dimen@<\z@
      % little space left
      \unvbox\nbsphinxbox
      \kern-.8\baselineskip
      \nobreak\vskip\z@\@plus1fil
      \penalty100
      \vskip\z@\@plus-1fil
      \kern.8\baselineskip
    \else
      \unvbox\nbsphinxbox
    \fi
}% end of \nbsphinxstopnotebook

% Ensure height of an included graphics fits in nbsphinxfancyoutput frame
\newdimen\nbsphinx@image@maxheight % set in nbsphinxfancyoutput environment
\newcommand*{\nbsphinxincludegraphics}[2][]{%
    \gdef\spx@includegraphics@options{#1}%
    \setbox\spx@image@box\hbox{\includegraphics[#1,draft]{#2}}%
    \in@false
    \ifdim \wd\spx@image@box>\linewidth
      \g@addto@macro\spx@includegraphics@options{,width=\linewidth}%
      \in@true
    \fi
    % no rotation, no need to worry about depth
    \ifdim \ht\spx@image@box>\nbsphinx@image@maxheight
      \g@addto@macro\spx@includegraphics@options{,height=\nbsphinx@image@maxheight}%
      \in@true
    \fi
    \ifin@
      \g@addto@macro\spx@includegraphics@options{,keepaspectratio}%
    \fi
    \setbox\spx@image@box\box\voidb@x % clear memory
    \expandafter\includegraphics\expandafter[\spx@includegraphics@options]{#2}%
}% end of "\MakeFrame"-safe variant of \sphinxincludegraphics
\makeatother

\makeatletter
\renewcommand*\sphinx@verbatim@nolig@list{\do\'\do\`}
\begingroup
\catcode`'=\active
\let\nbsphinx@noligs\@noligs
\g@addto@macro\nbsphinx@noligs{\let'\PYGZsq}
\endgroup
\makeatother
\renewcommand*\sphinxbreaksbeforeactivelist{\do\<\do\"\do\'}
\renewcommand*\sphinxbreaksafteractivelist{\do\.\do\,\do\:\do\;\do\?\do\!\do\/\do\>\do\-}
\makeatletter
\fvset{codes*=\sphinxbreaksattexescapedchars\do\^\^\let\@noligs\nbsphinx@noligs}
\makeatother



\title{hpc\sphinxhyphen{}tools}
\date{Feb 03, 2022}
\release{0.1.1dev1}
\author{Chris Finan}
\newcommand{\sphinxlogo}{\vbox{}}
\renewcommand{\releasename}{Release}
\makeindex
\begin{document}

\pagestyle{empty}
\sphinxmaketitle
\pagestyle{plain}
\sphinxtableofcontents
\pagestyle{normal}
\phantomsection\label{\detokenize{index::doc}}


\sphinxAtStartPar
hpc\sphinxhyphen{}tools a Python package with some additional bash scripts/configuration files designed to simplify submitting array jobs on a cluster. Whilst it is written with Grid Engine in mind it can probably be re\sphinxhyphen{}purposed for other systems (such as LSF) \sphinxhyphen{} although this has not been tested in any way. Also, this has only been tested on the University College London (UCL) systems, Myriad and the CS cluster.


\chapter{Contents}
\label{\detokenize{index:contents}}

\section{Getting started with hpc\sphinxhyphen{}tools}
\label{\detokenize{getting_started:getting-started-with-hpc-tools}}\label{\detokenize{getting_started::doc}}
\sphinxAtStartPar
\sphinxstylestrong{version}: \sphinxcode{\sphinxupquote{0.2.1dev0}}

\sphinxAtStartPar
\sphinxhref{https://gitlab.com/cfinan/cluster}{hpc\sphinxhyphen{}tools} is a Python package with some additional bash scripts/configuration files designed to simplify submitting array jobs on a cluster. Whilst it is written with Grid Engine in mind it can probably be re\sphinxhyphen{}purposed for other systems (such as LSF) \sphinxhyphen{} although this has not been tested in any way. Also, this has only been tested on the University College London (UCL) systems, Myriad and the CS cluster.

\sphinxAtStartPar
There is \sphinxhref{https://cfinan.gitlab.io/cluster/index.html}{online} documentation for hpc\sphinxhyphen{}tools and offline PDF documentation can be downloaded \sphinxhref{https://gitlab.com/cfinan/cluster/-/blob/master/resources/pdf/hpc-tools.pdf}{here}.


\subsection{Installation instructions}
\label{\detokenize{getting_started:installation-instructions}}
\sphinxAtStartPar
At present, hpc\sphinxhyphen{}tools is undergoing development and no packages exist yet on PyPi. Therefore it is recommended that it is installed in either of the two ways listed below. First, clone this repository and then \sphinxcode{\sphinxupquote{cd}} to the root of the repository.

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{git} \PYG{n}{clone} \PYG{n}{git}\PYG{n+nd}{@gitlab}\PYG{o}{.}\PYG{n}{com}\PYG{p}{:}\PYG{n}{cfinan}\PYG{o}{/}\PYG{n}{cluster}\PYG{o}{.}\PYG{n}{git}
\PYG{n}{cd} \PYG{n}{cluster}
\end{sphinxVerbatim}


\subsubsection{Installation not using any conda dependencies}
\label{\detokenize{getting_started:installation-not-using-any-conda-dependencies}}
\sphinxAtStartPar
If you are not using conda in any way then install the dependencies via \sphinxcode{\sphinxupquote{pip}} and install hpc\sphinxhyphen{}tools as an editable install also via pip:

\sphinxAtStartPar
Install dependencies:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{python} \PYG{o}{\PYGZhy{}}\PYG{n}{m} \PYG{n}{pip} \PYG{n}{install} \PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{upgrade} \PYG{o}{\PYGZhy{}}\PYG{n}{r} \PYG{n}{requirements}\PYG{o}{.}\PYG{n}{txt}
\end{sphinxVerbatim}

\sphinxAtStartPar
For an editable (developer) install run the command below from the root of the bio\sphinxhyphen{}misc repository (or drop \sphinxcode{\sphinxupquote{\sphinxhyphen{}e}} for static install):

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{python} \PYG{o}{\PYGZhy{}}\PYG{n}{m} \PYG{n}{pip} \PYG{n}{install} \PYG{o}{\PYGZhy{}}\PYG{n}{e} \PYG{o}{.}
\end{sphinxVerbatim}


\subsubsection{Installation using conda}
\label{\detokenize{getting_started:installation-using-conda}}
\sphinxAtStartPar
A conda build is also available for Python v3.7, v3.8 and v3.9 on linux\sphinxhyphen{}64.

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{conda} \PYG{n}{install} \PYG{o}{\PYGZhy{}}\PYG{n}{c} \PYG{n}{cfin} \PYG{n}{hpc}\PYG{o}{\PYGZhy{}}\PYG{n}{tools}
\end{sphinxVerbatim}

\sphinxAtStartPar
If you are using conda and require anything different then please install with pip and install the dependencies with the environments specified in \sphinxcode{\sphinxupquote{resources/conda\_env}}. There is also a build recipe in \sphinxcode{\sphinxupquote{./resourses/build/conda}} that can be used to create new packages.

\sphinxAtStartPar
This will also install \sphinxcode{\sphinxupquote{pyjasub}} and \sphinxcode{\sphinxupquote{make\sphinxhyphen{}jaf}} as console script endpoints that will be executable from the command line.


\subsubsection{Post Python install}
\label{\detokenize{getting_started:post-python-install}}
\sphinxAtStartPar
After installation of hpc\sphinxhyphen{}tools, there are some manual setup steps.


\paragraph{Setup a mapping file}
\label{\detokenize{getting_started:setup-a-mapping-file}}
\sphinxAtStartPar
The main submission script, \sphinxcode{\sphinxupquote{pyjasub}} works by building job submission commands from user arguments. Therefore there is a mapping step to map between the user aguments and the cluster arguments. This happens internally but how the mapping is performed is controlled by a mapping config file. Examples, can be seen in the \sphinxcode{\sphinxupquote{./resources/mappings}} directory of the repository and is also shown below:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{p}{[}\PYG{n}{mappings}\PYG{p}{]}
\PYG{n}{array}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{t} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}\PYG{o}{\PYGZhy{}}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{1}\PYG{p}{\PYGZcb{}}\PYG{p}{:}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{2}\PYG{p}{\PYGZcb{}}

\PYG{c+c1}{\PYGZsh{} Multiple arguments are separated by commas, such as two memory arguments}
\PYG{n}{mem}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{hmem}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}\PYG{p}{,} \PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{vmem}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{time}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{h\PYGZus{}rt}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{nodes}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{hostname}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{shell}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{S} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{out}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{o} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{err}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{e} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{batch}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{tc} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{tmp\PYGZus{}size}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{tmpfs}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{scratch\PYGZus{}size}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{tscratch}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\end{sphinxVerbatim}

\sphinxAtStartPar
This follows a \sphinxcode{\sphinxupquote{.ini}} format and and the Python formatting characters \sphinxcode{\sphinxupquote{\{0\}}}, \sphinxcode{\sphinxupquote{\{1\}}} etc… will be substituted with arguments. The mappings should be under the section header \sphinxcode{\sphinxupquote{{[}mappings{]}}}.

\sphinxAtStartPar
As a single the mappings file is expected per system, the location of the mappings file should be set in your \sphinxcode{\sphinxupquote{\textasciitilde{}/.bashrc}}:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{export} \PYG{n}{QSUB\PYGZus{}PYTHON\PYGZus{}MAPPING}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{/path/to/mappings.conf}\PYG{l+s+s2}{\PYGZdq{}}
\end{sphinxVerbatim}

\sphinxAtStartPar
As the mappings file is a potential security risk, is should only be readable by you. This is checked by \sphinxcode{\sphinxupquote{pyjasub}} and if any other users have any of \sphinxcode{\sphinxupquote{rwx}} on the mapping file then \sphinxcode{\sphinxupquote{pyjasub}} will error out with a \sphinxcode{\sphinxupquote{PermissionError}}.

\sphinxAtStartPar
The abstraction of the cluster arguments has some plusses and minuses, it allows similar commands and job files can be used in different setups. However, this is at the expense of granular control. Having said that \sphinxcode{\sphinxupquote{pyjasub}} can handle most everyday cluster uses and can probably be extended easily. Currently, \sphinxcode{\sphinxupquote{pyjasub}} has only really been tested using \sphinxcode{\sphinxupquote{GridEngine}}, however, it should be able to be adapted for \sphinxcode{\sphinxupquote{LSF}}.


\paragraph{Add the \sphinxstyleliteralintitle{\sphinxupquote{bin}} directory to your \sphinxstyleliteralintitle{\sphinxupquote{PATH}}}
\label{\detokenize{getting_started:add-the-bin-directory-to-your-path}}
\sphinxAtStartPar
There are a number of bash scripts used by \sphinxcode{\sphinxupquote{pyjasub}}. Whilst, they are not obligatory and the user has full control of the task script, they can be used to perform some setup and cleanup operations. If you make sure that the directory \sphinxcode{\sphinxupquote{.resources/bin}} is added to your \sphinxcode{\sphinxupquote{PATH}}, for example, in your \sphinxcode{\sphinxupquote{\textasciitilde{}/.bashrc}} file:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{export} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{\PYGZdl{}}\PYG{l+s+si}{\PYGZob{}PATH\PYGZcb{}}\PYG{l+s+s2}{:/path/to/cluster/resources/bin}\PYG{l+s+s2}{\PYGZdq{}}
\end{sphinxVerbatim}


\section{General Workflow}
\label{\detokenize{workflow:general-workflow}}\label{\detokenize{workflow::doc}}
\sphinxAtStartPar
Here the general idea behind hpc\sphinxhyphen{}tools is outlined along with some of the things that make your life on the cluster a bit easier. Once again, your experience on various HPCs may vary and this is geared towards GridEngine on UCL systems.


\subsection{Array Jobs}
\label{\detokenize{workflow:array-jobs}}
\sphinxAtStartPar
An array job is a single cluster job that is split into multiple tasks. for example each task might do the same thing do different input files and produce a corresponding output file. So, you might want to perform the same actions on 10 different input files producing 10 different output files. So you would submit an array job that is 10 tasks. Obviously, you also have the option of submitting 10 separate jobs in a for loop. However, each of these will have to be scheduled and will have it’s own job ID. So if you notice a problem with the jobs and they need to be killed, then you will have to kill 10 separate jobs. With an array job, you get a single job ID, so it is simple to kill with \sphinxcode{\sphinxupquote{qdel}} (on GridEngine). Additionally, submitting 10 separate jobs puts unnecessary strain on the scheduler. So, if you have multiple tasks, it is a good idea to use array jobs.

\sphinxAtStartPar
So, how to you supply all the parameters required for each task in a job array? The easiest approach is to use a flat file, which has each task on a separate line and each parameter for a task in a separate column. The line in the file that corresponds to the current task can then be interpreted by the script running the task (called the task script) and the values in that line can be used as parameters to run the task. In this readme and in the code, this is referred to as a job array file and I would typically give it a \sphinxcode{\sphinxupquote{.job}} extension although there is no requirement for this. However, there is no requirement for a job array file to be a flat file. It could be something more exotic such as an XML file, as long as the task script knows how to parse it to interpret what is needed to run the current task then that is all that is required.

\sphinxAtStartPar
So, how does the current task know what line it is supposed to read from the job array file? This is the job of a specially created environment variable called the \sphinxcode{\sphinxupquote{SGE\_TASK\_ID}} (on GridEngine). This is available as \sphinxcode{\sphinxupquote{\$SGE\_TASK\_ID}} in a bash script. This will contain an integer, corresponding to the task that is currently be run. So, if it is task numer 5 in the job array then \sphinxcode{\sphinxupquote{\$SGE\_TASK\_ID}} will equal \sphinxcode{\sphinxupquote{5}}. The task script will then know, to extract the line number 5 from the job array file and use it’s contents as parameters to run the task.

\sphinxAtStartPar
The most tricky part of managing an array job is working out what tasks have failed and what tasks have been successful. Tasks can fail for a number of reasons, it could be they have exceeded their memory or time requirements, input/output files are missing or simply a bug in the code. Figuring out what has been successful and what has failed maybe easy if the array job is only 10 tasks but can be tricky/time consuming if there are 100,000s of tasks. There are several good practices that you can use to make your life a lot easier, this package implements some of them directly and assumes that you are using others:
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
Always make your task script a bash script (or whatever your favourite shells is): Whilst not an absolute requirement You can make sure that all your environment is setup (for example conda). The bash task script can then call whatever downstream scripts you need (for example python)

\item {} 
\sphinxAtStartPar
Use \sphinxcode{\sphinxupquote{err}} and \sphinxcode{\sphinxupquote{log}} files and name them appropriately. \sphinxcode{\sphinxupquote{err}} files capture output to \sphinxcode{\sphinxupquote{STDERR}} and outfiles capture output to \sphinxcode{\sphinxupquote{STDOUT}}. For example make sure the variable \sphinxcode{\sphinxupquote{\$TASK\_ID}} is in the file name specification. Also, remember that these files will be appended to not overwritten. So if you re\sphinxhyphen{}run a job several times and point to the same files, then the output of all of your runs will be in the same files, this can get quite confusing.

\item {} 
\sphinxAtStartPar
Make sure that any normal output (progress) is output to \sphinxcode{\sphinxupquote{STDOUT}} and any error output is redirected to \sphinxcode{\sphinxupquote{STDERR}}. In \sphinxcode{\sphinxupquote{Python}}, this will happen implicitly for \sphinxcode{\sphinxupquote{print}} and \sphinxcode{\sphinxupquote{raises}}, in \sphinxcode{\sphinxupquote{Python}} a print statement can also be redirected to \sphinxcode{\sphinxupquote{STDERR}}, using the \sphinxcode{\sphinxupquote{file=sys.stderr}} argument. but in bash you will have to redirect \sphinxcode{\sphinxupquote{echo}} to \sphinxcode{\sphinxupquote{1\textgreater{}\&2}} for \sphinxcode{\sphinxupquote{STDERR}}.

\item {} 
\sphinxAtStartPar
Make sure that you can test your task script outside of the job array submission but in exactly the same way as it will be run when inside the job array. So, for example, you can submit an interactive job via \sphinxcode{\sphinxupquote{qrsh}} and profile the time and memory requirements, i.e. run via \sphinxcode{\sphinxupquote{/usr/bin/time \sphinxhyphen{}v}}. It will also, allow you to efficiently debug your task without submitting the job. It is very difficult to debug a task by continuously submitting jobs and watching them fail.

\item {} 
\sphinxAtStartPar
A common cause of job failures are inaccessible input files and output file locations. These could be caused by a mistake in the file location or a permission error. It is good practice to make sure that all file/directory locations are present and accessible \_\_before\_\_ the job has been submitted. This way, you do not waste time submitting loads of jobs and the deleting them when they all start to fail.

\item {} 
\sphinxAtStartPar
Many scripts will create temporary output or intermediate files. These way be created in a working output directory or in some sort of job runtime \sphinxcode{\sphinxupquote{tmp}} space. What happens to these files when your jobs finishes or if your job errors out? There could be temporary files littered all over the file system and filling up precious \sphinxcode{\sphinxupquote{tmp}} space on a node. It is good practice to make sure that your task script and any other processes it uses clean up after themselves both when they execute cleanly and when they fail. \sphinxcode{\sphinxupquote{bash}} \sphinxcode{\sphinxupquote{traps}} are very useful for this purpose, trapping \sphinxcode{\sphinxupquote{EXIT}} and \sphinxcode{\sphinxupquote{ERR}} signals and executing clean up functions in response. Similarly in \sphinxcode{\sphinxupquote{Python}}, \sphinxcode{\sphinxupquote{try}}, \sphinxcode{\sphinxupquote{except}} and \sphinxcode{\sphinxupquote{finally}} blocks or \sphinxcode{\sphinxupquote{context managers}} can perform these tasks.

\item {} 
\sphinxAtStartPar
If you are submitting thousands of jobs, then sometimes you will get failures for unspecified reasons, it could be other users overloading nodes of filling up \sphinxcode{\sphinxupquote{/tmp}}, sometimes, a job may run out of time or memory, even when you think it will be ok. Either way, the question is, how can you identify what tasks have failed and ones that have run to completion? If you have \sphinxcode{\sphinxupquote{err}} and \sphinxcode{\sphinxupquote{out}} files, these can be parsed to look for telltale signs of failure or completion, for example a \sphinxcode{\sphinxupquote{"*** JOB COMPLETED ***"}} message as the last thing to \sphinxcode{\sphinxupquote{echo}} out as the script exits, perhaps on a clean exit \sphinxcode{\sphinxupquote{trap}}. However, parsing all these files can be error prone and cumbersome. A far better approach, is to look for the presence of expected output files. These paths will often be present in your job array file, so they could be checked against that to determine and flag tasks where the expected output is absent. However, what if your job runs out of memory in the middle of writing an output file? Then you will have an expected output file present but it will be half written. If you have set up an ERROR trap, you could have it fire in response to ERR exit to remove output files. However, it is unclear if these will get fired in an out of memory situation. A good practice to get into is to write output files to a temporary location and when written one of the final commands in the task script is to move the output to it’s final location. That way, any during any mid run failures the output file will not be in it’s final location. An additional safety check is to take the \sphinxcode{\sphinxupquote{MD5}} hash of the file before and after moving. If they do not match then the output file can be deleted before the script exists. This gets around the possibility of something going wrong when moving the output file. Either way, the end goal is to make sure that the output files are not present in their final location until you are 100\% sure that they are the final output that you expect. Then you can safely say the job has completed by looking at the expect output locations in your job array file.

\end{enumerate}

\sphinxAtStartPar
With the points above in mind, the general workflow for submitting an array job would be:
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
Create a job array file. There is a script/module \sphinxcode{\sphinxupquote{make\sphinxhyphen{}jaf}}  that can aid in doing this. See the documentation below for full details.

\item {} 
\sphinxAtStartPar
Write a task script that will perform each task of the job array. Whist, this is probably very specific for the task, there are some general skeleton task files in the \sphinxcode{\sphinxupquote{resources}} directory that can be used as a base for this and adhere to many of the good practices listed above.

\item {} 
\sphinxAtStartPar
Write a job config (ini) file that details the parameters of the job.

\item {} 
\sphinxAtStartPar
Submit the job using \sphinxcode{\sphinxupquote{pyjasub}} (\sphinxtitleref{py} thon \sphinxtitleref{j} ob \sphinxtitleref{a} rray \sphinxtitleref{sub} mission)

\item {} 
\sphinxAtStartPar
After the submission has finished \sphinxhyphen{} re\sphinxhyphen{}run \sphinxcode{\sphinxupquote{pyjasub}} to determine if all the tasks were processed successfully and re\sphinxhyphen{}submit any failed tasks.

\end{enumerate}


\section{Example job array}
\label{\detokenize{example_job_array:example-job-array}}\label{\detokenize{example_job_array::doc}}
\sphinxAtStartPar
Here we will run through a simple toy example of submitting a job with \sphinxcode{\sphinxupquote{pyjasub}}. This assumes that you have installed the package and have your mapping file in place and the path to it set in your \sphinxcode{\sphinxupquote{\textasciitilde{}/.bashrc}}, if not please see \sphinxhref{https://gitlab.com/cfinan/cluster}{here}. Also, to run the scripts in the example you will need to clone the repository and have assess to \sphinxcode{\sphinxupquote{./cluster/resources/example}} where \sphinxcode{\sphinxupquote{.}} is the root of the repository.


\subsection{Creating example files}
\label{\detokenize{example_job_array:creating-example-files}}
\sphinxAtStartPar
For this there is a small script that will create a directory called \sphinxcode{\sphinxupquote{\textasciitilde{}/hpc\_tools\_examples}} in the root of your home directory. If it already exists, it will error out. This directory, will have everything in it to run the examples. Please make sure you are in the \sphinxcode{\sphinxupquote{./cluster/resources/example}} directory when running the script:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{o}{.}\PYG{o}{/}\PYG{n}{create\PYGZus{}example\PYGZus{}files}\PYG{o}{.}\PYG{n}{sh}
\end{sphinxVerbatim}

\sphinxAtStartPar
After running the test input files are available in \sphinxcode{\sphinxupquote{\textasciitilde{}/hpc\_tools\_examples/test\_input}} and the scripts etc are available in \sphinxcode{\sphinxupquote{\textasciitilde{}/hpc\_tools\_examples/example}}


\subsection{Creating a job array file}
\label{\detokenize{example_job_array:creating-a-job-array-file}}
\sphinxAtStartPar
Now we will use the file and directory structure just created to build a job array file that has the following columns:
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{rowidx}} \sphinxhyphen{} A counter starting at 1 for the file non\sphinxhyphen{}header row. This always has to be present in this position in job array files used by \sphinxcode{\sphinxupquote{pyjasub}}

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{infile}} \sphinxhyphen{} The location of the input file (what we just created with \sphinxcode{\sphinxupquote{./create\_example\_files.sh}}

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{outfile}} \sphinxhyphen{} The location of the output file that will be created by our task script

\end{enumerate}

\sphinxAtStartPar
We will use the helper script \sphinxcode{\sphinxupquote{make\sphinxhyphen{}jaf}} to create the basic file structure above. Now we create our job array file using the command below:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{make}\PYG{o}{\PYGZhy{}}\PYG{n}{jaf} \PYG{o}{\PYGZhy{}}\PYG{n}{v} \PYGZbs{}
         \PYG{o}{\PYGZhy{}}\PYG{n}{i} \PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{test\PYGZus{}input} \PYGZbs{}
         \PYG{o}{\PYGZhy{}}\PYG{n}{g} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{*/*.txt}\PYG{l+s+s1}{\PYGZsq{}} \PYGZbs{}
         \PYG{o}{\PYGZhy{}}\PYG{n}{o} \PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{test\PYGZus{}output} \PYGZbs{}
         \PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{regexp} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}(?P\PYGZlt{}B\PYGZgt{}}\PYG{l+s+s1}{\PYGZbs{}}\PYG{l+s+s1}{d+)\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}(?P\PYGZlt{}A\PYGZgt{}}\PYG{l+s+s1}{\PYGZbs{}}\PYG{l+s+s1}{w+)}\PYG{l+s+s1}{\PYGZbs{}}\PYG{l+s+s1}{.txt}\PYG{l+s+s1}{\PYGZsq{}} \PYGZbs{}
         \PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{levels} \PYG{l+m+mi}{1} \PYGZbs{}
         \PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{strip\PYGZus{}ext} \PYGZbs{}
         \PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{suffix} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{.out}\PYG{l+s+s2}{\PYGZdq{}} \PYGZbs{}
         \PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{outfile} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{./example\PYGZus{}job\PYGZus{}array.job}\PYG{l+s+s2}{\PYGZdq{}}
\end{sphinxVerbatim}

\sphinxAtStartPar
The command above can also be run via the script \sphinxcode{\sphinxupquote{create\_job\_array.sh}}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{o}{.}\PYG{o}{/}\PYG{n}{create\PYGZus{}job\PYGZus{}array}\PYG{o}{.}\PYG{n}{sh}
\end{sphinxVerbatim}

\sphinxAtStartPar
Running should produce an output similar to below:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
=== make\PYGZus{}general\PYGZus{}job\PYGZus{}array (hpc\PYGZus{}tools v0.1.0a1) ===
[info] indir value: /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input
[info] infiles length: 0
[info] inglob value: */*.txt
[info] levels value: 1
[info] outdir value: /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output
[info] outfile value: ./example\PYGZus{}job\PYGZus{}array.job
[info] outfile\PYGZus{}errors value: error
[info] prefix value: None
[info] regexp value: i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}(?P\PYGZlt{}B\PYGZgt{}\PYGZbs{}d+)\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}(?P\PYGZlt{}A\PYGZgt{}\PYGZbs{}w+)\PYGZbs{}.txt
[info] strip\PYGZus{}ext value: True
[info] suffix value: .out
[info] verbose value: True
[info] 10 input found
[info] *** END ***
\end{sphinxVerbatim}

\sphinxAtStartPar
The job array file should look similar to this, although the home directory paths will differ:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZdl{} cat example\PYGZus{}job\PYGZus{}array.job 
rowidx  infile  outfile
1       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/india/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}9\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}nine.txt /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/india/nine\PYGZus{}9.out
2       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/foxtrot/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}6\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}six.txt        /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/foxtrot/six\PYGZus{}6.out
3       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/bravo/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}2\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}two.txt  /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/bravo/two\PYGZus{}2.out
4       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/alfa/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}1\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}one.txt   /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/alfa/one\PYGZus{}1.out
5       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/golf/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}7\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}seven.txt /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/golf/seven\PYGZus{}7.out
6       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/hotel/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}8\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}eight.txt        /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/hotel/eight\PYGZus{}8.out
7       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/delta/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}4\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}four.txt /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/delta/four\PYGZus{}4.out
8       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/echo/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}5\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}five.txt  /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/echo/five\PYGZus{}5.out
9       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/charlie/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}3\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}three.txt      /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/charlie/three\PYGZus{}3.out
10      /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/juliett/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}10\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}as\PYGZus{}well\PYGZus{}ten.txt       /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/juliett/ten\PYGZus{}10.out
\end{sphinxVerbatim}

\sphinxAtStartPar
The command above illustrates several useful features of \sphinxcode{\sphinxupquote{make\_general\_job\_array\_file}}, in fact, \sphinxcode{\sphinxupquote{make\_general\_job\_array\_file}} can be imported and extended to make more specific job arrays with other parameters in the file.

\sphinxAtStartPar
So the first thing to point out is that we supplied an input directory \sphinxcode{\sphinxupquote{\sphinxhyphen{}i}} and an input glob \sphinxcode{\sphinxupquote{\sphinxhyphen{}g}}. These are placed together and any files matching that glob will be used. We could have also used the unix globing and placed \sphinxcode{\sphinxupquote{/home/rmjdcfi/hpc\_tools\_examples/test\_input/*/*.txt}} after the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}outfile}} argument. Also, if suppling a glob via \sphinxcode{\sphinxupquote{\sphinxhyphen{}g}} then make sure it is single quoted so unix path expansion does not take place.

\sphinxAtStartPar
We also specified an output directory argument. If we had not, then the input directory would have been used as the output directory. The output directory is also created if it does not exist.

\sphinxAtStartPar
You will also notice that the there are a few differences between the structure of the input file name and that if the output file. Firstly, they have different extensions, the input files have \sphinxcode{\sphinxupquote{.txt}} and the output files have \sphinxcode{\sphinxupquote{.out}}, this is achieved by a combination of \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}strip\_ext}} and \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}suffix}} arguments. Also, our long unweidly file name has been reduced considerably. This is a result of the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}regexp}} argument. Here we supplied a regular expression to match against the file name and capture the parts we want to keep using python named capture groups, labelled alphabetically in the ourder we want them in the output file name. This allows us to extract the relevent parts of the file name. If this option is used, each file name must match the regexp and the output files produced must be unique in the whole job array file (see the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}outfile\_errors}} argument if not). Also, another important point about the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}regexp}} argument is that it must also be single quoted when supplied on the command line.

\sphinxAtStartPar
Finally you will also noticed that we retained the “phonetic” sub\sphinxhyphen{}directory structure of the input files, with \sphinxcode{\sphinxupquote{alfa}}, \sphinxcode{\sphinxupquote{bravo}}, \sphinxcode{\sphinxupquote{charlie}}, \sphinxcode{\sphinxupquote{delta}} etc… . This was accomlished using the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels}} argument. So if \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels 0}} was supplied (the default) then your output files wouldnot have the phonetic sub directory attached, so instead of this \sphinxcode{\sphinxupquote{/home/rmjdcfi/hpc\_tools\_examples/test\_output/juliett/ten\_10.out}}, we would have has this: \sphinxcode{\sphinxupquote{/home/rmjdcfi/hpc\_tools\_examples/test\_output/ten\_10.out}}. Also, if we had supplied \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels 2}}, then we would have had this: \sphinxcode{\sphinxupquote{/home/rmjdcfi/hpc\_tools\_examples/test\_output/test\_input/juliett/ten\_10.out}}. So \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels N}} will take N many directory levels above the input file and add it to below the output directory. All the sub\sphinxhyphen{}directories defined with \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels}} are created automatically so do not have to be created by the task script.


\subsection{Testing the task script}
\label{\detokenize{example_job_array:testing-the-task-script}}
\sphinxAtStartPar
You will want/need to be able to test your task script from outside of your job array and that is what we will do here. The example task script is very simple. It accepts 3 required positional arguments and an optional argument
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
The job array file

\item {} 
\sphinxAtStartPar
Temp location

\item {} 
\sphinxAtStartPar
Step size

\item {} 
\sphinxAtStartPar
Task ID (optional) \sphinxhyphen{} if not supplied then it will default to the \sphinxcode{\sphinxupquote{\$SGE\_TASK\_ID}} environment variable

\end{enumerate}

\sphinxAtStartPar
When running as a job array, the first 3 arguments are supplied by \sphinxcode{\sphinxupquote{pyjasub}} and the forth one defaults to the \sphinxcode{\sphinxupquote{\$SGE\_TASK\_ID}} supplied by \sphinxcode{\sphinxupquote{GridEngine}}. However, when testing on the command line, we supply all 4 with the forth one being the job row in the job array file we want to test.

\sphinxAtStartPar
Now, ensure that the task script is executable with \sphinxcode{\sphinxupquote{chmod +x example\_task.sh}} and assuming you are in the \sphinxcode{\sphinxupquote{example}} directory. Running the first task from the job array should looklike this:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{example}\PYG{o}{/}\PYG{n}{example\PYGZus{}task}\PYG{o}{.}\PYG{n}{sh} \PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{example}\PYG{o}{/}\PYG{n}{example\PYGZus{}job\PYGZus{}array}\PYG{o}{.}\PYG{n}{job} \PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{Scratch}\PYG{o}{/} \PYG{l+m+mi}{1} \PYG{l+m+mi}{1}
\PYG{o}{==}\PYG{o}{=} \PYG{o}{.}\PYG{o}{/}\PYG{n}{example\PYGZus{}task}\PYG{o}{.}\PYG{n}{sh} \PYG{o}{==}\PYG{o}{=}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{started} \PYG{n}{at} \PYG{n}{Wed}  \PYG{l+m+mi}{5} \PYG{n}{Feb} \PYG{l+m+mi}{11}\PYG{p}{:}\PYG{l+m+mi}{39}\PYG{p}{:}\PYG{l+m+mi}{11} \PYG{n}{GMT} \PYG{l+m+mi}{2020}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{hostname}\PYG{o}{=}\PYG{n}{login12}\PYG{o}{.}\PYG{n}{myriad}\PYG{o}{.}\PYG{n}{ucl}\PYG{o}{.}\PYG{n}{ac}\PYG{o}{.}\PYG{n}{uk}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{job} \PYG{n}{array} \PYG{n}{file}\PYG{o}{=}\PYG{n}{example\PYGZus{}job\PYGZus{}array}\PYG{o}{.}\PYG{n}{job}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{temp} \PYG{n}{location}\PYG{o}{=}\PYG{o}{/}\PYG{n}{home}\PYG{o}{/}\PYG{n}{rmjdcfi}\PYG{o}{/}\PYG{n}{Scratch}\PYG{o}{/}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{step}\PYG{o}{=}\PYG{l+m+mi}{1}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{job} \PYG{n}{idx} \PYG{n}{line} \PYG{p}{(}\PYG{n}{SGE\PYGZus{}TASK\PYGZus{}ID}\PYG{p}{)}\PYG{o}{=}\PYG{l+m+mi}{1}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{pull} \PYG{n}{line} \PYG{p}{(}\PYG{n}{accounting} \PYG{k}{for} \PYG{n}{header}\PYG{p}{)}\PYG{o}{=}\PYG{l+m+mi}{2}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{quit} \PYG{n}{line}\PYG{o}{=}\PYG{l+m+mi}{3}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{c+c1}{\PYGZsh{} infiles=1}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{outfile}\PYG{o}{=}\PYG{o}{/}\PYG{n}{home}\PYG{o}{/}\PYG{n}{rmjdcfi}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{test\PYGZus{}output}\PYG{o}{/}\PYG{n}{india}\PYG{o}{/}\PYG{n}{nine\PYGZus{}9}\PYG{o}{.}\PYG{n}{out}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{temp} \PYG{n}{file}\PYG{o}{=}\PYG{o}{/}\PYG{n}{home}\PYG{o}{/}\PYG{n}{rmjdcfi}\PYG{o}{/}\PYG{n}{Scratch}\PYG{o}{/}\PYG{o}{.}\PYG{n}{RUN\PYGZus{}rmjdcfi\PYGZus{}1\PYGZus{}mipaXs6bp8}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{ended} \PYG{n}{at} \PYG{n}{Wed}  \PYG{l+m+mi}{5} \PYG{n}{Feb} \PYG{l+m+mi}{11}\PYG{p}{:}\PYG{l+m+mi}{39}\PYG{p}{:}\PYG{l+m+mi}{11} \PYG{n}{GMT} \PYG{l+m+mi}{2020}
\PYG{o}{*}\PYG{o}{*}\PYG{o}{*} \PYG{n}{END} \PYG{o}{*}\PYG{o}{*}\PYG{o}{*}
\end{sphinxVerbatim}

\sphinxAtStartPar
And will produce an output file like this:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZdl{} cat /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}output/india/nine\PYGZus{}9.out
output file: 
This is the file: /home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/test\PYGZus{}input/india/i\PYGZus{}am\PYGZus{}a\PYGZus{}big\PYGZus{}long\PYGZus{}file\PYGZus{}name\PYGZus{}with\PYGZus{}9\PYGZus{}and\PYGZus{}all\PYGZus{}this\PYGZus{}aswell\PYGZus{}nine.txt
\end{sphinxVerbatim}

\sphinxAtStartPar
Now we attempt to run the second task in the job array. Remember that we have set this to fail before the output is written

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{example}\PYG{o}{/}\PYG{n}{example\PYGZus{}task}\PYG{o}{.}\PYG{n}{sh} \PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{example}\PYG{o}{/}\PYG{n}{example\PYGZus{}job\PYGZus{}array}\PYG{o}{.}\PYG{n}{job} \PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{Scratch}\PYG{o}{/} \PYG{l+m+mi}{1} \PYG{l+m+mi}{2}
\PYG{o}{==}\PYG{o}{=} \PYG{o}{.}\PYG{o}{/}\PYG{n}{example\PYGZus{}task}\PYG{o}{.}\PYG{n}{sh} \PYG{o}{==}\PYG{o}{=}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{started} \PYG{n}{at} \PYG{n}{Wed}  \PYG{l+m+mi}{5} \PYG{n}{Feb} \PYG{l+m+mi}{11}\PYG{p}{:}\PYG{l+m+mi}{41}\PYG{p}{:}\PYG{l+m+mi}{45} \PYG{n}{GMT} \PYG{l+m+mi}{2020}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{hostname}\PYG{o}{=}\PYG{n}{login12}\PYG{o}{.}\PYG{n}{myriad}\PYG{o}{.}\PYG{n}{ucl}\PYG{o}{.}\PYG{n}{ac}\PYG{o}{.}\PYG{n}{uk}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{job} \PYG{n}{array} \PYG{n}{file}\PYG{o}{=}\PYG{n}{example\PYGZus{}job\PYGZus{}array}\PYG{o}{.}\PYG{n}{job}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{temp} \PYG{n}{location}\PYG{o}{=}\PYG{o}{/}\PYG{n}{home}\PYG{o}{/}\PYG{n}{rmjdcfi}\PYG{o}{/}\PYG{n}{Scratch}\PYG{o}{/}
\PYG{p}{[}\PYG{n}{info}\PYG{p}{]} \PYG{n}{step}\PYG{o}{=}\PYG{l+m+mi}{1}
\PYG{p}{[}\PYG{n}{error}\PYG{p}{]} \PYG{n}{simulated} \PYG{n}{failure} \PYG{n}{job} \PYG{n}{index}\PYG{p}{:} \PYG{l+m+mi}{2}
\end{sphinxVerbatim}

\sphinxAtStartPar
Whilst not indicated in the output, our second task should have output a file called \sphinxcode{\sphinxupquote{hpc\_tools\_examples/test\_output/foxtrot/six\_6.out}}, it will not exist.

\sphinxAtStartPar
So we know that our task script is working ok, we can now prepare to run our job array


\subsection{Job array arguments}
\label{\detokenize{example_job_array:job-array-arguments}}
\sphinxAtStartPar
When supplying arguments to \sphinxcode{\sphinxupquote{pyjasub}}, they can either be supplied via the command line or a config file (in \sphinxcode{\sphinxupquote{ini}} format). A config file is easier as it is less error prone. However, both can be used with command like arguments overiding those in the config file. An example config file is given to run the job. It will output the logs of the job into the root of the \sphinxcode{\sphinxupquote{\textasciitilde{}/hpc\_tools\_examples}} directory. It is also shown below:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{c+c1}{\PYGZsh{} A job array config file for extracting the top hits from the interval data}
\PYG{p}{[}\PYG{n}{qsub}\PYG{p}{]}
\PYG{c+c1}{\PYGZsh{} The general parameters for the job. The script has defaults for many of these}
\PYG{n}{step}\PYG{o}{=}\PYG{l+m+mi}{1}
\PYG{n}{batch}\PYG{o}{=}\PYG{l+m+mi}{50}
\PYG{n}{mem}\PYG{o}{=}\PYG{l+m+mi}{1}\PYG{n}{M}
\PYG{n}{shell}\PYG{o}{=}\PYG{o}{/}\PYG{n+nb}{bin}\PYG{o}{/}\PYG{n}{bash}
\PYG{n}{time}\PYG{o}{=}\PYG{l+m+mi}{00}\PYG{p}{:}\PYG{l+m+mi}{01}\PYG{p}{:}\PYG{l+m+mi}{00}
\PYG{n}{nodes}\PYG{o}{=}\PYG{k+kc}{None}
\PYG{n}{tmp\PYGZus{}size}\PYG{o}{=}\PYG{l+m+mi}{10}\PYG{n}{M}
\PYG{n}{tmp\PYGZus{}dir}\PYG{o}{=}\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{Scratch}

\PYG{p}{[}\PYG{n}{logs}\PYG{p}{]}
\PYG{c+c1}{\PYGZsh{} The root for all the job/run logs. If it does not exist it will be created}
\PYG{n}{log\PYGZus{}dir}\PYG{o}{=}\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{example\PYGZus{}job\PYGZus{}logs}

\PYG{p}{[}\PYG{n}{job\PYGZus{}array}\PYG{p}{]}
\PYG{c+c1}{\PYGZsh{} The location of the job array file, I have given a full path}
\PYG{c+c1}{\PYGZsh{} so there are no issues, relative paths are ok, but remember they will}
\PYG{c+c1}{\PYGZsh{} be relative to where the script is run not where the config file is}
\PYG{c+c1}{\PYGZsh{} also \PYGZti{}/ can be used}
\PYG{n}{ja\PYGZus{}file}\PYG{o}{=}\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{example}\PYG{o}{/}\PYG{n}{example\PYGZus{}job\PYGZus{}array}\PYG{o}{.}\PYG{n}{job}

\PYG{c+c1}{\PYGZsh{} The location of the task script that runs the job. Task scripts should}
\PYG{c+c1}{\PYGZsh{} accept 4 arguments.}
\PYG{c+c1}{\PYGZsh{} 1. the job array file}
\PYG{c+c1}{\PYGZsh{} 2. the location of the tmp folder i.e. scratch}
\PYG{c+c1}{\PYGZsh{} 3. the step size for the job array}
\PYG{c+c1}{\PYGZsh{} 4. the job ID i.e. SGE\PYGZus{}TASK\PYGZus{}ID}
\PYG{n}{task\PYGZus{}script}\PYG{o}{=}\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{example}\PYG{o}{/}\PYG{n}{example\PYGZus{}task}\PYG{o}{.}\PYG{n}{sh}

\PYG{c+c1}{\PYGZsh{} A comma separated list of column names in the jobarray files}
\PYG{c+c1}{\PYGZsh{} where the input files are located and the output files are located}
\PYG{c+c1}{\PYGZsh{} only list them if you want the presence of the input files/output}
\PYG{c+c1}{\PYGZsh{} files checked by the submission script. if the column name contains}
\PYG{c+c1}{\PYGZsh{} a comma, protenct with quotes \PYGZdq{}\PYGZdq{}}
\PYG{n}{infiles}\PYG{o}{=}\PYG{n}{infile}
\PYG{n}{outfiles}\PYG{o}{=}\PYG{n}{outfile}

\PYG{p}{[}\PYG{n}{script}\PYG{p}{]}
\PYG{c+c1}{\PYGZsh{} Arguments to the script, at the moment it is just verbose that can be}
\PYG{c+c1}{\PYGZsh{} true or false}
\PYG{n}{verbose}\PYG{o}{=}\PYG{n}{true}
\end{sphinxVerbatim}


\subsection{Running the jobs}
\label{\detokenize{example_job_array:running-the-jobs}}
\sphinxAtStartPar
Now with everything in place the jobs can be submitted with the command:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{pyjasub} \PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{hpc\PYGZus{}tools\PYGZus{}examples}\PYG{o}{/}\PYG{n}{example}\PYG{o}{/}\PYG{n}{example\PYGZus{}job}\PYG{o}{.}\PYG{n}{cnf}
\end{sphinxVerbatim}

\sphinxAtStartPar
Which should produce a report like this if successfully submitted. Note that \sphinxcode{\sphinxupquote{pyjasub}} detects that one of the tasks has already been run (it was run during testing) so it will only submit none tasks. We expect all the evenly numbered tasks to fail as we have engineered it that way, so we expect at least 4 failues on this run, then 2 failures on the second run, 1 failure onn the 3rd run and 0 failures on the forth run. So when we run the fifth time we should get a message saying that the job is complete. Of course, the numbers above assume that no failures occur for other reasons that we can’t control.

\begin{sphinxVerbatim}[commandchars=\\\{\}]
=== pyjasub (hpc\PYGZus{}tools v0.1.0a1) ===
[info] step: 1
[info] batch: 50
[info] mem: 1M
[info] shell: /bin/bash
[info] time: 00:01:00
[info] nodes: None
[info] tmp\PYGZus{}dir: \PYGZti{}/Scratch
[info] scratch\PYGZus{}size: None
[info] tmp\PYGZus{}size: 10M
[info] log\PYGZus{}dir: \PYGZti{}/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs
[info] ja\PYGZus{}file: \PYGZti{}/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}job\PYGZus{}array.job
[info] task\PYGZus{}script: \PYGZti{}/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}task.sh
[info] infiles: [\PYGZsq{}infile\PYGZsq{}]
[info] outfiles: [\PYGZsq{}outfile\PYGZsq{}]
[info] full job log directory \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs\PYGZsq{}
[info] creating log directory \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs\PYGZsq{}
[info] run started at: \PYGZsq{}2020\PYGZhy{}02\PYGZhy{}05 12:29:31.538651\PYGZsq{}
[info] current run number: \PYGZsq{}1\PYGZsq{}
[info] run log path: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/run.log\PYGZsq{}
[info] run directory: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/1\PYGZus{}1580905771\PYGZsq{}
[info] run out files: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/1\PYGZus{}1580905771/out/\PYGZbs{}\PYGZdl{}TASK\PYGZus{}ID\PYGZus{}1\PYGZus{}1580905771.out\PYGZsq{}
[info] run err files: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/1\PYGZus{}1580905771/err/\PYGZbs{}\PYGZdl{}TASK\PYGZus{}ID\PYGZus{}1\PYGZus{}1580905771.err\PYGZsq{}
[info] full job array path: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}job\PYGZus{}array.job\PYGZsq{}
[info] job array md5: \PYGZsq{}db812ee5a433b04843353f4ef813796b\PYGZsq{}
[info] full task script path: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}task.sh\PYGZsq{}
[info] task script md5: \PYGZsq{}1278fb188fb52d4c6615325db565fa8a\PYGZsq{}
[info] copying task script: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/1\PYGZus{}1580905771/1278fb188fb52d4c6615325db565fa8a.task\PYGZsq{}
[info] run job array path: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/1\PYGZus{}1580905771/run\PYGZus{}job\PYGZus{}array.job\PYGZsq{}
[info] check results below:
[info] input file present: \PYGZsq{}10\PYGZsq{}
[info] input file absent: \PYGZsq{}0\PYGZsq{}
[info] output file present: \PYGZsq{}1\PYGZsq{}
[info] output file absent: \PYGZsq{}9\PYGZsq{}
[info] total tasks: \PYGZsq{}10\PYGZsq{}
[info] tasks to be submitted: \PYGZsq{}9\PYGZsq{}
[run] You are about to run \PYGZsq{}9\PYGZsq{} tasks over \PYGZsq{}9\PYGZsq{} rows in a job array with a step of \PYGZsq{}1\PYGZsq{}, using the following command:
[run] qsub \PYGZhy{}t 1\PYGZhy{}9:1 \PYGZhy{}e /lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/1\PYGZus{}1580905771/err/\PYGZbs{}\PYGZdl{}TASK\PYGZus{}ID\PYGZus{}1\PYGZus{}1580905771.err \PYGZhy{}o /lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/1\PYGZus{}1580905771/out/\PYGZbs{}\PYGZdl{}TASK\PYGZus{}ID\PYGZus{}1\PYGZus{}1580905771.out \PYGZhy{}l mem=1M \PYGZhy{}S /bin/bash \PYGZhy{}l h\PYGZus{}rt=00:01:00 \PYGZhy{}l tmpfs=10M /lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}task.sh /lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/1\PYGZus{}1580905771/run\PYGZus{}job\PYGZus{}array.job /lustre/scratch/scratch/rmjdcfi 1
[run] if all this looks correct, press [ENTER] to submit or [CTRL\PYGZhy{}C] to quit \PYGZgt{} 

[run] submitting job...
[run] submitted array job with ID 2768482
[info] run log details below...
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
| run\PYGZus{}time                   | run\PYGZus{}dir      | step | batch | mem | time     | total\PYGZus{}tasks | no\PYGZus{}of\PYGZus{}tasks\PYGZus{}submitted | infile\PYGZus{}exist | infile\PYGZus{}missing | outfile\PYGZus{}exist | outfile\PYGZus{}missing | job\PYGZus{}id  |
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
| 2020\PYGZhy{}02\PYGZhy{}05 12:29:31.538651 | 1\PYGZus{}1580905771 | 1    | 50    | 1M  | 00:01:00 | 10          | 9                     | 10           | 0              | 1             | 9               | 2768482 |
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
[info] *** job submission finished ***
*** END ***
\end{sphinxVerbatim}

\sphinxAtStartPar
After the 5th run you should see something like this, and you will get asked if you want to archive all the log files. Pressing \sphinxcode{\sphinxupquote{ENTER}} will write a tar file on the run directory contents and delete the run directory:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZdl{} pyjasub example\PYGZus{}job.cnf
=== pyjasub (hpc\PYGZus{}tools v0.1.0a1) ===
[info] step: 1
[info] batch: 50
[info] mem: 1G
[info] shell: /bin/bash
[info] time: 00:10:00
[info] nodes: None
[info] tmp\PYGZus{}dir: \PYGZti{}/Scratch
[info] scratch\PYGZus{}size: None
[info] tmp\PYGZus{}size: 1G
[info] log\PYGZus{}dir: \PYGZti{}/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs
[info] ja\PYGZus{}file: \PYGZti{}/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}job\PYGZus{}array.job
[info] task\PYGZus{}script: \PYGZti{}/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}task.sh
[info] infiles: [\PYGZsq{}infile\PYGZsq{}]
[info] outfiles: [\PYGZsq{}outfile\PYGZsq{}]
[info] full job log directory \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs\PYGZsq{}
[info] run started at: \PYGZsq{}2020\PYGZhy{}02\PYGZhy{}05 14:15:24.519655\PYGZsq{}
[info] current run number: \PYGZsq{}5\PYGZsq{}
[info] run log path: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/run.log\PYGZsq{}
[info] run directory: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/5\PYGZus{}1580912124\PYGZsq{}
[info] run out files: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/5\PYGZus{}1580912124/out/\PYGZbs{}\PYGZdl{}TASK\PYGZus{}ID\PYGZus{}5\PYGZus{}1580912124.out\PYGZsq{}
[info] run err files: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/5\PYGZus{}1580912124/err/\PYGZbs{}\PYGZdl{}TASK\PYGZus{}ID\PYGZus{}5\PYGZus{}1580912124.err\PYGZsq{}
[info] full job array path: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}job\PYGZus{}array.job\PYGZsq{}
[info] job array md5: \PYGZsq{}db812ee5a433b04843353f4ef813796b\PYGZsq{}
[info] full task script path: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example/example\PYGZus{}task.sh\PYGZsq{}
[info] task script md5: \PYGZsq{}1278fb188fb52d4c6615325db565fa8a\PYGZsq{}
[info] previous task script md5: \PYGZsq{}1278fb188fb52d4c6615325db565fa8a\PYGZsq{}
[info] copying task script: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/5\PYGZus{}1580912124/1278fb188fb52d4c6615325db565fa8a.task\PYGZsq{}
[info] run job array path: \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs/5\PYGZus{}1580912124/run\PYGZus{}job\PYGZus{}array.job\PYGZsq{}
[info] check results below:
[info] input file present: \PYGZsq{}10\PYGZsq{}
[info] input file absent: \PYGZsq{}0\PYGZsq{}
[info] output file present: \PYGZsq{}10\PYGZsq{}
[info] output file absent: \PYGZsq{}0\PYGZsq{}
[info] total tasks: \PYGZsq{}10\PYGZsq{}
[info] tasks to be submitted: \PYGZsq{}0\PYGZsq{}
[complete] no tasks to submit \PYGZhy{} job is complete
[info] run log details below...
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
| run\PYGZus{}time                   | run\PYGZus{}dir      | step | batch | mem | time     | total\PYGZus{}tasks | no\PYGZus{}of\PYGZus{}tasks\PYGZus{}submitted | infile\PYGZus{}exist | infile\PYGZus{}missing | outfile\PYGZus{}exist | outfile\PYGZus{}missing | job\PYGZus{}id        |
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
| 2020\PYGZhy{}02\PYGZhy{}05 13:17:21.707045 | 1\PYGZus{}1580908641 | 1    | 50    | 1G  | 00:10:00 | 10          | 9                     | 10           | 0              | 1             | 9               | 2768778       |
| 2020\PYGZhy{}02\PYGZhy{}05 13:43:39.033742 | 2\PYGZus{}1580910219 | 1    | 50    | 1G  | 00:10:00 | 10          | 4                     | 10           | 0              | 6             | 4               | 2768812       |
| 2020\PYGZhy{}02\PYGZhy{}05 14:01:20.850012 | 3\PYGZus{}1580911280 | 1    | 50    | 1G  | 00:10:00 | 10          | 2                     | 10           | 0              | 8             | 2               | 2768958       |
| 2020\PYGZhy{}02\PYGZhy{}05 14:09:40.718982 | 4\PYGZus{}1580911780 | 1    | 50    | 1G  | 00:10:00 | 10          | 1                     | 10           | 0              | 9             | 1               | 2768969       |
| 2020\PYGZhy{}02\PYGZhy{}05 14:15:24.519655 | 5\PYGZus{}1580912124 | 1    | 50    | 1G  | 00:10:00 | 10          | 0                     | 10           | 0              | 10            | 0               | NOT\PYGZus{}SUBMITTED |
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
[info] job is complete
[info] *** job submission finished ***
[archive] do you want to archive the run files, press [ENTER] to archive or [CTRL\PYGZhy{}C] to quit \PYGZgt{} 

[archive] archiving to \PYGZsq{}/lustre/home/rmjdcfi/hpc\PYGZus{}tools\PYGZus{}examples/example\PYGZus{}job\PYGZus{}logs.tar.gz\PYGZsq{}
*** END ***
\end{sphinxVerbatim}

\sphinxAtStartPar
\sphinxstylestrong{TODO: Add some notes about the run directories}


\section{Command\sphinxhyphen{}line endpoints}
\label{\detokenize{scripts:command-line-endpoints}}\label{\detokenize{scripts::doc}}
\sphinxAtStartPar
Below is a list of all the command line endpoints installed with hpc\sphinxhyphen{}tools. Many of these are mentioned in their respective context throughout the documentation but they are listed here all in a single place.


\subsection{\sphinxstyleliteralintitle{\sphinxupquote{pyjasub}}}
\label{\detokenize{scripts/pyjasub:pyjasub}}\label{\detokenize{scripts/pyjasub::doc}}
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{pyjasub}} is the main job array submission script and should be available after installation of hpc\sphinxhyphen{}tools.


\subsubsection{Setup}
\label{\detokenize{scripts/pyjasub:setup}}
\sphinxAtStartPar
If you have already followed the post Python installation tasks in getting started, then you can skip this section. If not, then please complete the steps below. \sphinxcode{\sphinxupquote{pyjasub}} works by building job submission commands from user arguments. Therefore there is a mapping step to map between the user aguments and the cluster arguments. This happens internally but how the mapping is performed is controlled by a mapping config file. Examples, can be seen in the \sphinxcode{\sphinxupquote{./resources/mappings}} directory and is also shown below:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{p}{[}\PYG{n}{mappings}\PYG{p}{]}
\PYG{n}{array}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{t} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}\PYG{o}{\PYGZhy{}}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{1}\PYG{p}{\PYGZcb{}}\PYG{p}{:}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{2}\PYG{p}{\PYGZcb{}}

\PYG{c+c1}{\PYGZsh{} Multiple arguments are separated by commas, such as two memory arguments}
\PYG{n}{mem}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{hmem}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}\PYG{p}{,} \PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{vmem}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{time}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{h\PYGZus{}rt}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{nodes}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{hostname}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{shell}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{S} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{out}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{o} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{err}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{e} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{batch}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{tc} \PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{tmp\PYGZus{}size}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{tmpfs}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\PYG{n}{scratch\PYGZus{}size}\PYG{o}{=}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{tscratch}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+m+mi}{0}\PYG{p}{\PYGZcb{}}
\end{sphinxVerbatim}

\sphinxAtStartPar
This follows a \sphinxcode{\sphinxupquote{.ini}} format and and the python formatting characters \sphinxcode{\sphinxupquote{\{0\}}}, \sphinxcode{\sphinxupquote{\{1\}}} etc… will be substituted with arguments. The mappings should be under the section header \sphinxcode{\sphinxupquote{{[}mappings{]}}}.

\sphinxAtStartPar
As a single the mappings file is expected per system, the location of the mappings file should be set in your \sphinxcode{\sphinxupquote{\textasciitilde{}/.bashrc}}:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{export} \PYG{n}{QSUB\PYGZus{}PYTHON\PYGZus{}MAPPING}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{/path/to/mappings.conf}
\end{sphinxVerbatim}

\sphinxAtStartPar
As the mappings file is a potential security risk, is should only be readable by you. This is checked by \sphinxcode{\sphinxupquote{pyjasub}} and if any other users have any of \sphinxcode{\sphinxupquote{rwx}} on the mapping file then \sphinxcode{\sphinxupquote{pyjasub}} will error out with a \sphinxcode{\sphinxupquote{PermissionError}}.

\sphinxAtStartPar
The abstraction of the cluster arguments has some plusses and minuses, it allows similar commands and job files can be used in different setups. However, this is at the expense of granular control. Having said that \sphinxcode{\sphinxupquote{pyjasub}} can handle most everyday cluster uses and can probably be extended easily. Currently, \sphinxcode{\sphinxupquote{pyjasub}} has only really been tested using \sphinxcode{\sphinxupquote{GridEngine}}, however, it should be able to be adapted for \sphinxcode{\sphinxupquote{LSF}}.

\sphinxAtStartPar
There are a number of bash scripts used by \sphinxcode{\sphinxupquote{pyjasub}}. Whilst, they are not obligatory and the user has full control of the task script, they can be used to perform some setup and cleanup operations. If you make sure that the directory \sphinxcode{\sphinxupquote{.resources/bin}} is added to your \sphinxcode{\sphinxupquote{PATH}}, for example, in your \sphinxcode{\sphinxupquote{\textasciitilde{}/.bashrc}} file:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{export} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{\PYGZdl{}}\PYG{l+s+si}{\PYGZob{}PATH\PYGZcb{}}\PYG{l+s+s2}{:/path/to/cluster/resources/bin}\PYG{l+s+s2}{\PYGZdq{}}
\end{sphinxVerbatim}


\subsubsection{Usage}
\label{\detokenize{scripts/pyjasub:usage}}
\sphinxAtStartPar
The options from the command line are as follows, these take priority over options defined in the config file.

\sphinxAtStartPar

\sphinxAtStartPar
Submit a qsub job array accordint to the options and configuration that has been supplied


\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{usage}\PYG{p}{:} \PYG{n}{pyjasub} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{h}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{project} \PYG{n}{PROJECT}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{paid} \PYG{n}{PAID}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{step} \PYG{n}{STEP}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{batch} \PYG{n}{BATCH}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{mem} \PYG{n}{MEM}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{shell} \PYG{n}{SHELL}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{time} \PYG{n}{TIME}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{nodes} \PYG{n}{NODES}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{scratch\PYGZus{}dir} \PYG{n}{SCRATCH\PYGZus{}DIR}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{tmp\PYGZus{}dir} \PYG{n}{TMP\PYGZus{}DIR}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{scratch\PYGZus{}size} \PYG{n}{SCRATCH\PYGZus{}SIZE}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{tmp\PYGZus{}size} \PYG{n}{TMP\PYGZus{}SIZE}\PYG{p}{]}
               \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{log\PYGZus{}dir} \PYG{n}{LOG\PYGZus{}DIR}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{ja\PYGZus{}file} \PYG{n}{JA\PYGZus{}FILE}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{task\PYGZus{}script} \PYG{n}{TASK\PYGZus{}SCRIPT}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{infiles} \PYG{n}{INFILES} \PYG{p}{[}\PYG{n}{INFILES} \PYG{o}{.}\PYG{o}{.}\PYG{o}{.}\PYG{p}{]}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{outfiles} \PYG{n}{OUTFILES} \PYG{p}{[}\PYG{n}{OUTFILES} \PYG{o}{.}\PYG{o}{.}\PYG{o}{.}\PYG{p}{]}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{v}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{w}\PYG{p}{]}
               \PYG{p}{[}\PYG{n}{config}\PYG{p}{]}
\end{sphinxVerbatim}


\paragraph{Positional Arguments}
\label{\detokenize{scripts/pyjasub:Positional Arguments}}\begin{optionlist}{3cm}
\item [config]  
\sphinxAtStartPar
The job config file, options in the config file are overwritten by options on the command line
\end{optionlist}


\paragraph{Named Arguments}
\label{\detokenize{scripts/pyjasub:Named Arguments}}\begin{optionlist}{3cm}
\item [\sphinxhyphen{}\sphinxhyphen{}project]  
\sphinxAtStartPar
Use specific project nodes and potentially a different policy
\item [\sphinxhyphen{}\sphinxhyphen{}paid]  
\sphinxAtStartPar
Force the tasks to use any paid resources specified by project
\item [\sphinxhyphen{}\sphinxhyphen{}step]  
\sphinxAtStartPar
the step size for the array job
\item [\sphinxhyphen{}\sphinxhyphen{}batch]  
\sphinxAtStartPar
the number of concurrent jobs that can be ran simultaniously
\item [\sphinxhyphen{}\sphinxhyphen{}mem]  
\sphinxAtStartPar
The memory for the job
\item [\sphinxhyphen{}\sphinxhyphen{}shell]  
\sphinxAtStartPar
The shell to use for the job
\item [\sphinxhyphen{}\sphinxhyphen{}time]  
\sphinxAtStartPar
The max time for the job
\item [\sphinxhyphen{}\sphinxhyphen{}nodes]  
\sphinxAtStartPar
restrict the job to running on certain nodes
\item [\sphinxhyphen{}\sphinxhyphen{}scratch\_dir]  
\sphinxAtStartPar
The scratch location
\item [\sphinxhyphen{}\sphinxhyphen{}tmp\_dir]  
\sphinxAtStartPar
The location or tmp
\item [\sphinxhyphen{}\sphinxhyphen{}scratch\_size]  
\sphinxAtStartPar
The scratch space that is required
\item [\sphinxhyphen{}\sphinxhyphen{}tmp\_size]  
\sphinxAtStartPar
The tmp space that is required
\item [\sphinxhyphen{}\sphinxhyphen{}log\_dir]  
\sphinxAtStartPar
The space to log all the job parameters and err/out files
\item [\sphinxhyphen{}\sphinxhyphen{}ja\_file]  
\sphinxAtStartPar
The location of the job array file
\item [\sphinxhyphen{}\sphinxhyphen{}task\_script]  
\sphinxAtStartPar
The location of the job array task script
\item [\sphinxhyphen{}\sphinxhyphen{}infiles]  
\sphinxAtStartPar
space separated list of column names in the job array file that contain input files to check they are present
\item [\sphinxhyphen{}\sphinxhyphen{}outfiles]  
\sphinxAtStartPar
space separated list of column names in the job array file that contain output files to check they are present
\item [\sphinxhyphen{}v, \sphinxhyphen{}\sphinxhyphen{}verbose]  
\sphinxAtStartPar
Print out progress

\sphinxAtStartPar
Default: False
\item [\sphinxhyphen{}w, \sphinxhyphen{}\sphinxhyphen{}whole\sphinxhyphen{}steps]  
\sphinxAtStartPar
If step size is \textgreater{} 1 then this ensures that the step size is a multiple of total tasks and will throw an error if not. This treats all the rows steps in the job array file as a logical unit and ensures that all the output files for the steps in the task exists if not they are tasked for re\sphinxhyphen{}running again. If this is not set then the rows are not treated as logical blocks

\sphinxAtStartPar
Default: False
\end{optionlist}


\subsubsection{Job submission workflow}
\label{\detokenize{scripts/pyjasub:job-submission-workflow}}
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{pyjasub}} is designed to make it easier to manage array job submissions and reruns of failed tasks. It can be run on the command line or be imported and the main submission class can be inherited from to taylor a job submission.

\sphinxAtStartPar
The general workflow for using \sphinxcode{\sphinxupquote{pyjasub}} is as follows:
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
Create a job array file  using a script such as \sphinxcode{\sphinxupquote{make\_general\_job\_array\_file}}, or deriving from it. A job array file is a standard flat file that should have a header and be tab delimited. Its role is to supply task specific arguments to each task in the job array. The first column should be a column called \sphinxcode{\sphinxupquote{rowidx}} and  should contain an incremental counter with the row after the header being 1.

\item {} 
\sphinxAtStartPar
Create a job config file that will supply the parameters for the job. Whilst this is optional and the parameters can be supplied on the command line it is recommended to use a config file to avoid errors, arguments given on the command line can be used to override config file parameters. An example of a config file is shown below:

\end{enumerate}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{c+c1}{\PYGZsh{} A job array config file for extracting the top hits from the interval data}
\PYG{p}{[}\PYG{n}{qsub}\PYG{p}{]}
\PYG{c+c1}{\PYGZsh{} The general parameters for the job. The script has defaults for many of these}
\PYG{n}{step}\PYG{o}{=}\PYG{l+m+mi}{1}
\PYG{n}{batch}\PYG{o}{=}\PYG{l+m+mi}{50}
\PYG{n}{mem}\PYG{o}{=}\PYG{l+m+mi}{2}\PYG{n}{G}
\PYG{n}{shell}\PYG{o}{=}\PYG{o}{/}\PYG{n+nb}{bin}\PYG{o}{/}\PYG{n}{bash}
\PYG{n}{time}\PYG{o}{=}\PYG{l+m+mi}{00}\PYG{p}{:}\PYG{l+m+mi}{20}\PYG{p}{:}\PYG{l+m+mi}{00}
\PYG{n}{nodes}\PYG{o}{=}\PYG{k+kc}{None}
\PYG{n}{tmp\PYGZus{}size}\PYG{o}{=}\PYG{l+m+mi}{10}\PYG{n}{G}
\PYG{n}{tmp\PYGZus{}dir}\PYG{o}{=}\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{Scratch}

\PYG{p}{[}\PYG{n}{logs}\PYG{p}{]}
\PYG{c+c1}{\PYGZsh{} The root for all the job/run logs. If it does not exist it will be created}
\PYG{n}{log\PYGZus{}dir}\PYG{o}{=}\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{Scratch}\PYG{o}{/}\PYG{n}{somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs}

\PYG{p}{[}\PYG{n}{job\PYGZus{}array}\PYG{p}{]}
\PYG{c+c1}{\PYGZsh{} The location of the job array file}
\PYG{n}{ja\PYGZus{}file}\PYG{o}{=}\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{somalogic\PYGZus{}interval}\PYG{o}{.}\PYG{n}{jobs}

\PYG{c+c1}{\PYGZsh{} The location of the task script that runs the job. Task scripts should}
\PYG{c+c1}{\PYGZsh{} accept 4 arguments.}
\PYG{c+c1}{\PYGZsh{} 1. the job array file}
\PYG{c+c1}{\PYGZsh{} 2. the location of the tmp folder i.e. scratch}
\PYG{c+c1}{\PYGZsh{} 3. the step size for the job array}
\PYG{c+c1}{\PYGZsh{} 4. the job ID i.e. SGE\PYGZus{}TASK\PYGZus{}ID}
\PYG{n}{task\PYGZus{}script}\PYG{o}{=}\PYG{o}{\PYGZti{}}\PYG{o}{/}\PYG{n}{code}\PYG{o}{/}\PYG{n}{gwas\PYGZus{}import\PYGZus{}pipeline}\PYG{o}{/}\PYG{n}{scripts}\PYG{o}{/}\PYG{n}{extract\PYGZus{}pvalue\PYGZus{}job\PYGZus{}array\PYGZus{}task}\PYG{o}{.}\PYG{n}{sh}

\PYG{c+c1}{\PYGZsh{} A comma separated list of column names in the jobarray files}
\PYG{c+c1}{\PYGZsh{} where the input files are located and the output files are located}
\PYG{c+c1}{\PYGZsh{} only list them if you want the presence of the input files/output}
\PYG{c+c1}{\PYGZsh{} files checked by the submission script. if the column name contains}
\PYG{c+c1}{\PYGZsh{} a comma, protenct with quotes \PYGZdq{}\PYGZdq{}}
\PYG{n}{infiles}\PYG{o}{=}\PYG{n}{infile}
\PYG{n}{outfiles}\PYG{o}{=}\PYG{n}{outfile}

\PYG{p}{[}\PYG{n}{script}\PYG{p}{]}
\PYG{c+c1}{\PYGZsh{} Arguments to the script, at the moment it is just verbose that can be}
\PYG{c+c1}{\PYGZsh{} true or false}
\PYG{n}{verbose}\PYG{o}{=}\PYG{n}{true}
\end{sphinxVerbatim}
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\setcounter{enumi}{2}
\item {} 
\sphinxAtStartPar
Keep on running \sphinxcode{\sphinxupquote{pyjasub}} until all the tasks have been completed. If you have some stubbort tasks the keep failing the check your \sphinxcode{\sphinxupquote{error}} and \sphinxcode{\sphinxupquote{out}} files. These will be located in the log directory under the different run directories.

\item {} 
\sphinxAtStartPar
Once completed, you will asked iff you want to archive your run files. Archiving creates a compressed tar archive and will delete your run directories. An example from a job array that took 4 runs to complete is shown below:

\end{enumerate}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZdl{} pyjasub interval.ja.conf
=== pyjasub (hpc\PYGZus{}tools v0.1.0a1) ===
[info] step: 1
[info] batch: 50
[info] mem: 2G
[info] shell: /bin/bash
[info] time: 00:20:00
[info] nodes: None
[info] tmp\PYGZus{}dir: \PYGZti{}/Scratch
[info] scratch\PYGZus{}size: None
[info] tmp\PYGZus{}size: 10G
[info] log\PYGZus{}dir: \PYGZti{}/Scratch/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs
[info] ja\PYGZus{}file: \PYGZti{}/somalogic\PYGZus{}interval.jobs
[info] task\PYGZus{}script: \PYGZti{}/code/gwas\PYGZus{}import\PYGZus{}pipeline/scripts/extract\PYGZus{}pvalue\PYGZus{}job\PYGZus{}array\PYGZus{}task.sh
[info] infiles: [\PYGZsq{}infile\PYGZsq{}]
[info] outfiles: [\PYGZsq{}outfile\PYGZsq{}]
[info] full job log directory \PYGZsq{}/lustre/scratch/scratch/rmjdcfi/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs\PYGZsq{}
[info] run started at: \PYGZsq{}2020\PYGZhy{}02\PYGZhy{}04 18:07:12.963490\PYGZsq{}
[info] current run number: \PYGZsq{}5\PYGZsq{}
[info] run log path: \PYGZsq{}/lustre/scratch/scratch/rmjdcfi/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs/run.log\PYGZsq{}
[info] run directory: \PYGZsq{}/lustre/scratch/scratch/rmjdcfi/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs/5\PYGZus{}1580839632\PYGZsq{}
[info] run out files: \PYGZsq{}/lustre/scratch/scratch/rmjdcfi/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs/5\PYGZus{}1580839632/out/\PYGZbs{}\PYGZdl{}TASK\PYGZus{}ID\PYGZus{}5\PYGZus{}1580839632.out\PYGZsq{}
[info] run err files: \PYGZsq{}/lustre/scratch/scratch/rmjdcfi/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs/5\PYGZus{}1580839632/err/\PYGZbs{}\PYGZdl{}TASK\PYGZus{}ID\PYGZus{}5\PYGZus{}1580839632.err\PYGZsq{}
[info] full job array path: \PYGZsq{}/lustre/home/rmjdcfi/somalogic\PYGZus{}interval.jobs\PYGZsq{}
[info] job array md5: \PYGZsq{}6b637ce3197d9fcd1ca9bfa875b88a10\PYGZsq{}
[info] full task script path: \PYGZsq{}/lustre/home/rmjdcfi/code/gwas\PYGZus{}import\PYGZus{}pipeline/scripts/extract\PYGZus{}pvalue\PYGZus{}job\PYGZus{}array\PYGZus{}task.sh\PYGZsq{}
[info] task script md5: \PYGZsq{}9094df2fa17e6ff390493094b2c32c9c\PYGZsq{}
[info] previous task script md5: \PYGZsq{}9094df2fa17e6ff390493094b2c32c9c\PYGZsq{}
[info] copying task script: \PYGZsq{}/lustre/scratch/scratch/rmjdcfi/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs/5\PYGZus{}1580839632/9094df2fa17e6ff390493094b2c32c9c.task\PYGZsq{}
[info] run job array path: \PYGZsq{}/lustre/scratch/scratch/rmjdcfi/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs/5\PYGZus{}1580839632/run\PYGZus{}job\PYGZus{}array.job\PYGZsq{}
[info] check results below:
[info] input file present: \PYGZsq{}72226\PYGZsq{}
[info] input file absent: \PYGZsq{}0\PYGZsq{}
[info] output file present: \PYGZsq{}3283\PYGZsq{}
[info] output file absent: \PYGZsq{}0\PYGZsq{}
[info] total tasks: \PYGZsq{}3283\PYGZsq{}
[info] tasks to be submitted: \PYGZsq{}0\PYGZsq{}
[complete] no tasks to submit \PYGZhy{} job is complete
[info] run log details below...
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
| run\PYGZus{}time                   | run\PYGZus{}dir      | step | batch | mem | time     | total\PYGZus{}tasks | no\PYGZus{}of\PYGZus{}tasks\PYGZus{}submitted | infile\PYGZus{}exist | infile\PYGZus{}missing | outfile\PYGZus{}exist | outfile\PYGZus{}missing | job\PYGZus{}id        |
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
| 2020\PYGZhy{}01\PYGZhy{}22 17:03:34.802277 | 1\PYGZus{}1579712614 | 1    | 50    | 2G  | 00:20:00 | 3283        | 3283                  | 72226        | 0              | 0             | 3283            | 2582791       |
| 2020\PYGZhy{}02\PYGZhy{}03 17:38:19.356899 | 2\PYGZus{}1580751499 | 1    | 50    | 2G  | 00:20:00 | 3283        | 246                   | 72226        | 0              | 3037          | 246             | 2751956       |
| 2020\PYGZhy{}02\PYGZhy{}04 05:56:59.818196 | 3\PYGZus{}1580795819 | 1    | 50    | 2G  | 00:20:00 | 3283        | 246                   | 72226        | 0              | 3037          | 246             | 2755344       |
| 2020\PYGZhy{}02\PYGZhy{}04 06:34:21.751488 | 4\PYGZus{}1580798061 | 1    | 50    | 2G  | 00:20:00 | 3283        | 245                   | 72226        | 0              | 3038          | 245             | 2755345       |
| 2020\PYGZhy{}02\PYGZhy{}04 18:07:12.963490 | 5\PYGZus{}1580839632 | 1    | 50    | 2G  | 00:20:00 | 3283        | 0                     | 72226        | 0              | 3283          | 0               | NOT\PYGZus{}SUBMITTED |
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
[info] job is complete
[info] *** job submission finished ***
[archive] do you want to archive the run files, press [ENTER] to archive or [CTRL\PYGZhy{}C] to quit \PYGZgt{}

[archive] archiving to \PYGZsq{}/lustre/scratch/scratch/rmjdcfi/somalogic\PYGZus{}interval\PYGZus{}top\PYGZus{}hits\PYGZus{}jobs.tar.gz\PYGZsq{}
*** END ***
\end{sphinxVerbatim}


\subsubsection{Checks on the job array file}
\label{\detokenize{scripts/pyjasub:checks-on-the-job-array-file}}
\sphinxAtStartPar
Each run of \sphinxcode{\sphinxupquote{pyjasub}} will start with a check or the job array file. The following are checked, if they to not pass then we get a fatal error
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
Has the job array file changed? The job array file is crucial to the integrety of all the jobs, therefore it must stay constant for the lifetime off all runs of the job.

\item {} 
\sphinxAtStartPar
Is the first column of the job array \sphinxcode{\sphinxupquote{rowidx}}, if not it is an error.

\item {} 
\sphinxAtStartPar
For each each row in the job array file, is it the same length (no of columns) as the header

\item {} 
\sphinxAtStartPar
For each each row in the job array file, is the first column of the row an expected incremental counter i.e. first column of the first row after the header should have the value \sphinxcode{\sphinxupquote{1}}.

\end{enumerate}


\subsubsection{Defining a completed job}
\label{\detokenize{scripts/pyjasub:defining-a-completed-job}}
\sphinxAtStartPar
So here it can get tricky. We have to take into account the step size when evaluating the output files. So, we have to have all output files present for all steps in the task.


\subsection{\sphinxstyleliteralintitle{\sphinxupquote{make\sphinxhyphen{}jaf}}}
\label{\detokenize{scripts/make_jaf:make-jaf}}\label{\detokenize{scripts/make_jaf::doc}}
\sphinxAtStartPar
Create a general job array file that has 3 columns. The file also has a header and is tab separated. This can me executed as a script or imported as a module to aid building of more specific job array files.


\subsubsection{Usage}
\label{\detokenize{scripts/make_jaf:usage}}
\sphinxAtStartPar
The options from the command line are as follows, these take priority over options defined in the config file.

\sphinxAtStartPar

\sphinxAtStartPar
Generate a skeleton for a job array file with input file, output file columns and an index column. All input files are checked to make sure they can be accessed and it is a fatal error if they can not, similarly, each output location is checked to see if it is writable. If the output file already exists then it is a fatal error


\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{usage}\PYG{p}{:} \PYG{n}{make}\PYG{o}{\PYGZhy{}}\PYG{n}{jaf} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{h}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{f} \PYG{n}{OUTFILE}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{i} \PYG{n}{INDIR}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{g} \PYG{n}{INGLOB}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{o} \PYG{n}{OUTDIR}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{l} \PYG{n}{LEVELS}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{e} \PYG{p}{\PYGZob{}}\PYG{n}{error}\PYG{p}{,}\PYG{n}{ignore}\PYG{p}{,}\PYG{n}{flatten}\PYG{p}{\PYGZcb{}}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{p} \PYG{n}{PREFIX}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{s} \PYG{n}{SUFFIX}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{r} \PYG{n}{REGEXP}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{x}\PYG{p}{]} \PYG{p}{[}\PYG{o}{\PYGZhy{}}\PYG{n}{v}\PYG{p}{]} \PYG{p}{[}\PYG{n}{infiles} \PYG{p}{[}\PYG{n}{infiles} \PYG{o}{.}\PYG{o}{.}\PYG{o}{.}\PYG{p}{]}\PYG{p}{]}
\end{sphinxVerbatim}


\paragraph{Positional Arguments}
\label{\detokenize{scripts/make_jaf:Positional Arguments}}\begin{optionlist}{3cm}
\item [infiles]  
\sphinxAtStartPar
The input files to for the job array, if none are supplied then \textendash{}indir should be set and all the files in indir are used as input
\end{optionlist}


\paragraph{Named Arguments}
\label{\detokenize{scripts/make_jaf:Named Arguments}}\begin{optionlist}{3cm}
\item [\sphinxhyphen{}f, \sphinxhyphen{}\sphinxhyphen{}outfile]  
\sphinxAtStartPar
the output file, if not provided, output is to STDOUT
\item [\sphinxhyphen{}i, \sphinxhyphen{}\sphinxhyphen{}indir]  
\sphinxAtStartPar
the input directory, this should be set if thereare no input files, it can also be used in conjunction with input files
\item [\sphinxhyphen{}g, \sphinxhyphen{}\sphinxhyphen{}inglob]  
\sphinxAtStartPar
An input directory glob that is used in conjunction with the \textendash{}indir, the default is no pattern
\item [\sphinxhyphen{}o, \sphinxhyphen{}\sphinxhyphen{}outdir]  
\sphinxAtStartPar
An output directory, if this is not set then it is assumed that the output directory is the same directory as the directory that each input file resides in. If the outdir does not exist then it will be created
\item [\sphinxhyphen{}l, \sphinxhyphen{}\sphinxhyphen{}levels]  
\sphinxAtStartPar
active when \textendash{}outdir is used, this uses number of levels directory structure down from infile and is appended to the outdir. e.g. outdir=/home/bob/my\_data , input file=’/data/to/process/file.txt . If \textendash{}levels = 0 then the outfile will be ‘/home/bob/my\_data/file.txt’, if \textendash{}levels 1 is used, then ‘/home/bob/my\_data/process/file.txt’ and iflevels 2 is used, then ‘/home/bob/my\_data/to/process/file.txt’

\sphinxAtStartPar
Default: 0
\item [\sphinxhyphen{}e, \sphinxhyphen{}\sphinxhyphen{}outfile\_errors]  
\sphinxAtStartPar
Possible choices: error, ignore, flatten

\sphinxAtStartPar
action to take if there are errors with non\sphinxhyphen{}unique output files. ‘flatten’, will make the output file unique and group all the input files into a single column delimited with a pipe ‘|’

\sphinxAtStartPar
Default: “error”
\item [\sphinxhyphen{}p, \sphinxhyphen{}\sphinxhyphen{}prefix]  
\sphinxAtStartPar
A prefix to add to each output file, the default is ‘’
\item [\sphinxhyphen{}s, \sphinxhyphen{}\sphinxhyphen{}suffix]  
\sphinxAtStartPar
A suffix to add to each output file, the default is ‘\_output’
\item [\sphinxhyphen{}r, \sphinxhyphen{}\sphinxhyphen{}regexp]  
\sphinxAtStartPar
A regular expression used to generate the output file name from the full input file path. The regexp will be applied using re.search. All groups will be captured, although the groups required in the output file name should be labels ‘1’, ‘2’, ‘3’ using the named capure groups \sphinxtitleref{?P\textless{}1\textgreater{}} etc… . Remember to also use non\sphinxhyphen{}capturing groups \sphinxtitleref{(?:)} where necessary.
\item [\sphinxhyphen{}x, \sphinxhyphen{}\sphinxhyphen{}strip\_ext]  
\sphinxAtStartPar
shall the extension be stripped from the outputfile name

\sphinxAtStartPar
Default: False
\item [\sphinxhyphen{}v, \sphinxhyphen{}\sphinxhyphen{}verbose]  
\sphinxAtStartPar
shall I print progress to STDERR?

\sphinxAtStartPar
Default: False
\end{optionlist}


\subsubsection{Output file}
\label{\detokenize{scripts/make_jaf:output-file}}
\sphinxAtStartPar
When executed as a script it will generate a three column TAB delimited job array file with the columns:
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{rowidx}} An integer counter of the job number. Note that \sphinxcode{\sphinxupquote{pyjasub}} always expects the first column to be a \sphinxcode{\sphinxupquote{rowidx}} column.

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{infile}} An input file column, this is not mandatory but most job tasks will probably require at least a single input file. Note, that is is valid for this column to contain a pipe ‘|’ separated set on input files in this column also. These are generated if the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}flatten}} option is active and the output file names are not unique.

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{outfile}} An output file column. Note it is also valid to have a pipe \sphinxcode{\sphinxupquote{|}} separated set of output files in this column. Although, \sphinxcode{\sphinxupquote{make\sphinxhyphen{}jaf}} does not do this.

\end{enumerate}
\begin{description}
\item[{The \sphinxcode{\sphinxupquote{make\sphinxhyphen{}jaf}} script firsts gathers all the available input files, depending on the user arguments. The user can either supply input files as positional arguments to the script or additionally an input directory (\sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}indir}}) and/or a file globing option to match unix wild cards (\sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}inglob}}). The \sphinxcode{\sphinxupquote{inglob}} is applied to the input directory and may actually be made into a single option in future. If \sphinxcode{\sphinxupquote{indir}} is supplied and \sphinxcode{\sphinxupquote{inglob}} is not supplied then the entire contents of \sphinxcode{\sphinxupquote{indir}} are used as input files. All}] \leavevmode
\sphinxAtStartPar
input files are checked to make sure that they are actually files and not directories, any input files that do end up being directories are ignored. If the \sphinxcode{\sphinxupquote{infiles}} positional argument, \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}indir}} and \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}inglob}} are all used then the full complement of files gathered by all of them will be used. All input files are checked to make sure they are readable by opening and closing the file. If this fails then it is a fatal error.

\item[{Output file names are generated from the input file names. The \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}outdir}}, \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels}}, \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}regexp}}, \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}strip\_ext}}, \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}prefix}} and \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}suffix}} options control how output file names are generated. If \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}outdir}} is not specified then the output file for each input file in the job array is the same as the input file directory and \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels}} has no effect. Also, in this instance, if no other output options are used then the output file name will}] \leavevmode
\sphinxAtStartPar
be the same as the input file name and this will cause an error. If \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}outdir}} is supplied then the basename of the input file is added to the output file name. If \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels}} is also supplied, then the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}levels}} number of directories up from the input files are created below the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}outdir}}. The default is 0, i.e. no levels down from the input directory.

\sphinxAtStartPar
The \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}prefix}} and \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}suffix}} generates an output file name with strings added to the start or the end of the input file basename. \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}suffix}} is added before the file extension. \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}strip\_ext}} will generate an output file name from the input file name with the extension removed. So a new output file extension can be generated with a combination of \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}strip\_ext}} and \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}suffix}}.

\end{description}

\sphinxAtStartPar
The \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}regexp}} argument allows the user to supply a regexp on the command line that will be applied to the full path of the input file and generate an output file name. Even if \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}regexp}} is included, all the other options still apply. This regexp uses named capture groups to extract the parts that should be used in the output file name. the capture groups should be named in uppercase alphabet characters and the order in the alphabet is the order they appear in the output file name. i.e. \sphinxcode{\sphinxupquote{r\textquotesingle{}(?P\textless{}A\textgreater{}first\_bit)\textquotesingle{}}}.

\sphinxAtStartPar
By default, if all the output file names are not unique, then an error is generated (\sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}outfile\_errors \textquotesingle{}error\textquotesingle{}}}). This can be set to \sphinxcode{\sphinxupquote{\textquotesingle{}ignore\textquotesingle{}}} if this is what is intended, i.e. it could be that the job array has a different step size. Also, this can be set to \sphinxcode{\sphinxupquote{\textquotesingle{}flatten\textquotesingle{}}} and in this case the input files are flattened into a pipe \sphinxcode{\sphinxupquote{|}} separated list to give a unique number of output files.


\subsubsection{Basic API usage}
\label{\detokenize{scripts/make_jaf:basic-api-usage}}
\sphinxAtStartPar
This can also be imported as a module. See the reference documentation for the functions within.

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{k+kn}{from} \PYG{n+nn}{cluster} \PYG{k+kn}{import} \PYG{n}{make\PYGZus{}general\PYGZus{}job\PYGZus{}array\PYGZus{}file}
\end{sphinxVerbatim}


\section{Contributing to the project}
\label{\detokenize{contributing:contributing-to-the-project}}\label{\detokenize{contributing::doc}}
\sphinxAtStartPar
I welcome contributions from all users to the project:
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
Documentation fixes/updates

\item {} 
\sphinxAtStartPar
Example notebooks to illustrate features/methods/Example bash scripts for \sphinxcode{\sphinxupquote{./resources/bin}}

\item {} 
\sphinxAtStartPar
Bug fixes

\item {} 
\sphinxAtStartPar
New scripts

\end{enumerate}

\sphinxAtStartPar
However, we ask that contributors please follow these guidelines when contributing to the package. This allows us to retain some level of homogenerity throughout a project and will make it easiy to maintain and improve in the long run.

\sphinxAtStartPar
A list of current contributors can be found in the \sphinxhref{https://gitlab.com/cfinan/cluster}{root of the repository}.
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
Please ensure that all code follows the Python \sphinxhref{https://www.python.org/dev/peps/pep-0008/}{PEP\sphinxhyphen{}8} standard. the easiest way to do this is to use an IDE with a good \sphinxhref{https://sourcelevel.io/blog/what-is-a-linter-and-why-your-team-should-use-it}{linter}.

\item {} 
\sphinxAtStartPar
Please ensure that your code is well commented, so that programmers of varying abilities have a good chance in following it.

\item {} 
\sphinxAtStartPar
Please make sure that any code, or bug\sphinxhyphen{}fixes you contribute are accompanied by a \sphinxhref{https://docs.pytest.org}{pytest}.

\item {} 
\sphinxAtStartPar
Please run a full \sphinxhref{https://docs.pytest.org}{pytest} prior to merging your code and ensure all tests pass (or that your new code has not caused any new test failures).

\item {} 
\sphinxAtStartPar
Please make sure that all modules, classes, functions (even private ones) have docstrings. This enables us to easily produce high quality documentation and improves the user experience. hpc\_tools uses Numpy docstrings for the most part. Please follow \sphinxhref{https://developer.lsst.io/python/numpydoc.html}{this} excellent guide if unsure.

\item {} 
\sphinxAtStartPar
Any HOWTOs or extended user documentation should be written in \sphinxhref{https://docutils.sourceforge.io/rst.html}{reStructuredText} (preferably) or \sphinxhref{https://en.wikipedia.org/wiki/Markdown}{Markdown}.

\item {} 
\sphinxAtStartPar
We incorporate Jupyter notebooks into the documentation, so please ensure they are fairly quick to run and do not have reams of output associated with them. Also, please make sure that all output is collapsed (removed) prior to committing the notebook to git.

\item {} 
\sphinxAtStartPar
Please make sure that any test data you submit is small and compressed and please do not commit any individual level data that is not already in the public domain.

\end{enumerate}

\sphinxAtStartPar
If you want to contribute and are unsure on any of these points please \sphinxhref{mailto:c.finan@ucl.ac.uk}{contact us}. We will acknowledge any contributions in future publications, either as acknowledgements or authorship positions in the case of significant contributions.

\sphinxAtStartPar
Many thanks in advance!


\section{Package management}
\label{\detokenize{package_management:package-management}}\label{\detokenize{package_management::doc}}
\sphinxAtStartPar
Here is documentation on the maintenance of the package. This is not really geared towards end users but rather a log of how version numbering is handled and how documentation is built and maintained. Throughout this section \sphinxcode{\sphinxupquote{./}} indicates the root of the repository.


\subsection{Versioning}
\label{\detokenize{package_management:versioning}}
\sphinxAtStartPar
hpc\sphinxhyphen{}tools uses 3 level versioning:
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
Major version number

\item {} 
\sphinxAtStartPar
Minor version number

\item {} 
\sphinxAtStartPar
Patch version number

\end{enumerate}

\sphinxAtStartPar
For all non\sphinxhyphen{}stable versions (which is all of them at present), these are augmented with a:
\begin{itemize}
\item {} 
\sphinxAtStartPar
release \sphinxhyphen{} either \sphinxcode{\sphinxupquote{dev}}, \sphinxcode{\sphinxupquote{a}} (alpha), \sphinxcode{\sphinxupquote{b}} (beta), \sphinxcode{\sphinxupquote{rc}} (release candidate), \sphinxcode{\sphinxupquote{prod}} (production)

\item {} 
\sphinxAtStartPar
build number, this build number is associated with a release

\end{itemize}

\sphinxAtStartPar
So, a development version number will look like \sphinxcode{\sphinxupquote{0.2.0dev0}}

\sphinxAtStartPar
While hpc\sphinxhyphen{}tools is unfinished, the versioning will be slightly ad\sphinxhyphen{}hoc but will stabilise once the minimal code base is in place.

\sphinxAtStartPar
hpc\sphinxhyphen{}tools uses \sphinxhref{https://github.com/c4urself/bump2version}{\sphinxcode{\sphinxupquote{bump2version}}} to increment the version numbers throughout the package. The version numbers are located in several different places (a \sphinxcode{\sphinxupquote{.}} denotes the root of the repository):
\begin{itemize}
\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{./.bumpversion.cfg}}

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{./README.md}}

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{./genomic\_config/\_\_init\_\_.py}}

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{./setup.py}}

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{./VERSION}}

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{./docs/source/conf.py}}

\item {} 
\sphinxAtStartPar
\sphinxcode{\sphinxupquote{./resources/build/conda/py*/meta.yaml}}

\end{itemize}

\sphinxAtStartPar
The bumpversion config file is shown below but is also a hidden file in the root of the repository:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
[bumpversion]
current\PYGZus{}version = 0.2.0dev0
commit = True
tag = True
parse = (?P\PYGZlt{}major\PYGZgt{}\PYGZbs{}d+)\PYGZbs{}.(?P\PYGZlt{}minor\PYGZgt{}\PYGZbs{}d+)\PYGZbs{}.(?P\PYGZlt{}patch\PYGZgt{}\PYGZbs{}d+)(\PYGZbs{}\PYGZhy{}?(?P\PYGZlt{}release\PYGZgt{}[a\PYGZhy{}z]+)(?P\PYGZlt{}build\PYGZgt{}\PYGZbs{}d+))?
serialize = 
	\PYGZob{}major\PYGZcb{}.\PYGZob{}minor\PYGZcb{}.\PYGZob{}patch\PYGZcb{}\PYGZob{}release\PYGZcb{}\PYGZob{}build\PYGZcb{}
	\PYGZob{}major\PYGZcb{}.\PYGZob{}minor\PYGZcb{}.\PYGZob{}patch\PYGZcb{}

[bumpversion:part:release]
optional\PYGZus{}value = prod
first\PYGZus{}value = dev
values = 
	dev
	a
	b
	rc
	prod

[bumpversion:part:build]

[bumpversion:file:VERSION]

[bumpversion:file:./README.md]
search = \PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{}: `\PYGZob{}current\PYGZus{}version\PYGZcb{}`
replace = \PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{}: `\PYGZob{}new\PYGZus{}version\PYGZcb{}`

[bumpversion:file:./\PYGZus{}version.py]
search = \PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{} = \PYGZsq{}\PYGZob{}current\PYGZus{}version\PYGZcb{}\PYGZsq{}
replace = \PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{} = \PYGZsq{}\PYGZob{}new\PYGZus{}version\PYGZcb{}\PYGZsq{}

[bumpversion:file:./genomic\PYGZus{}config/\PYGZus{}\PYGZus{}init\PYGZus{}\PYGZus{}.py]
search = \PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{} = \PYGZsq{}\PYGZob{}current\PYGZus{}version\PYGZcb{}\PYGZsq{}
replace = \PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{} = \PYGZsq{}\PYGZob{}new\PYGZus{}version\PYGZcb{}\PYGZsq{}

[bumpversion:file:./setup.py]
search = VERSION = \PYGZsq{}\PYGZob{}current\PYGZus{}version\PYGZcb{}\PYGZsq{}
replace = VERSION = \PYGZsq{}\PYGZob{}new\PYGZus{}version\PYGZcb{}\PYGZsq{}

[bumpversion:file:./docs/source/conf.py]
search = release = \PYGZsq{}\PYGZob{}current\PYGZus{}version\PYGZcb{}\PYGZsq{}
replace = release = \PYGZsq{}\PYGZob{}new\PYGZus{}version\PYGZcb{}\PYGZsq{}
\end{sphinxVerbatim}

\sphinxAtStartPar
The procedure for incrementing the version number is as follows. This assumes that the \sphinxcode{\sphinxupquote{patch}} number is being bumped and you are located in the root of the repository (where your \sphinxcode{\sphinxupquote{.bumpconfig.cfg}} file is located):
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
run \sphinxcode{\sphinxupquote{bump2version}} in “dry\sphinxhyphen{}run” (\sphinxcode{\sphinxupquote{\sphinxhyphen{}n}}) mode with \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}verbose}} to make sure everything is ok:

\end{enumerate}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZdl{} bump2version \PYGZhy{}\PYGZhy{}verbose \PYGZhy{}n patch
current\PYGZus{}version=0.2.0dev0
commit=True
tag=True
parse=(?P\PYGZlt{}major\PYGZgt{}\PYGZbs{}d+)\PYGZbs{}.(?P\PYGZlt{}minor\PYGZgt{}\PYGZbs{}d+)\PYGZbs{}.(?P\PYGZlt{}patch\PYGZgt{}\PYGZbs{}d+)(\PYGZbs{}\PYGZhy{}?(?P\PYGZlt{}release\PYGZgt{}[a\PYGZhy{}z]+)(?P\PYGZlt{}build\PYGZgt{}\PYGZbs{}d+))?
serialize=
\PYGZob{}major\PYGZcb{}.\PYGZob{}minor\PYGZcb{}.\PYGZob{}patch\PYGZcb{}\PYGZob{}release\PYGZcb{}\PYGZob{}build\PYGZcb{}
\PYGZob{}major\PYGZcb{}.\PYGZob{}minor\PYGZcb{}.\PYGZob{}patch\PYGZcb{}
new\PYGZus{}version=0.2.1dev0
\end{sphinxVerbatim}
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
If it all looks good (which it does above) then run for real:

\end{enumerate}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZdl{} bump2version \PYGZhy{}\PYGZhy{}verbose patch
current\PYGZus{}version=0.2.0dev0
commit=True
tag=True
parse=(?P\PYGZlt{}major\PYGZgt{}\PYGZbs{}d+)\PYGZbs{}.(?P\PYGZlt{}minor\PYGZgt{}\PYGZbs{}d+)\PYGZbs{}.(?P\PYGZlt{}patch\PYGZgt{}\PYGZbs{}d+)(\PYGZbs{}\PYGZhy{}?(?P\PYGZlt{}release\PYGZgt{}[a\PYGZhy{}z]+)(?P\PYGZlt{}build\PYGZgt{}\PYGZbs{}d+))?
serialize=
\PYGZob{}major\PYGZcb{}.\PYGZob{}minor\PYGZcb{}.\PYGZob{}patch\PYGZcb{}\PYGZob{}release\PYGZcb{}\PYGZob{}build\PYGZcb{}
\PYGZob{}major\PYGZcb{}.\PYGZob{}minor\PYGZcb{}.\PYGZob{}patch\PYGZcb{}
new\PYGZus{}version=0.2.1dev0
\end{sphinxVerbatim}
\begin{enumerate}
\sphinxsetlistlabels{\arabic}{enumi}{enumii}{}{.}%
\item {} 
\sphinxAtStartPar
Push to git. \sphinxcode{\sphinxupquote{bump2version}} will produce git tags (imagine these are bookmarks in your repository). Not make sure these are updated in your repository you have to push with the \sphinxcode{\sphinxupquote{\sphinxhyphen{}\sphinxhyphen{}tags}} flag.

\end{enumerate}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{git} \PYG{n}{push} \PYG{o}{\PYGZhy{}}\PYG{o}{\PYGZhy{}}\PYG{n}{tags} \PYG{n}{origin} \PYG{n}{master}
\end{sphinxVerbatim}


\subsection{Building the documentation}
\label{\detokenize{package_management:building-the-documentation}}
\sphinxAtStartPar
Sphinx is used to document the package and the documentation is maintained as a set of HTML files and a combined PDF document. The HTML pages use the readthedocs \sphinxhref{https://sphinx-rtd-theme.readthedocs.io/en/stable/}{theme}. This can be installed with:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{pip} \PYG{n}{install} \PYG{n}{sphinx}\PYG{o}{\PYGZhy{}}\PYG{n}{rtd}\PYG{o}{\PYGZhy{}}\PYG{n}{theme}
\end{sphinxVerbatim}

\sphinxAtStartPar
There are several other packages that are required for building and handling markdown (\sphinxhref{https://myst-parser.readthedocs.io/en/latest/using/intro.html\#installation}{myst\sphinxhyphen{}parser})

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{pip} \PYG{n}{install} \PYG{n}{myst}\PYG{o}{\PYGZhy{}}\PYG{n}{parser}
\end{sphinxVerbatim}

\sphinxAtStartPar
Handling jupyter\sphinxhyphen{}notebook inclusion into the documentation (\sphinxhref{https://nbsphinx.readthedocs.io/en/0.8.0/}{nbsphinx})

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{pip} \PYG{n}{install} \PYG{n}{nbsphinx}
\end{sphinxVerbatim}

\sphinxAtStartPar
Associated with nbsphinx is \sphinxhref{https://pypi.org/project/nbsphinx-link/}{nbssphinx\sphinxhyphen{}link} which allows notebooks to be included in the documentation when they are not physically located inside the Sphinx build directory.

\sphinxAtStartPar
The documentation configuration and static HTML files are located in \sphinxcode{\sphinxupquote{./docs}}.

\sphinxAtStartPar
In order to build the PDF documentation, you will need \sphinxcode{\sphinxupquote{pdftex}}, this can be installed with, see \sphinxhref{https://www.sphinx-doc.org/en/1.8/usage/builders/index.html\#sphinx.builders.latex.LaTeXBuilder}{here}:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{sudo} \PYG{n}{apt}\PYG{o}{\PYGZhy{}}\PYG{n}{get} \PYG{n}{install} \PYG{n}{texlive}\PYG{o}{\PYGZhy{}}\PYG{n}{latex}\PYG{o}{\PYGZhy{}}\PYG{n}{recommended}
\PYG{n}{sudo} \PYG{n}{apt}\PYG{o}{\PYGZhy{}}\PYG{n}{get} \PYG{n}{install} \PYG{n}{texlive}\PYG{o}{\PYGZhy{}}\PYG{n}{fonts}\PYG{o}{\PYGZhy{}}\PYG{n}{recommended}
\PYG{n}{sudo} \PYG{n}{apt}\PYG{o}{\PYGZhy{}}\PYG{n}{get} \PYG{n}{install} \PYG{n}{texlive}\PYG{o}{\PYGZhy{}}\PYG{n}{latex}\PYG{o}{\PYGZhy{}}\PYG{n}{extra}
\PYG{n}{sudo} \PYG{n}{apt}\PYG{o}{\PYGZhy{}}\PYG{n}{get} \PYG{n}{install} \PYG{n}{latexmk}
\end{sphinxVerbatim}


\subsubsection{Initialisation}
\label{\detokenize{package_management:initialisation}}
\sphinxAtStartPar
This only needs to be done once to create the directory structure for Sphinx to work in. Change into the \sphinxcode{\sphinxupquote{./docs}} directory and run \sphinxcode{\sphinxupquote{sphinx\sphinxhyphen{}quickstart}} then answer the questions. Here, separate build and source directories were created.

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYGZdl{} sphinx\PYGZhy{}quickstart
Welcome to the Sphinx 3.4.0 quickstart utility.

Please enter values for the following settings (just press Enter to
accept a default value, if one is given in brackets).

Selected root path: .

You have two options for placing the build directory for Sphinx output.
Either, you use a directory \PYGZdq{}\PYGZus{}build\PYGZdq{} within the root path, or you separate
\PYGZdq{}source\PYGZdq{} and \PYGZdq{}build\PYGZdq{} directories within the root path.
\PYGZgt{} Separate source and build directories (y/n) [n]: y

The project name will occur in several places in the built documentation.
\PYGZgt{} Project name: hpc\PYGZhy{}tools
\PYGZgt{} Author name(s): Chris Finan
\PYGZgt{} Project release []: 0.1.0a1

If the documents are to be written in a language other than English,
you can select a language here by its language code. Sphinx will then
translate text that it generates into that language.

For a list of supported codes, see
https://www.sphinx\PYGZhy{}doc.org/en/master/usage/configuration.html\PYGZsh{}confval\PYGZhy{}language.
\PYGZgt{} Project language [en]: 

Creating file ./cluster/docs/source/conf.py.
Creating file ./cluster/docs/source/index.rst.
Creating file ./cluster/docs/Makefile.
Creating file ./cluster/docs/make.bat.

Finished: An initial directory structure has been created.

You should now populate your master file cluster/docs/source/index.rst and create other documentation
source files. Use the Makefile to build the docs, like so:
   make builder
where \PYGZdq{}builder\PYGZdq{} is one of the supported builders, e.g. html, latex or linkcheck.
\end{sphinxVerbatim}


\subsubsection{Configuration}
\label{\detokenize{package_management:configuration}}
\sphinxAtStartPar
Ensure that the Sphinx configuration file (\sphinxcode{\sphinxupquote{./docs/conf.py}}) is updated to load all the required extensions.

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{extensions} \PYG{o}{=} \PYG{p}{[}
    \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{sphinx\PYGZus{}rtd\PYGZus{}theme}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,}
    \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{myst\PYGZus{}parser}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,}
    \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{nbsphinx}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,}
    \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{nbsphinx\PYGZus{}link}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,}
    \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{sphinx.ext.autodoc}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,}
    \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{sphinx.ext.napoleon}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,}
    \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{sphinx.ext.todo}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,}
    \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{sphinx.ext.viewcode}\PYG{l+s+s1}{\PYGZsq{}}
\PYG{p}{]}
\end{sphinxVerbatim}

\sphinxAtStartPar
And the theme:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n}{html\PYGZus{}theme} \PYG{o}{=} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{sphinx\PYGZus{}rtd\PYGZus{}theme}\PYG{l+s+s1}{\PYGZsq{}}
\end{sphinxVerbatim}


\subsubsection{Building HTML}
\label{\detokenize{package_management:building-html}}
\sphinxAtStartPar
The HTML documentation can be built with:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{o}{.}\PYG{o}{/}\PYG{n}{docs}\PYG{o}{/}\PYG{n}{make} \PYG{n}{html}
\end{sphinxVerbatim}


\subsubsection{Building PDF}
\label{\detokenize{package_management:building-pdf}}
\sphinxAtStartPar
The PDF documentation can be built with:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{o}{.}\PYG{o}{/}\PYG{n}{docs}\PYG{o}{/}\PYG{n}{make} \PYG{n}{latexpdf}
\end{sphinxVerbatim}

\sphinxAtStartPar
The resulting pdf file is located at \sphinxcode{\sphinxupquote{./docs/build/latex/hpc\sphinxhyphen{}tools.pdf}}. This is copied to \sphinxcode{\sphinxupquote{./resources/pdf/hpc\sphinxhyphen{}tools.pdf}}.


\subsubsection{Helper script}
\label{\detokenize{package_management:helper-script}}
\sphinxAtStartPar
The build commands and copying of the PDF file are wrapped into a small helper script. This can be run by:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{o}{.}\PYG{o}{/}\PYG{n}{resources}\PYG{o}{/}\PYG{n}{build}\PYG{o}{/}\PYG{n}{build\PYGZus{}docs}\PYG{o}{.}\PYG{n}{sh}
\end{sphinxVerbatim}


\chapter{Indices and tables}
\label{\detokenize{index:indices-and-tables}}\begin{itemize}
\item {} 
\sphinxAtStartPar
\DUrole{xref,std,std-ref}{genindex}

\item {} 
\sphinxAtStartPar
\DUrole{xref,std,std-ref}{modindex}

\item {} 
\sphinxAtStartPar
\DUrole{xref,std,std-ref}{search}

\end{itemize}



\renewcommand{\indexname}{Index}
\printindex
\end{document}