#!/bin/bash
# This script dumps latest month readings from every TESS given in an instrument list file.

# ------------------------------------------------------------------------------
#                             AUXILIARY FUNCTIONS
# ------------------------------------------------------------------------------

# --------
# Commands
# --------

NICE=$(which nice)
IONICE=$(which ionice)
RSYNC=$(which rsync)
TEE=$(which tee)
DATE=$(which date)
SQLITE=$(which sqlite3)
RCLONE=$(which rclone)
RCLONE_COMMAND=sync
SSH=$(which ssh)
TESS_IDA=/usr/local/bin/tess-ida

# --------------
# Default values
# --------------

# For download part
DEFAULT_SRC_USER="pi"
DEFAULT_SRC_HOST="cortex.fis.ucm.es"
DEFAULT_SRC_DIR="/var/dbase"
DEFAULT_DST_DIR="/var/dbase"

# For IDA file generation
DEFAULT_DATABASE="/var/dbase/tess.db"
DEFAULT_IDA_DIR="/var/dbase/reports/IDA"
DEFAULT_MONTH="latest"
DEFAULT_LOG_FILE=/var/log/ida_daily.log

# For IDA selective regeneration
DEFAULT_REGEN_DIR=/var/dbase/regen

# For WebDAV synchronization to NextCloud
DEFAULT_WEB_DIR="IDA"
# WebDAV Server is identified by a tag in .config/rclone/rclone.conf
DEFAULT_WEBDAV_TAG="IDA"
DEFAULT_WEBDAV_FLAGS="--verbose"

DEFAULT_PIPE_STAGE=all

# ------------------
# Auxilary functions
# ------------------

query_names() {
    dbase=$1
    ${SQLITE} ${dbase} <<EOF
SELECT name 
FROM tess_v 
WHERE name like 'stars%' 
AND valid_state = 'Current' 
ORDER by CAST(substr(name, 6) as decimal) ASC;
EOF
}

log() {
    local level=$1
    local message=$2
    local tstamp=$(date '+%Y-%m-%d %H:%M:%S.%3N')
    printf "%s [%-8s] %s\n" "${tstamp}" "${level}" "${message}"
    #echo "${tstamp} [${level}] ${message}"
}

check_database() {
    local dbase=$1
    local log_file=$2
    if  [[ ! -f ${dbase} || ! -r ${dbase} ]]; then
        log ERROR "Database file ${dbase} does not exists or is not readable."  2>&1 | ${TEE} -a ${log_file}
        log ERROR "Exiting" 2>&1 | ${TEE} -a ${log_file}
        exit 1
    fi
}

help() {
    local name=$(basename ${0%.sh})
    echo "Usage:"
    echo "$name -p <pipeline stage> -u <remote user> -s <source host> -d <source dbase dir> -o<dst dbase dir> -m <month> -i <IDA dir> -w <Web dir> -l <log file path prefix>"
    echo "Defaults to:"
    echo "$name -p $DEFAULT_PIPE_STAGE -u $DEFAULT_SRC_USER -s $DEFAULT_SRC_HOST -i $DEFAULT_SRC_DIR -o $DEFAULT_DST_DIR -m $DEFAULT_MONTH -i $DEFAULT_IDA_DIR -w $DEFAULT_WEB_DIR \
 -l $DEFAULT_LOG_FILE"
}


wait_for_backup() {
    local date=$1
    local src_user=$2
    local src_host=$3
    local src_dir=$4
    local log_file=$5
    local yesterday=$(/usr/bin/date +%Y%m%d -d "yesterday")
    log INFO "Wait for today's backup ${date}" 2>&1 | ${TEE} -a ${log_file}
    for i in {1..60}; do
        log INFO "[$i/60] Waiting 60 seconds for tess.db-${yesterday} to be removed" 2>&1 | ${TEE} -a ${log_file}
        ${SSH} ${src_user}@${src_host} test -f ${src_dir}/tess.db-${yesterday}
        local exit_code=$?
        if [ "${exit_code}" -ne 0 ]; then
            break
        fi
        sleep 60
    done
    if [[ "$i" -ge 60 ]]; then
        log ERROR "No tess.db-${date} was available today" 2>&1 | ${TEE} -a ${log_file}
        log CRITICAL "Aborting the whole IDA pipe line generation today" 2>&1 | ${TEE} -a ${log_file}
        exit 1
    fi
    sleep 5
    log INFO "Database backup tess.db-${date} available" 2>&1 | ${TEE} -a ${log_file}
}

remote_backup() {
    local date=$1
    local src_user=$2
    local src_host=$3
    local src_dir=$4
    local dst_dir=$5
    local log_file=$6
    log INFO "getting remote file by rsync" | ${TEE} -a ${log_file}
    log DEBUG "${NICE} -n 19 ${IONICE} -c3 ${RSYNC} -zavh ${src_user}@${src_host}:${src_dir}/tess.db-${date} ${dst_dir}/tess.db" 2>&1 | ${TEE} -a ${log_file}
    ${NICE} -n 19 ${IONICE} -c3 ${RSYNC} -zavh ${src_user}@${src_host}:${src_dir}/tess.db-${date} ${dst_dir}/tess.db 2>&1 | ${TEE} -a ${log_file}
}

do_ida_from_month() {
    local instrument=$1
    local from_month=$2
    local regen_dir=$3
    local dbase=$4
    local out_dir=$5
    local log_file=$6
    if [[ -z "$from_month" ]]; then
        from_month=${DEFAULT_START_DATE}
    fi
    log INFO "Selective IDA file for TESS $instrument from ${from_month} under ${out_dir}/${instrument}" | ${TEE} -a ${log_file}
    ${TESS_IDA} ${instrument} --log-file ${log_file} --from-month ${from_month} -d ${dbase} -o ${out_dir}
   
    if  [[ -f ${regen_dir}/processed/${instrument} ]]; then
        log WARNING "Moving previous input control file to ${regen_dir}/processed/${instrument}.${DATE}" | ${TEE} -a ${log_file}
        mv ${regen_dir}/processed/${instrument} ${regen_dir}/processed/${instrument}.${DATE}
    fi
    log INFO "Moving control file ${regen_dir}/input/${instrument} => ${regen_dir}/processed/${instrument}" | ${TEE} -a ${log_file}
    test -f ${regen_dir}/input/${instrument} && mv ${regen_dir}/input/${instrument} ${regen_dir}/processed/${instrument}
}

do_ida_for_month() {
    local instrument=$1
    local for_month=$2
    local regen_dir=$3
    local dbase=$4
    local out_dir=$5
    local log_file=$6
    if [[ -z "$for_month" ]]; then
        for_month=${DEFAULT_START_DATE}
    fi
    log INFO "Selective IDA file for TESS $instrument for ${for_month} under ${out_dir}/${instrument}" | ${TEE} -a ${log_file}
    ${TESS_IDA} ${instrument} --log-file ${log_file} --for-month ${for_month} -d ${dbase} -o ${out_dir}
    if  [[ -f ${regen_dir}/processed/${instrument} ]]; then
        log WARNING "Moving previous input control file to ${regen_dir}/processed/${instrument}.${DATE}" | ${TEE} -a ${log_file}
        mv ${regen_dir}/processed/${instrument} ${regen_dir}/processed/${instrument}.${DATE}
    fi
    log INFO "Moving control file ${regen_dir}/input/${instrument} => ${regen_dir}/processed/${instrument}" | ${TEE} -a ${log_file}
    mv ${regen_dir}/input/${instrument} ${regen_dir}/processed/${instrument}
}


ida_latest() {
    local month=$1
    local dbase=$2
    local out_dir=$3
    local log_file=$4
    check_database $dbase $log_file
    local photometers=$(query_names ${dbase})
    # Loops over the instruments file and dumping data
    for instrument in ${photometers}; do
        if [[ "${month}" = "latest" ]]; then
            log INFO "Generating latest month IDA file for TESS ${instrument} under ${out_dir}/${instrument}" | ${TEE} -a ${log_file}
            ${TESS_IDA} ${instrument} --log-file ${log_file} --latest-month -d ${dbase} -o ${out_dir} 
        else
            log INFO "Generating previous month IDA file for TESS ${instrument} under ${out_dir}/${instrument}" | ${TEE} -a ${log_file}
            ${TESS_IDA} ${instrument} --log-file ${log_file} --previous-month -d ${dbase} -o ${out_dir}
        fi
    done
}


ida_from_month() {
    local regen_dir=$1/from_month
    local dbase=$2
    local ida_dir=$3
    local log_file=$4
    check_database $dbase $log_file
    if  [[ ! -d "${regen_dir}/input" ]]; then
            log INFO "making IDA regeneration files dir structure under ${regen_dir}" | ${TEE} -a ${log_file}
            mkdir -p "${regen_dir}/input"
            mkdir -p "${regen_dir}/processed"
    fi
    # Get selective photometers list
    local photometers=$(ls -1 ${regen_dir}/input)
    for instrument in $photometers; do
        from_month=$(cat ${regen_dir}/input/${instrument})
        if [[ "$instrument" = "all" ]]; then
            local all_photometers=$(query_names ${dbase})
            for instrument2 in $all_photometers; do
                do_ida_from_month $instrument2 $from_month $regen_dir $dbase $ida_dir $log_file
            done
            log INFO "Moving control file ${regen_dir}/input/${instrument} => ${regen_dir}/processed/${instrument}" | ${TEE} -a ${log_file}
            mv ${regen_dir}/input/${instrument} ${regen_dir}/processed/${instrument}
        else
            do_ida_from_month $instrument $from_month $regen_dir $dbase $ida_dir $log_file
        fi
    done
}

ida_for_month() {
    local regen_dir=$1/for_month
    local dbase=$2
    local ida_dir=$3
    local log_file=$4
    check_database $dbase $log_file
    if  [[ ! -d "${regen_dir}/input" ]]; then
            log INFO "making IDA regeneration files dir structure under ${regen_dir}" | ${TEE} -a ${log_file}
            mkdir -p "${regen_dir}/input"
            mkdir -p "${regen_dir}/processed"
    fi
    # Get selective photometers list
    local photometers=$(ls -1 ${regen_dir}/input)
    for instrument in $photometers; do
        for_month=$(cat ${regen_dir}/input/${instrument})
        if [[ "$instrument" = "all" ]]; then
            local all_photometers=$(query_names ${dbase})
            for instrument2 in $all_photometers; do
                do_ida_for_month $instrument2 $for_month $regen_dir $dbase $ida_dir $log_file
            done
            log INFO "Moving control file ${regen_dir}/input/${instrument} => ${regen_dir}/processed/${instrument}" | ${TEE} -a ${log_file}
            mv ${regen_dir}/input/${instrument} ${regen_dir}/processed/${instrument}
        else
            do_ida_for_month $instrument $for_month $regen_dir $dbase $ida_dir $log_file
        fi
    done
}

sync_nextcloud() {
    local ida_dir=$1
    local web_dir=$2
    local log_file=$3
    log INFO "Syncing to NextCloud server from source dir ${ida_dir}" | ${TEE} -a ${log_file}
    log DEBUG "${NICE} -n 19 ${IONICE} -c3 ${RCLONE} ${RCLONE_COMMAND} ${ida_dir} ${DEFAULT_WEBDAV_TAG}:${web_dir} ${DEFAULT_WEBDAV_FLAGS}"  2>&1 | ${TEE} -a ${log_file}
    ${NICE} -n 19 ${IONICE} -c3 ${RCLONE} ${RCLONE_COMMAND} ${ida_dir} ${DEFAULT_WEBDAV_TAG}:${web_dir} ${DEFAULT_WEBDAV_FLAGS}  2>&1 | ${TEE} -a ${log_file}
}

# ------------------------------------------------------------------------------- #


# Arguments from the command line & default values

src_user="${DEFAULT_SRC_USER}"
src_dbase_host="${DEFAULT_SRC_HOST}"
src_dbase_dir="${DEFAULT_SRC_DIR}"
dst_dbase_dir="${DEFAULT_DST_DIR}"
dbase="${dst_dir}/tess.db"
month="${DEFAULT_MONTH}"
ida_dir=${DEFAULT_IDA_DIR}
web_dir=${DEFAULT_WEB_DIR}
log_file="${DEFAULT_LOG_FILE}"
step="${DEFAULT_PIPE_STAGE}"
date=$(/usr/bin/date +%Y%m%d)

while getopts ":hu:s:d:o:i:m:w:l:p:r:" opt; do
    case ${opt} in
    u)
        src_user="${OPTARG}"
        ;;
    s)
        src_dbase_host="${OPTARG}"
        ;;
    d)
        src_dbase_dir="${OPTARG}"
        ;;
    o)
        dst_dbase_dir="${OPTARG}"
        dbase="${dst_dbase_dir}/tess.db"
        ;;
    i)
        ida_dir="${OPTARG}"
        ;;
    m)
        month="${OPTARG}"
        ;;
    w)
        web_dir="${OPTARG}"
        ;;
    l)
        log_file="${OPTARG}"
        ;;
    p)
        step="${OPTARG}"
        ;;
    r)
        regen_dir="${OPTARG}"
        ;;
    h)
        help
        exit 0
        ;;
    :)
        echo "Option -${OPTARG} requires an argument."
        exit 1
        ;;
    ?)
        echo "Invalid option: -${OPTARG}."
        exit 1
        ;;
  esac
done
shift "$((OPTIND-1))"

# Execute Pipeline

log INFO "Start executing pipeline"
if [[ "${step}" == "all" ]]; then   
    wait_for_backup ${date} ${src_user} ${src_dbase_host} ${src_dbase_dir} ${log_file}
    remote_backup ${date} ${src_user} ${src_dbase_host} ${src_dbase_dir} ${dst_dbase_dir} ${log_file}
    ida_latest ${month} ${dbase} ${ida_dir} ${log_file}
    ida_from_month ${regen_dir} ${dbase} ${ida_dir} ${log_file}
    ida_for_month ${regen_dir} ${dbase} ${ida_dir} ${log_file}
    sync_nextcloud ${ida_dir} ${web_dir}
elif [[ "${step}" == "1" ]]; then
    wait_for_backup ${date} ${src_user} ${src_dbase_host} ${src_dbase_dir} ${log_file}
elif [[ "${step}" == "2" ]]; then
     remote_backup ${date} ${src_user} ${src_dbase_host} ${src_dbase_dir} ${dst_dbase_dir} ${log_file}
elif [[ "${step}" == "3" ]]; then
    ida_from_month ${regen_dir} ${dbase} ${ida_dir} ${log_file}
    ida_for_month ${regen_dir} ${dbase} ${ida_dir} ${log_file}
elif [[ "${step}" == "4" ]]; then
    ida_latest ${month} ${dbase} ${ida_dir} ${log_file}
    ida_from_month ${regen_dir} ${dbase} ${ida_dir} ${log_file}
    ida_for_month ${regen_dir} ${dbase} ${ida_dir} ${log_file}
elif [[ "${step}" == "5" ]]; then
    sync_nextcloud ${ida_dir} ${web_dir}
elif [[ "${step}" == "6" ]]; then
    remote_backup ${date} ${src_user} ${src_dbase_host} ${src_dbase_dir} ${dst_dbase_dir} ${log_file}
    ida_latest ${month} ${dbase} ${ida_dir} ${log_file}
    ida_from_month ${regen_dir} ${dbase} ${ida_dir} ${log_file}
    ida_for_month ${regen_dir} ${dbase} ${ida_dir} ${log_file}
    sync_nextcloud ${ida_dir} ${web_dir}
else
    log ERROR "Unsupported pipeline step ${step}"
fi
