#! /bin/bash
# -*- coding: utf-8 -*-
#
# oe_watchdog
# Odoo service watchdog
# Continously check for Odoo service running
#
# This free software is released under GNU Affero GPL3
# author: Antonio M. Vigliotti - antoniomaria.vigliotti@gmail.com
# (C) 2015-2017 by SHS-AV s.r.l. - http://www.shs-av.com - info@shs-av.com
#
READLINK=$(which greadlink 2>/dev/null) || READLINK=$(which readlink 2>/dev/null)
export READLINK
# Based on template 2.0.0
THIS=$(basename "$0")
TDIR=$(readlink -f $(dirname $0))
[ $BASH_VERSINFO -lt 4 ] && echo "This script $0 requires bash 4.0+!" && exit 4
if [[ -z $HOME_DEVEL || ! -d $HOME_DEVEL ]]; then
  [[ -d $HOME/odoo/devel ]] && HOME_DEVEL="$HOME/odoo/devel" || HOME_DEVEL="$HOME/devel"
fi
[[ -x $TDIR/../bin/python3 ]] && PYTHON=$(readlink -f $TDIR/../bin/python3) || [[ -x $TDIR/python3 ]] && PYTHON="$TDIR/python3" || PYTHON=$(which python3 2>/dev/null) || PYTHON="python"
[[ -z $PYPATH ]] && PYPATH=$(echo -e "import os,sys\no=os.path\na=o.abspath\nj=o.join\nd=o.dirname\nb=o.basename\nf=o.isfile\np=o.isdir\nC=a('"$TDIR"')\nD='"$HOME_DEVEL"'\nif not p(D) and '/devel/' in C:\n D=C\n while b(D)!='devel':  D=d(D)\nN='venv_tools'\nU='setup.py'\nO='tools'\nH=o.expanduser('~')\nT=j(d(D),O)\nR=j(d(D),'pypi') if b(D)==N else j(D,'pypi')\nW=D if b(D)==N else j(D,'venv')\nS='site-packages'\nX='scripts'\ndef pt(P):\n P=a(P)\n if b(P) in (X,'tests','travis','_travis'):\n  P=d(P)\n if b(P)==b(d(P)) and f(j(P,'..',U)):\n  P=d(d(P))\n elif b(d(C))==O and f(j(P,U)):\n  P=d(P)\n return P\ndef ik(P):\n return P.startswith((H,D,K,W)) and p(P) and p(j(P,X)) and f(j(P,'__init__.py')) and f(j(P,'__main__.py'))\ndef ak(L,P):\n if P not in L:\n  L.append(P)\nL=[C]\nK=pt(C)\nfor B in ('z0lib','zerobug','odoo_score','clodoo','travis_emulator'):\n for P in [C]+sys.path+os.environ['PATH'].split(':')+[W,R,T]:\n  P=pt(P)\n  if B==b(P) and ik(P):\n   ak(L,P)\n   break\n  elif ik(j(P,B,B)):\n   ak(L,j(P,B,B))\n   break\n  elif ik(j(P,B)):\n   ak(L,j(P,B))\n   break\n  elif ik(j(P,S,B)):\n   ak(L,j(P,S,B))\n   break\nak(L,os.getcwd())\nprint(' '.join(L))\n"|$PYTHON)
[[ $TRAVIS_DEBUG_MODE -ge 8 ]] && echo "PYPATH=$PYPATH"
for d in $PYPATH /etc; do
  if [[ -e $d/z0librc ]]; then
    . $d/z0librc
    Z0LIBDIR=$(readlink -e $d)
    break
  fi
done
[[ -z "$Z0LIBDIR" ]] && echo "Library file z0librc not found in <$PYPATH>!" && exit 72
[[ $TRAVIS_DEBUG_MODE -ge 8 ]] && echo "Z0LIBDIR=$Z0LIBDIR"
ODOOLIBDIR=$(findpkg odoorc "$PYPATH" "clodoo")
[[ -z "$ODOOLIBDIR" ]] && echo "Library file odoorc not found!" && exit 72
. $ODOOLIBDIR
[[ $TRAVIS_DEBUG_MODE -ge 8 ]] && echo "ODOOLIBDIR=$ODOOLIBDIR"

# DIST_CONF=$(findpkg ".z0tools.conf" "$PYPATH")
# TCONF="$HOME/.z0tools.conf"
CFG_init "ALL"
link_cfg_def
link_cfg $DIST_CONF $TCONF
[[ $TRAVIS_DEBUG_MODE -ge 8 ]] && echo "DIST_CONF=$DIST_CONF" && echo "TCONF=$TCONF"
get_pypi_param ALL
RED="\e[1;31m"
GREEN="\e[1;32m"
CLR="\e[0m"

__version__=2.0.15


detached_login () {
    if [ $opt_cron -gt 0 -a "$SHELL" == "/bin/sh" ]; then
      if [ $opt_dry_run -eq 0 ]; then
        fqcmd=$(readlink -f $0)
        SHELL=/bin/bash exec "$fqcmd" "$@"
      fi
    elif [ $opt_cron -gt 0 ]; then
      . ~/.bash_profile
    fi
    if [ $(echo ":$PATH:"|grep -v "/usr/local/bin" 2>/dev/null) ]; then
      export PATH=/usr/local/bin:$PATH
    fi
    if [ $(echo ":$PATH:"|grep -v "/dev/" 2>/dev/null) ]; then
      if [ -d ~/dev ]; then
        export PATH=$PATH:~/dev
      else
        export PATH=$PATH:/opt/odoo/dev
      fi
    fi
}


kill_daemon() {
# kill_daemon
    local opt_xmlport=$(build_odoo_param RPCPORT $CUR_ODOO_VER)
    local pid=$(netstat -atunp|grep "$port.*LISTEN"|awk '{print $7}'|awk -F/ '{print $1}')
    if [ -n "$pid" ]; then
      kill $pid
    fi
}


set_db_list () {
    local c f new_odoo_ver
    c=0
    while [ -z "$DBLIST" ]; do
      if [ $opt_multi -gt 0 ]; then
        f=0
        new_odoo_ver=
        for v in $ODOO_VID 6 7 8 9 10 11; do
          if [ "$v" == "$CUR_ODOO_VER" ]; then
            f=1
          elif [ $f -eq 1]; then
            new_odoo_ver=$v
            break
          fi
        done
        if [ -z "$new_odoo_ver" ]; then
          CUR_ODOO_VER="$ODOO_VID"
          if [ $DEBUG_MODE -ne 0 ]; then
            elog "\$ CUR_ODOO_VER=$CUR_ODOO_VER; (($c++))"
          fi
          ((c++))
        else
          CUR_ODOO_VER=$new_odoo_ver
          if [ $DEBUG_MODE -ne 0 ]; then
            elog "\$ CUR_ODOO_VER=$CUR_ODOO_VER"
          fi
        fi
        if [ $c -gt 1 ]; then
          TIME_SLEEP=300
          if [ $DEBUG_MODE -ne 0 ]; then
            elog "\$ TIME_SLEEP=$TIME_SLEEP"
            elog "\$ DBFILTER=\$(get_cfg_value "" DBFILTER)"
          fi
          DBFILTER=$(get_cfg_value "" "DBFILTER")
          if [ $DEBUG_MODE -ne 0 ]; then
            elog "\$ if [ -z \"\$DBFILTER=$DBFILTER\" ]; then DBFILTER=\"demo\"; fi"
          fi
          if [ -z "$DBFILTER" ]; then
            DBFILTER="demo"
          fi
        else
          if [ "$CUR_ODOO_VER" == "$ODOO_VID" ]; then
            DBFILTER=$(get_cfg_value "" "DBFILTER")
            if [ $DEBUG_MODE -ne 0 ]; then
              elog "\$ DBFILTER=\$(get_cfg_value "" DBFILTER)"
            fi
          else
            DBFILTER=$(get_cfg_value "" "DBFILTER_$CUR_ODOO_VER")
            if [ $DEBUG_MODE -ne 0 ]; then
              elog "\$ DBFILTER=\$(get_cfg_value "" DBFILTER_$CUR_ODOO_VER)"
            fi
          fi
          if [ -z "$DBFILTER" ]; then
            if [ $DEBUG_MODE -ne 0 ]; then
              elog "\$ if [ -z \"\$DBFILTER=$DBFILTER\" ]; then DBFILTER=\"demo\"; fi"
            fi
            if [ "$CUR_ODOO_VER" == "$ODOO_VID" ]; then
              DBFILTER="demo"
            else
              if [ $DEBUG_MODE -ne 0 ]; then
                elog "\$ if [ -z \"\$DBFILTER=$DBFILTER\" ]; then DBFILTER=\"demo$CUR_ODOO_VER\"; fi"
              fi
              DBFILTER="demo$CUR_ODOO_VER"
            fi
          fi
        fi
      fi
      if [ $DEBUG_MODE -ne 0 ]; then
        elog "Creating DB list ..."
      fi
      if [ $DEBUG_MODE -ne 0 ]; then
        elog "psql -tU$PG_USER -l|grep -E \"^[[:space:]]+$DBFILTER[[:space:]]+.[[:space:]]+$DB_USER[[:space:]]+\"|awk '{print \$1}'|tr '\n' ' '"
      fi
      DBLIST=$(psql -tU$PG_USER -l|grep -E "^[[:space:]]+$DBFILTER[[:space:]]+.[[:space:]]+$DB_USER[[:space:]]+"|awk '{print $1}'|tr '\n' ' ')
      DB_ix=0
      wlog "DBLIST=($DBLIST)"
    done
}

select_db () {
    set_db_list
    local c=0
    DBNAME=""
    for d in $DBLIST; do
      if [ $DEBUG_MODE -ne 0 ]; then
        elog "\$ [ $c -eq $DB_ix ] && ((DB_ix++)); DBNAME=\"$d\""
      fi
      if [ $c -eq $DB_ix ]; then
        ((DB_ix++))
        DBNAME="$d"
        break
      fi
      ((c++))
    done
}

test_db_svr_response () {
    local odoo_fver=$(build_odoo_param FULLVER $CUR_ODOO_VER)
    local opt_xmlport=$(build_odoo_param RPCPORT $CUR_ODOO_VER)
    if [ -n "$opt_DB" ]; then
      DBNAME="$opt_DB"
    else
      select_db
      if [ -z "$DBNAME" ]; then
        DBLIST=""
        select_db
      fi
      if [ -z "$DBNAME" ]; then
        DBNAME="demo"
        TIME_SLEEP=300
      fi
    fi
    wlog "\$ clodoo.py $opts -b$odoo_fver -d=$DBNAME -A=unit_test -r$opt_xmlport  # Check for DB $DBNAME"
    clodoo.py $opts -b$odoo_fver -d=$DBNAME -A=unit_test -r$opt_xmlport
    sts=$?
    if [ $sts -eq $STS_SUCCESS ]; then
      ((DBSTS[$DBNAME]++))
      pg_db_active -w &>/dev/null
      pg_db_active -az $DBNAME>>$LOGFILE
    else
      ((DBSTS[$DBNAME]--))
    fi
}


test_hung_up () {
    if [ $opt_multi -gt 0 ]; then
      ODOO_LOG=$(build_odoo_param FLOG $CUR_ODOO_VER "search")
    fi
    if [ $DEBUG_MODE -ne 0 ]; then
      elog "\$ HUNG_UP=\$(cat $ODOO_LOG|grep -n 'RuntimeError: maximum recursion depth exceeded'|tail -n1)"
    fi
    HUNG_UP=$(cat $ODOO_LOG|grep -n "RuntimeError: maximum recursion depth exceeded"|tail -n1)
    if [ -n "$HUNG_UP" ]; then
      if [ "$HUNG_UP" == "$LAST_HUP_DEEP" ]; then
        HUNG_UP=
      else
        LAST_HUP_DEEP="$HUNG_UP"
      fi
    fi
    if [ -z "$HUNG_UP" ]; then
      if [ $DEBUG_MODE -ne 0 ]; then
        elog "\$ HUNG_UP=\$(cat $ODOO_LOG|grep -n 'FATAL: *connection slots.*reserved for.*superuser'|tail -n1)"
      fi
      HUNG_UP=$(cat $ODOO_LOG|grep -n "FATAL: *connection slots.*reserved for.*superuser"|tail -n1)
      if [ -n "$HUNG_UP" ]; then
        if [ "$HUNG_UP" == "$LAST_HUP_SLOT" ]; then
          HUNG_UP=
        else
          LAST_HUP_SLOT="$HUNG_UP"
        fi
      fi
      if [ -n "$HUNG_UP" ]; then
        wlog "Kill all sessions out of pool!"
        pg_db_active -k>>$LOGFILE
      fi
    fi
}

conf_default () {
    CFG_set "DEV_HOST" "shsdev16"
    CFG_set "PRD_HOST" "shsprd14"
    # CFG_set "ODOO_SETUP" "__openerp__.py"
    CFG_set "SVC_NAME" ""
    CFG_set "login_user" "zeroadm"
    CFG_set "login_password" "Wg\"0JK!P"
    CFG_set "ODOO_VID" "v7"
    CFG_set "CLODIR" "/opt/odoo/clodoo"
    CFG_set "CLOBIN" "/opt/odoo/tools/clodoo"
    CFG_set "ODOO_CONF" ""
    CFG_set "ODOO_LOG" ""
    CFG_set "MAX_RETRY" "3"
    CFG_set "TIME_SLEEP" "90"
    CFG_set "TIME_WAIT" "10"
    CFG_set "HELLO_CTR" "21"
    CFG_set "HUPTST_CTR" "3"
    CFG_set "HUPTST_TAIL" "0"
    CFG_set "DEBUG_MODE" "0"
    CFG_set "DBFILTER" ".*"
    CFG_set "PG_USER" "postgres"
    CFG_set "DB_USER" ""
    CFG_set "ODOO_MULTI_VERSION" "0"
    TCONF=/etc/odoo/oe_watchdog.conf
    if [ ! -f $TCONF -a ! -f $TCONF.sample ]; then
      TCONF=/etc/oe_watchdog.conf
    fi
    if [ ! -f $TCONF -a ! -f $TCONF.sample ]; then
      if [ -d /etc/odoo ]; then
        TCONF=/etc/odoo/oe_watchdog.conf
      else
        TCONF=/etc/oe_watchdog.conf
      fi
    fi
}


OPTOPTS=(h        D      K        n            s            V           v           X)
OPTDEST=(opt_help opt_DB opt_cron opt_dry_run  opt_svc_name opt_version opt_verbose opt_lxs)
OPTACTI=("+"      "=>"   "1>"     "1>"         "=>"         "*>"        1           "1>")
OPTDEFL=(1        ""     0        0            ""           ""          0           0)
OPTMETA=("help"   "db"   "cron"   "do nothing" "svc_name"   "version"   "verbose"   "")
OPTHELP=("this help"\
 "DB to test"\
 "run in cron environment"\
 "do nothing (dry-run)"\
 "service name"\
 "show version"\
 "verbose mode"\
 "run continuosly")
OPTARGS=()

parseoptargs "$@"
if [ "$opt_version" ]
then
  echo "$__version__"
  exit 0
fi
if [ $opt_help -gt 0 ]
then
  print_help "Odoo watchdog"\
  "(C) 2015-2017 by zeroincombenze®\nhttp://wiki.zeroincombenze.org/en/Odoo\nAuthor: antoniomaria.vigliotti@gmail.com"
  exit 0
fi
PIDFILE=/var/run/odoo/oe_watchdog.pid
LOGFILE=/var/log/odoo/oe_watchdog.log
if [[ -t 0 || -p /dev/stdin ]]; then
   LECHO=echo
else
   LECHO=
fi
set_tlog_file "$LOGFILE" "" "$LECHO"
if [ $opt_cron -gt 0 ]; then
  chmod +r $LOGFILE
  wlog "Run cron mode"
  detached_login "$@"
  wlog "$PATH"
fi
if [ $opt_lxs -gt 0 ]; then
  chmod +r $LOGFILE
  echo $$ > $PIDFILE
  wlog "********** Odoo server watchdog $__version__ starting (pid=$$)... **********"
fi

CFG_init
conf_default
link_cfg $TCONF
sts=$STS_SUCCESS

ODOO_VID=$(get_cfg_value "" "ODOO_VID")
CUR_ODOO_VER=$ODOO_VID
opt_multi=$(get_cfg_value "" "ODOO_MULTI_VERSION")
if [ -n "$opt_svc_name" ]; then
  SVC_NAME=$opt_svc_name
else
  SVC_NAME=$(get_cfg_value "" "SVC_NAME")
  if [ -z "$SVC_NAME" ]; then
    SVC_NAME=$(build_odoo_param SVCNAME $CUR_ODOO_VER)
  fi
fi
CLODIR=$(get_cfg_value "" "CLODIR")
CLOBIN=$(get_cfg_value "" "CLOBIN")
ODOO_CONF=$(get_cfg_value "" "ODOO_CONF")
if [ -z "$ODOO_CONF" ]; then
  ODOO_CONF=$(build_odoo_param CONFN "$CUR_ODOO_VER" "search")
fi
ODOO_LOG=$(get_cfg_value "" "ODOO_LOG")
if [ -z "$ODOO_LOG" ]; then
  ODOO_LOG=$(build_odoo_param FLOG "$CUR_ODOO_VER" "search")
fi
DBFILTER=$(get_cfg_value "" "DBFILTER")
PG_USER=$(get_cfg_value "" "PG_USER")
DB_USER=$(get_cfg_value "" "DB_USER")
if [ -z "$DB_USER" ]; then
  DB_USER=$(build_odoo_param USER "$CUR_ODOO_VER")
fi
MAX_RETRY=$(get_cfg_value "" "MAX_RETRY")
if [ $MAX_RETRY -gt 5 -o $MAX_RETRY -lt 1 ]; then
  MAX_RETRY=3
fi
TIME_SLEEP=$(get_cfg_value "" "TIME_SLEEP")
if [ $TIME_SLEEP -gt 300 -o $TIME_SLEEP -lt 30 ]; then
  TIME_SLEEP=90
fi
TIME_WAIT=$(get_cfg_value "" "TIME_WAIT")
if [ $TIME_WAIT -gt 15 -o $TIME_WAIT -lt 5 ]; then
  TIME_WAIT=10
fi
let TIME_SWAIT="$TIME_WAIT/2"
HELLO_CTR=$(get_cfg_value "" "HELLO_CTR")
if [ $HELLO_CTR -gt 30 -o $HELLO_CTR -lt 3 ]; then
  TIME_SLEEP=21
fi
HUPTST_CTR=$(get_cfg_value "" "HUPTST_CTR")
if [ $HUPTST_CTR -gt 30 -o $HUPTST_CTR -lt 3 ]; then
  HUPTST_CTR=7
fi
HUPTST_TAIL=$(get_cfg_value "" "HUPTST_TAIL")
if [ $HUPTST_TAIL -gt 300 -o $HUPTST_TAIL -lt 75 ]; then
  HUPTST_TAIL=$TIME_SLEEP
  if [ $HUPTST_TAIL -lt 75 ]; then
    HUPTST_TAIL=75
  fi
fi
DEBUG_MODE=$(get_cfg_value "" "DEBUG_MODE")
if [ $DEBUG_MODE -ne 0 ]; then
  set_tlog_file "$LOGFILE" "" "echo"
fi
WCMD=$(which pg_db_active 2>/dev/null)
if [ -z "$WCMD" ]; then
  wlog "Command pg_db_active not found!"
  exit 1
fi
if [ ! -d $CLODIR ]; then
  wlog "Directory $CLODIR not found!"
  exit 1
fi
clodir=$CLODIR/oe_watchdog
if [ ! -d $clodir ]; then
  mkdir $clodir
fi
cd $clodir
login_user=$(get_cfg_value "" "login_user")
login_password=$(get_cfg_value "" "login_password")
if [ ! -f clodoo.conf ]; then
  cat <<EOF >clodoo.conf
[options]
actions=unit_test
login_user=$login_user
login_password=$login_password
EOF
else
  sed -i -e "s|^dbfilter *=.*|dbfilter=$DBFILTER|" clodoo.conf
  sed -i -e "s|^actions *=.*|actions=unit_test|" clodoo.conf
fi
if [ $opt_verbose -gt -0 ]; then
   opts=-v
else
   opts=-q
fi
wlog "login_user=$login_user"
wlog "TCONF=$TCONF"
wlog "CLODIR=$clodir"
wlog "CLOBIN=$CLOBIN"
wlog "SVC_NAME=$SVC_NAME"
wlog "ODOO_VID=$ODOO_VID"
wlog "ODOO_CONF=$ODOO_CONF"
wlog "ODOO_LOG=$ODOO_LOG"
wlog "MAX_RETRY=$MAX_RETRY"
wlog "TIME_SLEEP=$TIME_SLEEP"
wlog "TIME_WAIT=$TIME_WAIT"
wlog "HELLO_CTR=$HELLO_CTR"
wlog "HUPTST_CTR=$HUPTST_CTR"
wlog "DBFILTER=$DBFILTER"
wlog "PG_USER=$PG_USER"
wlog "DB_USER=$DB_USER"
wlog "ODOO_MULTI_VERSION=$opt_multi"

DBLIST=""
declare -A DBSTS
if [ $opt_lxs -gt 0 ]; then
  ctr=5256000
else
  ctr=1
fi
helloctr=0
huptstctr=0
OK_ctr=0
FAIL_ctr=0
TOT_OK_ctr=0
TOT_FAIL_ctr=0
DB_ix=0
PATH=$CLOBIN:$PATH
LAST_HUP_DEEP=
LAST_HUP_SLOT=
while [ $ctr -gt 0 ]; do
  if [ $DEBUG_MODE -ne 0 ]; then
     pg_db_active -k -v >>$LOGFILE
  else
     pg_db_active -k >>$LOGFILE
  fi
  test_db_svr_response
  HUNG_UP=
  if [ $sts -ne 0 ]; then
    wlog "Warning: invalid response"
    test_hung_up
  elif [ $huptstctr -eq 0 ]; then
    test_hung_up
    huptstctr=$HUPTST_CTR
  else
    ((huptstctr--))
  fi
  if [ -n "$HUNG_UP" ]; then
    ctr2=0
    huptstctr=$HUPTST_CTR
    sts=1
    wlog "Warning: hung-up detected"
  else
    ctr2=$MAX_RETRY
  fi
  while [ $sts -ne 0 -a $ctr2 -gt 0 ]; do
    ((ctr2--))
    sleep $TIME_SWAIT
    test_db_svr_response
  done
  if [ $sts -ne 0 ]; then
    OK_ctr=0
    ((FAIL_ctr++))
    ((TOT_FAIL_ctr++))
    if [ $TOT_OK_ctr -gt $TOT_FAIL_ctr -a ${DBSTS[$DBNAME]:-0} -gt 0 ]; then
      wlog "\$ service $svc_name restart"
      service $svc_name restart
      wlog "Hung-up message: $HUNG_UP"
      sleep $TIME_WAIT
    else
      FAIL_ctr=0
    fi
  else
    ((OK_ctr++))
    FAIL_ctr=0
    ((TOT_OK_ctr++))
  fi
  if [ $FAIL_ctr -ge $MAX_RETRY ]; then
    wlog "???? SERVICE FAILURE ????"
    wlog "\$ service $svc_name stop"
    service $svc_name stop
    sleep $TIME_WAIT
    kill_daemon
    sleep $TIME_SWAIT
    wlog "\$ service $svc_name start"
    service $svc_name start
    sleep $TIME_WAIT
  fi
  ((ctr--))
  if [ $ctr -gt 0 ]; then
    if [ $helloctr -eq 0 ]; then
      helloctr=$HELLO_CTR
      wlog "Odoo server watchdog $__version__ is running (pid=$$)"
      c=0
      for d in ${!DBSTS[*]};do
        ((c++))
        if [ ${DBSTS[$d]} -gt 0 ]; then
          dbsts="active"
        elif [ ${DBSTS[$d]} -eq 0 ]; then
          dbsts="idle"
        else
          dbsts="OFF"
        fi
        wlog " $c) DB $d: $dbsts (${DBSTS[$d]})"
      done
    else
      ((helloctr--))
    fi
    sleep $TIME_SLEEP
  fi
done
exit $sts