*********************************************************************************
Log in to a machine with at least 16 Gb RAM and get into ska environment
*********************************************************************************

  ssh kadi
  cd ~/git/eng_archive
  ska
  unset ENG_ARCHIVE

Content types with just one fileset (no distinct 1999 dir):

  dp_eps16, dp_eps8, hrc0hk, hrc0ss, simdiag, ccdm14eng, ccdm15eng

First replicate the data structure and existing files that are required.

  cd /proj/sot/ska/data/eng_archive
  rsync -av --exclude=arch/ --exclude='*.h5' --exclude=archfiles.db3 --exclude=NEW --exclude=OLD 1999/ /data/aca/tmp/eng_archive/1999/
  rsync -av --exclude=arch/ --exclude='*.h5' --exclude=archfiles.db3 --exclude=NEW --exclude=OLD data/ /data/aca/tmp/eng_archive/data/

  cd ~/git/eng_archive
  ln -s /data/aca/tmp/eng_archive/data ./
  ln -s /data/aca/tmp/eng_archive/1999 ./

Now do the computations::

  ./update_archive.py --no-full --state-codes-only --date-now=2000:005 --max-lookback-time=180 --data-root=1999 >& run1.log &
  ./update_archive.py --no-full --state-codes-only --date-now=2000:010 --max-lookback-time=15 --data-root=. >& run2.log &
  ./update_archive.py --no-full --state-codes-only --date-start=2000:020 --max-lookback-time=180 --data-root=. >& run3.log &

  ./update_archive.py --no-full --state-codes-only --date-now=2000:005 --max-lookback-time=180 --data-root=1999 >& run10.log &
  ./update_archive.py --no-full --state-codes-only --date-now=2000:010 --max-lookback-time=15 --data-root=. >& run20.log &
  ./update_archive.py --no-full --state-codes-only --date-start=2000:020 --max-lookback-time=180 --data-root=. >& run30.log &

Bring local archive up to date (after taking a break from this)::

  ./update_archive.py --no-full --state-codes-only --max-lookback-time=200 --data-root=. >& run50.log &

Install
=======

HEAD::

  # First make list of state code MSIDs that are getting updated
  cd ~/git/eng_archive
  /bin/ls -1 data/*/{5min,daily}/*.h5 > state_msids
  /bin/ls -1 1999/data/*/{5min,daily}/*.h5 >> state_msids

  # Make backup directory
  tmproot=/data/aca/tmp/eng_archive
  mkdir -p $tmproot/stats_bin_backup

  # Make backup copy of relevant 5min/daily MSIDs
  rsync -v --files-from=state_msids \
     /proj/sot/ska/data/eng_archive/ \
     $tmproot/stats_bin_backup/

  # Practice and check
  head -1 state_msids > state_msids_one

  # just the first entry
  rsync --dry-run -v --files-from=state_msids_one ./ /proj/sot/ska/data/eng_archive/

  # Do it
  rsync --dry-run -v --files-from=state_msids ./ /proj/sot/ska/data/eng_archive/

- Install version 0.41 of Ska.engarchive to Ska flight (or test on GRETA).

CONTINGENCY::

  # Restore backup copy of relevant 5min/daily MSIDs
  rsync -v --files-from=state_msids \
     $tmproot/stats_bin_backup/ \
     /proj/sot/ska/data/eng_archive/


GRETA::

  set tmproot=/proj/sot/ska/tmp/eng_archive

  # Copy over new stats files for state-values MSIDs
  cd /home/SOT/git/eng_archive
  rm -rf data 1999
  mkdir data
  mkdir 1999
  rsync -av aldcroft@ccosmos:/home/aldcroft/git/eng_archive/data/ data/
  rsync -av aldcroft@ccosmos:/home/aldcroft/git/eng_archive/1999/ 1999/

Now follow the HEAD instructions, but remembering to instal to Ska test.
