####################################################
## Develop and test
####################################################

git checkout -b derived

- Update Ska/engarchive/filetypes.dat
- Update Ska/engarchive/version.py (inc. version, add "dev" e.g. "0.13dev"
- Update Ska/engarchive/derived/__init__.py
- Update Ska/engarchive/derived/<new>.py

# In top of dev directory (for bash)

content=<new>
mkdir -p /data/cosmos2/eng_archive/derived/data
ln -s /data/cosmos2/eng_archive/derived/data ./

# Make $PWD the default root in fetch for read but also allow fall-through to
# $ska.  This is needed so that update_archive can read the needed data to
# compute the derived.  ENG_ARCHIVE only affects fetch for reading so it is
# safe to do this.  The --data-root value controls the write dir.

export ENG_ARCHIVE=${PWD}:/proj/sot/ska/data/eng_archive

# Add_derived.py checks for any new derived content types or columns.  Note
# that you cannot just add a new column to an existing content type because the
# ingest process (update_archive.py) does not currently handle this.
# This reads data from 1999:200 to 1999:260

./add_derived.py --data-root=$PWD --content=dp_$content

# Update the archive, first in a small test.  --content takes a match regex.

./update_archive.py --data-root=$PWD --content=dp_$content --date-now 1999:260
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 1999:270
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 1999:280

ipython --pylab
impska

msid = 'newmsid'
figure()
subplot(3, 1, 1)
dat = fetch.Msid(msid, '1999:200')
dat.plot()

subplot(3, 1, 2)
dat5 = fetch.Msid(msid, '1999:200', stat='5min')
dat5.plot()

subplot(3, 1, 3)
datd = fetch.Msid(msid, '1999:200', stat='daily')
datd.plot()

# Intermediate commits
git commit -a
git push origin derived

####################################################
## Full ingest / update
####################################################

# CAREFULLY do this (don't remove everything, just the new bit)
set content=<new>
rm -rf data/dp_${content}*   

./add_derived.py --data-root=$PWD

For a quantity sampled at 32.8 sec or slower, this should work:
# First a couple of short ingests to seed the stats
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2000:010 --max-lookback-time=400 
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2000:020 --max-lookback-time=400 
# Now go
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2001:001 --max-lookback-time=400 >&! run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2002:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2003:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2004:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2005:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2006:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2007:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2008:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2009:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2010:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2011:001 --max-lookback-time=400 >>& run.log
./update_archive.py --data-root=$PWD --content=dp_$content --date-now 2012:001 --max-lookback-time=400 >>& run.log

Otherwise see run_add_derived.csh.

####################################################
## Update version and docs
####################################################

# Remove the "dev" from version string in version.py

# Update the derived parameters section of doc/pseudo_msids.rst
# and build docs locally.

####################################################
## Commit
####################################################

# commit new release:
  - filetypes
  - <new> derived classes
  - setup.py version

git commit -a
git push origin derived:derived

# merge feature into master
git checkout master
git merge derived --no-ff
git push

####################################################
## Install
####################################################

# Think about permissions here, depending on who is running update_archive.

# check what is in data/ and be sure that should be installed
rsync [--dry-run] -av data/dp_* /proj/sot/ska/data/eng_archive/data/

# From mac terminal window
cd ~/tmp
scp ccosmos:/data/baffin/tom/git/eng_archive/data/dp_* ./
scp dp_* SOT@131.142.113.102:/proj/sot/ska/data/eng_archive/data/

# Back in linux git repo
python setup.py sdist
cp dist/Ska.engarchive-<version>.tar.gz $ska/pkgs/

# Update skare (pkgs.manifest)
As user:
 git commit -am "Update Ska.engarchive => ..."
 git push

As aca:
 cd ~/git/skare
 git pull
 ./configure --prefix=/proj/sot/ska   # if needed
 make python_modules

# Install scripts from git dev directory (probably as aldcroft)
make install


###################################################################
Add 1999 derived params
###################################################################

  export ENG_ARCHIVE=${PWD}:/proj/sot/ska/data/eng_archive

# Add_derived.py checks for any new derived content types or columns.  Note
# that you cannot just add a new column to an existing content type because the
# ingest process (update_archive.py) does not currently handle this.

  ./add_derived.py --data-root=$PWD --start='1999:205' --stop='1999:210'

The dp_pcad4 data depend on ephemeris values which don't start until 1999:226:08:00:00.
(Use dev_utils/print_msidfile_times.py to see this).  So remove that and start that one later.

  rm -rf data/dp_pcad4
  ./add_derived.py --data-root=$PWD --start='1999:226' --stop='1999:231' --content=dp_pcad4

Now update everything for a few days:

  ./update_archive.py --data-root=$PWD --content=dp_ --date-now 1999:240 --content dp_pcad4

  ./update_archive.py --data-root=$PWD --date-now 1999:241 --content dp_pcad >& log_pcad &
  ./update_archive.py --data-root=$PWD --date-now 1999:241 --content dp_acispow >& log_acispow &
  ./update_archive.py --data-root=$PWD --date-now 1999:214 --content dp_thermal >& log_thermal.1 &

Finally update through the beginning of 2000.  This takes a little while:

  ./update_archive.py --data-root=$PWD --date-now 2000:010 --max-lookback-time=400 \
      --content dp_ >& log_derived.1 &
