###  Memo for development commands
###
###  This is essentially a list of command procedures I (=SS) am using.
###  (not sure how useful it is for others....;D)
###

### source distribution
# update version number
$ rm -rf dist/*
$ python3 setup.py sdist
$ twine check dist/*        # check syntax
$ python3 -m twine upload --repository testpypi dist/*    # upload at testpypi
$ python3 -m twine upload dist/*                          # uploade to the official

### check test PyPI distribution
pip3 install -i  https://test.pypi.org/simple  mfem==4.8.0rc10  -verbose


### building step-by-step
git clone https://github.com/mfem/PyMFEM.git
cd PyMFEM

1)  build external libraries
pip install . -C"ext-only=Yes" --verbose
pip install . -C"with-parallel=Yes" -C"ext-only=Yes" --verbose
 
2) generate wrapper codes
pip install . -C"skip-ext=Yes"  -C"swig=Yes" --verbose
pip install . -C"with-parallel=Yes" -C"skip-ext=Yes"  -C"swig=Yes" --verbose

3) compile wrapper
pip install . -C"skip-ext=Yes"  -C"skip-swig=Yes" --verbose
pip install . -C"with-parallel=Yes" -C"skip-ext=Yes"  -C"skip-swig=Yes" --verbose

### Test linux binary build locally using docker (1) for Python <= 3.9

sudo docker pull quay.io/pypa/manylinux2014_x86_64
sudo docker run -it -v $(pwd):/io quay.io/pypa/manylinux2014_x86_64
yum install openssh-server openssh-clients zlib-devel
#git clone git@github.com:mfem/PyMFEM.git
git clone https://github.com/mfem/PyMFEM.git
export PATH=/opt/python/cp38-cp38/bin:$PATH
pip install six numpy  auditwheel twine cmake swig
python3 setup.py install
rm -rf dist/*
python3 setup.py bdist_wheel
export LD_LIBRARY_PATH=/root/PyMFEM/external/mfem/cmbuild_ser/:$LD_LIBRARY_PATH
rm -rf wheelhouse/*
auditwheel repair dist/mfem-*-linux_x86_64.whl
rm -rf dist/*
mv wheelhouse/* dist/
python3 setup.py sdist

### Test linux binary build locally using docker (2) for Python <= 3.10

sudo docker pull quay.io/pypa/manylinux_2_24_x86_64
sudo docker run -it -v $(pwd):/io quay.io/pypa/manylinux_2_24_x86_64
apt-get update
apt-get install openssh-server zlib1g-dbg
git clone https://github.com/mfem/PyMFEM.git
export PATH=/opt/python/cp38-cp38/bin:$PATH
pip install six numpy scipy auditwheel twine cmake swig
python3 setup.py install
rm -rf dist/*
python3 setup.py bdist_wheel
export LD_LIBRARY_PATH=/root/PyMFEM/external/mfem/cmbuild_ser/:$LD_LIBRARY_PATH
rm -rf wheelhouse/*
auditwheel repair dist/mfem-*-linux_x86_64.whl
rm -rf dist/*
mv wheelhouse/* dist/
python3 setup.py sdist


### make a test release
testrelease_binary uses test_release branch and push a package to test PyPI.
this action triggers when a new tag whose name stats with test_ is
created.

### debugging
gdb python
gdb) run ../examples/ex1p
gdb) bt    <--- backtrace

### debugging GDB (MPI)
Open an xterm window for each MPI process, with gdb attached to each python process.
The -x flag tells gdb to run the commands given in the specified file.

$ echo "run model.py" > gdb.in
$ mpirun -np 4 xterm -e "gdb -x gdb.in python"
$ mpirun -np 4 xterm -e "gdb -ex 'break MPI_Abort' -x gdb.in python" (if MPI kills the process)

### debugging Valgrind (MPI)
-sl : length of scroll back
-hold : do not close xterm after valgrind dies

$ export PYTHONMALLOC=malloc
$ mpirun -np 2 xterm -sl 1000 -hold -e "valgrind python model.py -p"

### profiling
python -m cProfile -o output ../examples/ex18.py


### test sequence for centos7.9 (prepared during communication Issue#109)
$ sudo docker run -it centos:centos7.9.2009
$ yum install openssh-server openssh-clients git wget
$ yum groupinstall "Development Tools"
$ wget https://repo.anaconda.com/archive/Anaconda3-2021.11-Linux-x86_64.sh
$ bash Anaconda3-2021.11-Linux-x86_64.sh
$ source .bashrc
$ pip install cmake 
$ conda install -c conda-forge/label/gcc9 gcc_linux-64
$ conda install -c conda-forge/label/gcc9 gxx_linux-64
$ conda install -c conda-forge/label/gcc9 mpi4py

### test using python:3.9 image
sudo docker pull python:3.9
sudo docker run -it python:3.9 bash


# git clone https://github.com/mfem/PyMFEM.git
# cd PyMFEM
# git checkout next
# pip install SWIG cmake numba-scipy
# python3 setup.py install --mfem-branch=master
# cd test
# python run_examples.py -serial -verbose

for parallel
# apt-get-update
# apt-get install libopenmpi-dev libscalapack-openmpi-dev
# pip install mpi4py --install-option="CC=mpicc"
# python3 setup.py install --with-parallel --mfem-branch=master

### test using python:3.10 image
sudo docker pull python:3.10
sudo docker run -it python:3.10 bash

python -m pip install ./ --user --install-option='--mfem-branch=master'


### test using python:3.9 image wit parallel
sudo docker pull python:3.9
sudo docker run -it python:3.9 bash

(usually we need an editor...)
apt-get update
apt-get install libmpich-dev

pip install six numpy scipy auditwheel twine cmake swig mpi4py
git clone https://github.com/mfem/PyMFEM.git

# testing it without swig
python setup.py install --with-parallel --mfem-branch='master'

# testing it with swig
git checkout tmop_support
python setup.py clean --swig
python setup.py install --ext-only --with-gslib --with-parallel --mfem-branch='master'
python setup.py install --swig --with-gslib --with-parallel 
python setup.py install --skip-ext --with-gslib --with-parallel


### test using python:3.9 image wit parallel as MFEM, HYPRE and METIS external
sudo docker pull python:3.9
sudo docker run -it python:3.9 bash

(usually we need an editor...)
apt-get update
apt-get install libmpich-dev

pip install cmake six numba-scipy mpi4py

wget https://github.com/mfem/tpls/raw/gh-pages/metis-5.1.0.tar.gz
wget https://github.com/hypre-space/hypre/archive/v2.28.0.tar.gz

* Hypre
tar -zxvf v2.28.0.tar.gz
mv hypre-2.28.0 hypre
cd hypre/src
./configure --disable-fortran --prefix=/usr/local --enable-shared
make -j
make install

* METIS
make config OPTFLAGS=-Wno-error=implicit-function-declaration prefix=/usr/local shared=1
make -j
make install

* MFEM
git clone https://github.com/mfem/mfem.git
cd mfem
make config MFEM_USE_MPI=YES MFEM_USE_METIS_5=YES PREFIX=/usr/local SHARED=YES
make install -j

#PyMFEM
git clone https://github.com/mfem/pyMFEM.git
cd PyMFEM
python3 setup.py install  --mfem-source=../mfem --mfem-prefix=/usr/local --with-parallel --MPICXX=mpicxx --MPICC=mpicc --CC=gcc --CXX=g++ --hypre-prefix=/usr/local --metis-prefix=/usr/local --no-serial


### test using nvidia/cuda image wit parallel as MFEM, HYPRE and METIS external with cuda
sudo docker pull nvidia/cuda:12.3.1-devel-ubuntu22.04

This image need container-toolkit
* instruction is here https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
* After registering repo and installing it using apt-get install, we need to do 
  sudo nvidia-ctk runtime configure --runtime=docker
  sudo systemctl restart docker

sudo docker run  --gpus all -it nvidia/cuda:12.3.1-devel-ubuntu22.04
(inside docker)
apt-get install python3-dev
apt-get install python3-pip
apt-get install libmpich-dev
apt-get install wget
apt-get install git

pip install cmake swig six numba-scipy mpi4p
wget https://github.com/mfem/tpls/raw/gh-pages/metis-5.1.0.tar.gz
wget https://github.com/hypre-space/hypre/archive/v2.28.0.tar.gz

* Hypre
tar -zxvf v2.28.0.tar.gz
mv hypre-2.28.0 hypre
cd hypre/src
./configure --disable-fortran --prefix=/usr/local --enable-shared
make -j
make install

* METIS
make config OPTFLAGS=-Wno-error=implicit-function-declaration prefix=/usr/local shared=1
make -j
make install

* MFEM
git clone https://github.com/mfem/mfem.git
cd mfem
make config MFEM_USE_MPI=YES MFEM_USE_METIS_5=YES PREFIX=/usr/local SHARED=YES
make  -j
make install

# PyMFEM
git clone https://github.com/mfem/pyMFEM.git
cd PyMFEM
python3 setup.py install  --mfem-source=../mfem --mfem-prefix=/usr/local --with-parallel --with-cuda --MPICXX=mpicxx --MPICC=mpicc --CC=gcc --CXX=g++ --hypre-prefix=/usr/local --metis-prefix=/usr/local --no-serial
(above installes mfem under site-pakcage dir. somehow, the Python in this docker looks at dist-packages
 for now, the following export works)
export PYTHONPATH=/usr/lib/python3.10/site-packages/:$PYTHONPATH
cd examples
python3.10 ./ex1p.py -pa -d cuda
