#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os, sys, re, glob, time, traceback
import h5py
import mpi4py
from mpi4py import MPI
import pickle
import gzip
import shutil
import psutil
import gc
import pathlib
from pathlib import Path, PurePosixPath
import subprocess
import io
import copy
import datetime
import timeit
import textwrap
import tqdm
from tqdm import tqdm
import math
import numpy as np
import scipy as sp
from scipy import interpolate, integrate, signal, stats, special

# import numba
# from numba import jit, njit

import skimage
from skimage import color

import matplotlib as mpl
import matplotlib.pyplot as plt
import cmocean, colorcet, cmasher

import colorspacious
from colorspacious import cspace_converter

# import vtk
# from vtk.util import numpy_support

## required for EAS3
import struct

'''
========================================================================

turbx: an extensible toolkit for analyzing turbulent flow datasets
--> version Spring 2024

$> wget https://raw.githubusercontent.com/iagappel/turbx/main/turbx/turbx.py
$> git clone git@gitlab.iag.uni-stuttgart.de:transi/turbx.git
$> git clone git@github.com:iagappel/turbx.git

h5py Documentation:
https://docs.h5py.org/_/downloads/en/3.8.0/pdf/

compiling parallel HDF5 & h5py:
https://docs.h5py.org/en/stable/mpi.html#building-against-parallel-hdf5

========================================================================
'''

# data container interface classes for HDF5 containers
# ======================================================================

class cgd(h5py.File):
    '''
    Curvilinear Grid Data (CGD)
    ---------------------------
    - super()'ed h5py.File class
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.cgd()')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## cgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 32    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8     )
        perms          = kwargs.pop('perms'          , '640' )
        
        ## passthrough arg to get_header() to automatically read the full 3D grid on every MPI rank
        ## upon opening. This is in general a bad idea for CGD, which has a 3D grid. If every rank reads the
        ## full grid, RAM can very quickly fill up.
        read_grid = kwargs.pop('read_grid', False)
        
        if not isinstance(stripe_count, int):
            raise ValueError
        if not isinstance(stripe_size_mb, int):
            raise ValueError
        if not isinstance(perms, str):
            raise ValueError
        if not len(perms)==3:
            raise ValueError
        if not re.fullmatch(r'\d{3}',perms):
            raise ValueError
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using romio --> causes crash
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        
        ## set ROMIO hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(16*1024**2))) ) ## 16 [MB]
                ##
                #mpi_info.Set('romio_no_indep_rw' , 'true' ) ## deferred open + only collective I/O 
                #mpi_info.Set('cb_nodes' , str(int(round(1*self.n_ranks))) )
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        ## | rdcc_nbytes:
        ## | ------------
        ## | Integer setting the total size of the raw data chunk cache for this dataset in bytes.
        ## | In most cases increasing this number will improve performance, as long as you have 
        ## | enough free memory. The default size is 1 MB
        
        ## --> gets passed to H5Pset_chunk_cache
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        ## | rdcc_nslots:
        ## | ------------
        ## | Integer defining the number of chunk slots in the raw data chunk cache for this dataset.
        
        ## if ('rdcc_nslots' not in kwargs):
        ##     kwargs['rdcc_nslots'] = 521
        
        ## rgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with cgd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(0.1)
                Path(self.fname).touch() ## touch a new file
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        ## call actual h5py.File.__init__()
        super(cgd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose, read_grid=read_grid)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(cgd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed CGD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(cgd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of CGD class instance
        --> this gets called automatically upon opening the file
        '''
        
        verbose = kwargs.get('verbose',True)
        
        ## by default, do NOT read CGD grid upon opening
        ## leave it up to the individual functions to read grid into RAM
        read_grid = kwargs.get('read_grid',False) 
        
        if (self.rank!=0):
            verbose=False
        
        # if verbose: print('\n'+'cgd.get_header()'+'\n'+72*'-')
        # t_start_func = timeit.default_timer()
        
        # === attrs
        if ('duration_avg' in self.attrs.keys()):
            self.duration_avg = self.attrs['duration_avg']
        if ('rectilinear' in self.attrs.keys()):
            self.rectilinear = self.attrs['rectilinear']
        if ('curvilinear' in self.attrs.keys()):
            self.curvilinear = self.attrs['curvilinear']
        
        ## these should be set in the (init_from_() funcs)
        if ('fclass' in self.attrs.keys()):
            self.fclass = self.attrs['fclass'] ## 'cgd'
        if ('fsubtype' in self.attrs.keys()):
            self.fsubtype = self.attrs['fsubtype'] ## 'unsteady','mean','prime'
        
        # === udef
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## just make udef_real a dict with udef_char as keys
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.C_Suth      = self.udef['C_Suth']
            self.S_Suth      = self.udef['S_Suth']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            
            # === characteristic values : derived
            
            rho_inf = self.rho_inf = self.p_inf/(self.R * self.T_inf)
            mu_inf  = self.mu_inf  = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            nu_inf  = self.nu_inf  = self.mu_inf/self.rho_inf
            a_inf   = self.a_inf   = np.sqrt(self.kappa*self.R*self.T_inf)
            U_inf   = self.U_inf   = self.Ma*self.a_inf
            cp      = self.cp      = self.R*self.kappa/(self.kappa-1.)
            cv      = self.cv      = self.cp/self.kappa
            r       = self.r       = self.Pr**(1/3)
            Tw      = self.Tw      = self.T_inf
            Taw     = self.Taw     = self.T_inf + self.r*self.U_inf**2/(2*self.cp)
            lchar   = self.lchar   = self.Re*self.nu_inf/self.U_inf
            
            tchar   = self.tchar = self.lchar / self.U_inf
            uchar   = self.uchar = self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf' , '%0.3f [kg/m³]'    % self.rho_inf )
            if verbose: even_print('mu_inf'  , '%0.6E [kg/(m·s)]' % self.mu_inf  )
            if verbose: even_print('nu_inf'  , '%0.6E [m²/s]'     % self.nu_inf  )
            if verbose: even_print('a_inf'   , '%0.6f [m/s]'      % self.a_inf   )
            if verbose: even_print('U_inf'   , '%0.6f [m/s]'      % self.U_inf   )
            if verbose: even_print('cp'      , '%0.3f [J/(kg·K)]' % self.cp      )
            if verbose: even_print('cv'      , '%0.3f [J/(kg·K)]' % self.cv      )
            if verbose: even_print('r'       , '%0.6f [-]'        % self.r       )
            if verbose: even_print('Tw'      , '%0.3f [K]'        % self.Tw      )
            if verbose: even_print('Taw'     , '%0.3f [K]'        % self.Taw     )
            if verbose: even_print('lchar'   , '%0.6E [m]'        % self.lchar   )
            if verbose: even_print('tchar'   , '%0.6E [s]'        % self.tchar   )
            if verbose: print(72*'-')
            #if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the CGD instance
            udef_char_deriv = ['rho_inf', 'mu_inf', 'nu_inf', 'a_inf', 'U_inf', 'cp', 'cv', 'r', 'Tw', 'Taw', 'lchar']
            udef_real_deriv = [ rho_inf,   mu_inf,   nu_inf,   a_inf,   U_inf,   cp,   cv,   r,   Tw,   Taw,   lchar ]
            self.udef_deriv = dict(zip(udef_char_deriv, udef_real_deriv))
        
        else:
            pass
        
        # === read coordinate vectors
        # - (full) grid will only be read (to every rank!!!) if read_grid=True
        # - reading the grid is often not necessary. when it is necessary, just read it directly from the HDF5/h5py handle in
        #    the corresponding function
        
        if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
            
            if read_grid:
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## read 3D/1D coordinate arrays
                ## ( dont transpose right away --> allows for 1D storage )
                
                if self.usingmpi:
                    
                    dset = self['dims/x']
                    with dset.collective:
                        x = self.x = np.copy( dset[()] )
                    
                    self.comm.Barrier()
                    
                    dset = self['dims/y']
                    with dset.collective:
                        y = self.y = np.copy( dset[()] )
                    
                    self.comm.Barrier()
                    
                    dset = self['dims/z']
                    with dset.collective:
                        z = self.z = np.copy( dset[()] )
                
                else:
                    
                    x = self.x = np.copy( self['dims/x'][()] )
                    y = self.y = np.copy( self['dims/y'][()] )
                    z = self.z = np.copy( self['dims/z'][()] )
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ( x.nbytes + y.nbytes + z.nbytes ) * self.n_ranks / 1024**3
                if verbose:
                    even_print('read x,y,z (full)', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                if True: ## transpose the coordinate arrays
                    
                    '''
                    nx,ny,nz should probably just be stored as attributes
                    '''
                    
                    if (x.ndim==1):
                        nx = self.nx = x.shape[0]
                    elif (x.ndim==3):
                        x  = self.x  = np.copy( x.T )
                        nx = self.nx = x.shape[0]
                    else:
                        raise AssertionError('x.ndim=%i'%(x.ndim,))
                    
                    if (y.ndim==1):
                        ny = self.ny = y.shape[0]
                    elif (y.ndim==3):
                        y  = self.y  = np.copy( y.T )
                        ny = self.ny = y.shape[1]
                    else:
                        raise AssertionError('y.ndim=%i'%(y.ndim,))
                    
                    if (z.ndim==1):
                        nz = self.nz = z.shape[0]
                    elif (z.ndim==3):
                        z  = self.z  = np.copy( z.T )
                        nz = self.nz = z.shape[2]
                    else:
                        raise AssertionError('z.ndim=%i'%(z.ndim,))
            
            else:
                
                x_shp = self['dims/x'].shape
                y_shp = self['dims/y'].shape
                z_shp = self['dims/z'].shape
                
                ## assert 3D
                if (self['dims/x'].ndim != 3):
                    raise ValueError
                
                ## assert shapes agree 
                if (x_shp != y_shp):
                    raise AssertionError
                if (y_shp != z_shp):
                    raise AssertionError
                
                ## datasets are stored transposed!
                nz,ny,nx = self['dims/x'].shape
                self.nx = nx
                self.ny = ny
                self.nz = nz
                
                self.x = None
                self.y = None
                self.z = None
            
            ngp = self.ngp = nx*ny*nz
            
            if verbose: even_print('nx',  '%i'%nx  )
            if verbose: even_print('ny',  '%i'%ny  )
            if verbose: even_print('nz',  '%i'%nz  )
            if verbose: even_print('ngp', '%i'%ngp )
            #if verbose: print(72*'-')
            
            if False:
                if verbose: even_print('x_min', '%0.2f'%x.min())
                if verbose: even_print('x_max', '%0.2f'%x.max())
                if verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (x[1]-x[0]), (x[-1]-x[-2]) ))
                if verbose: even_print('y_min', '%0.2f'%y.min())
                if verbose: even_print('y_max', '%0.2f'%y.max())
                if verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (y[1]-y[0]), (y[-1]-y[-2]) ))
                if verbose: even_print('z_min', '%0.2f'%z.min())
                if verbose: even_print('z_max', '%0.2f'%z.max())        
                if verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (z[1]-z[0]), (z[-1]-z[-2]) ))
                if verbose: print(72*'-'+'\n')
        
        # === time vector
        
        if ('dims/t' in self):
            
            self.t = np.copy(self['dims/t'][()])
            
            if ('data' in self): ## check t dim and data arr agree
                nt,_,_,_ = self['data/%s'%list(self['data'].keys())[0]].shape
                if (nt!=self.t.size):
                    raise AssertionError('nt!=self.t.size : %i!=%i'%(nt,self.t.size))
            
            try:
                self.dt = self.t[1] - self.t[0]
            except IndexError:
                self.dt = 0.
            
            self.nt       = nt       = self.t.size
            self.duration = duration = self.t[-1] - self.t[0]
            self.ti       = ti       = np.arange(self.nt, dtype=np.int64)
        
        elif all([ ('data' in self) , ('dims/t' not in self) ]): ## data but no time --> make dummy time vector
            self.scalars = list(self['data'].keys())
            nt,_,_,_ = self['data/%s'%self.scalars[0]].shape
            self.nt  = nt
            self.t   =      np.arange(self.nt, dtype=np.float64)
            self.ti  = ti = np.arange(self.nt, dtype=np.int64)
            self.dt  = 1.
            self.duration = duration = self.t[-1]-self.t[0]
        
        else: ## no data, no time
            self.t  = np.array([], dtype=np.float64)
            self.ti = np.array([], dtype=np.int64)
            self.nt = nt = 0
            self.dt = 0.
            self.duration = duration = 0.
        
        #if verbose: print(72*'-')
        if verbose: even_print('nt', '%i'%self.nt )
        if verbose: even_print('dt', '%0.6f'%self.dt)
        if verbose: even_print('duration', '%0.2f'%self.duration )
        if hasattr(self, 'duration_avg'):
            if verbose: even_print('duration_avg', '%0.2f'%self.duration_avg )
        #if verbose: print(72*'-'+'\n')
        
        if hasattr(self,'rectilinear'):
            if verbose: even_print('rectilinear', str(self.rectilinear) )
        if hasattr(self,'curvilinear'):
            if verbose: even_print('curvilinear', str(self.curvilinear) )
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys()) ## 4D : string names of scalars : ['u','v','w'],...
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self['data/%s'%scalar].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    # === I/O
    
    def init_from_eas4(self, fn_eas4, **kwargs):
        '''
        initialize a CGD from an EAS4 (NS3D output format)
        '''
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        verbose  = kwargs.get('verbose',True)
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        if (self.rank!=0):
            verbose=False
        
        # # === spatial resolution filter : take every nth grid point
        # sx = kwargs.get('sx',1)
        # sy = kwargs.get('sy',1)
        # sz = kwargs.get('sz',1)
        # #st = kwargs.get('st',1)
        
        # # === spatial resolution filter : set x/y/z bounds
        # x_min = kwargs.get('x_min',None)
        # y_min = kwargs.get('y_min',None)
        # z_min = kwargs.get('z_min',None)
        # 
        # x_max = kwargs.get('x_max',None)
        # y_max = kwargs.get('y_max',None)
        # z_max = kwargs.get('z_max',None)
        # 
        # xi_min = kwargs.get('xi_min',None)
        # yi_min = kwargs.get('yi_min',None)
        # zi_min = kwargs.get('zi_min',None)
        # 
        # xi_max = kwargs.get('xi_max',None)
        # yi_max = kwargs.get('yi_max',None)
        # zi_max = kwargs.get('zi_max',None)
        
        ## grid filters are currently not supported for CGD
        self.hasGridFilter=False
        
        ## set default attributes
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'cgd'
        
        if verbose: print('\n'+'cgd.init_from_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        #if verbose: even_print('infile', f'{os.path.basename(os.path.dirname(fn_eas4))}/{os.path.basename(fn_eas4)}')
        #if verbose: even_print('infile', f'{Path(fn_eas4).parent.resolve().name}/{Path(fn_eas4).name}')
        if verbose: even_print('infile', str(Path(fn_eas4).relative_to(Path())))
        if verbose: even_print('infile size', '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3))
        if verbose: even_print('outfile', self.fname)
        
        with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=MPI.COMM_WORLD, read_3d_grid=False) as hf_eas4:
            
            ## original GMODE in eas4 file
            ## remember than 1,2 get automatically expanded to 4 during eas4.get_header()
            # if verbose: even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1_orig, gmode_dict[hf_eas4.gmode_dim1_orig] ) )
            # if verbose: even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2_orig, gmode_dict[hf_eas4.gmode_dim2_orig] ) )
            # if verbose: even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3_orig, gmode_dict[hf_eas4.gmode_dim3_orig] ) )
            
            if verbose: even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1, gmode_dict[hf_eas4.gmode_dim1] ) )
            if verbose: even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2, gmode_dict[hf_eas4.gmode_dim2] ) )
            if verbose: even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3, gmode_dict[hf_eas4.gmode_dim3] ) )
            
            if verbose: even_print( 'rectilinear' , str(hf_eas4.is_rectilinear) )
            if verbose: even_print( 'curvilinear' , str(hf_eas4.is_curvilinear) )
            
            self.attrs['rectilinear'] = hf_eas4.is_rectilinear
            self.attrs['curvilinear'] = hf_eas4.is_curvilinear
            #self.rectilinear = self.attrs['rectilinear']
            #self.curvilinear = self.attrs['curvilinear']
            
            # === copy over header info if needed
            
            if all([('header/udef_real' in self),('header/udef_char' in self)]):
                raise ValueError('udef already present')
            else:
                udef         = hf_eas4.udef
                udef_real    = list(udef.values())
                udef_char    = list(udef.keys())
                udef_real_h5 = np.array(udef_real, dtype=np.float64)
                udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
                
                self.create_dataset('header/udef_real', data=udef_real_h5, dtype=np.float64)
                self.create_dataset('header/udef_char', data=udef_char_h5, dtype='S128')
                self.udef      = udef
                self.udef_real = udef_real
                self.udef_char = udef_char
            
            # === copy over dims info
            
            if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
                raise ValueError('dims/x, dims/y, dims/z already in CGD file')
            
            ## do NOT copy EAS4 grid into memory at this point!
            #x = np.copy(hf_eas4.x)
            #y = np.copy(hf_eas4.y)
            #z = np.copy(hf_eas4.z)
            
            self.nx = nx = hf_eas4.nx
            self.ny = ny = hf_eas4.ny
            self.nz = nz = hf_eas4.nz
            
            ngp = nx*ny*nz
            ## nt = hf_eas4.nt --> no time data yet
            
            # === rank 3D grid ranges
            
            if self.usingmpi:
                
                comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                t4d = comm4d.Get_coords(self.rank)
                
                rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
                ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
                rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
                #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
                
                rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                
                rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
            
            else:
                
                nxr = self.nx
                nyr = self.ny
                nzr = self.nz
                #ntr = self.nt
            
            # === 3D grid expansion --> the grid is expanded rank-local for CGD, can easily be too big otherwise
            
            ## if ANY grid dimension has FULL_G
            if any([ (hf_eas4.gmode_dim1==5) , (hf_eas4.gmode_dim2==5) , (hf_eas4.gmode_dim3==5) ]):
                
                ## GMODE=(5,5,5)
                if all([ (hf_eas4.gmode_dim1==5) , (hf_eas4.gmode_dim2==5) , (hf_eas4.gmode_dim3==5) ]):
                    
                    raise NotImplementedError ## sorry :( this needs to be implemented as a collective read
                    
                    ## collective
                    # ...
                    
                    ## independent
                    # dset = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim01']
                    # x = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                    # dset = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim02']
                    # y = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                    # dset = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim03']
                    # z = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                
                ## GMODE=(5,5,4)
                elif all([ (hf_eas4.gmode_dim1==5) , (hf_eas4.gmode_dim2==5) , (hf_eas4.gmode_dim3==4) ]):
                    
                    x_ = np.copy(hf_eas4.x) ## will be 2D+1 from eas4() --> (nx,ny,1)
                    y_ = np.copy(hf_eas4.y) ## will be 2D+1 from eas4() --> (nx,ny,1)
                    z_ = np.copy(hf_eas4.z) ## will be 1D   from eas4() --> (nz,)
                    
                    ## assert that ndim is indeed correct
                    if (x_.ndim!=3):
                        raise ValueError
                    if (y_.ndim!=3):
                        raise ValueError
                    if (z_.ndim!=1):
                        raise ValueError
                    
                    ## rank-local 3D coordinate buffer
                    x = np.zeros((nxr,nyr,nzr), dtype=np.float64)
                    y = np.zeros((nxr,nyr,nzr), dtype=np.float64)
                    z = np.zeros((nxr,nyr,nzr), dtype=np.float64)
                    
                    ## broadcast sections of 1D/2D to rank-local 3D
                    x[:,:,:] = x_[rx1:rx2,ry1:ry2,:] ## broadcast on axis 2
                    y[:,:,:] = y_[rx1:rx2,ry1:ry2,:] ## broadcast on axis 2
                    z[:,:,:] = z_[rz1:rz2]           ## broadcast on axes (0,1)
                
                else:
                    
                    ## GMODE combos like (4,5,5) / (5,4,5) not yet implemented
                    raise NotImplementedError
            
            else: ## if no dim has FULL_G --> this means a rectilinear grid is being read, so rgd() could actually be used
                
                ## GMODE=(4,4,4)
                ## remember that eas4() automatically expands 1,2 to 4, so e.g. EAS4 (4,4,2) automatically becomes (4,4,4)
                if all([ (hf_eas4.gmode_dim1==4) , (hf_eas4.gmode_dim2==4) , (hf_eas4.gmode_dim3==4) ]):
                    
                    x = np.copy(hf_eas4.x)
                    y = np.copy(hf_eas4.y)
                    z = np.copy(hf_eas4.z)
                    
                    ## assert 1D
                    if (x.ndim!=1):
                        raise ValueError
                    if (y.ndim!=1):
                        raise ValueError
                    if (z.ndim!=1):
                        raise ValueError
                    
                    ## expand to 3D
                    #x, y, z = np.meshgrid(x, y, z, indexing='ij')
                    x, y, z = np.meshgrid(x[rx1:rx2], y[ry1:ry2], z[rz1:rz2], indexing='ij') ## rank-local
                
                else:
                    raise NotImplementedError
            
            # === by this point, the grid should have been expanded to 3D
            
            if (x.ndim!=3):
                raise ValueError('grid should be 3D expanded (rank-local) at this point')
            if (y.ndim!=3):
                raise ValueError('grid should be 3D expanded (rank-local) at this point')
            if (z.ndim!=3):
                raise ValueError('grid should be 3D expanded (rank-local) at this point')
            
            ## old
            ## broadcast in dimensions with shape=1
            ## this can easily eat up all RAM in MPI parallel mode
            if False:
                if ( hf_eas4.gmode_dim1==5 ) and ( x.shape != (nx,ny,nz) ):
                    x = np.broadcast_to(x, (nx,ny,nz))
                    if verbose: print('broadcasted x')
                
                if ( hf_eas4.gmode_dim2==5 ) and ( y.shape != (nx,ny,nz) ):
                    y = np.broadcast_to(y, (nx,ny,nz))
                    if verbose: print('broadcasted y')
                
                if ( hf_eas4.gmode_dim3==5 ) and ( z.shape != (nx,ny,nz) ):
                    z = np.broadcast_to(z, (nx,ny,nz))
                    if verbose: print('broadcasted z')
            
            shape  = (nz,ny,nx)
            chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=4, itemsize=8)
            
            # === write coord arrays
            
            if ('dims/x' in self):
                del self['dims/x']
            if ('dims/y' in self):
                del self['dims/y']
            if ('dims/z' in self):
                del self['dims/z']
            
            if False: ## serial write
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                dset = self.create_dataset('dims/x', data=x.T, shape=shape, chunks=chunks)
                dset = self.create_dataset('dims/y', data=y.T, shape=shape, chunks=chunks)
                dset = self.create_dataset('dims/z', data=z.T, shape=shape, chunks=chunks)
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                if verbose:
                    even_print('write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            if True: ## collective write
                
                ## initialize datasets
                dset_x = self.create_dataset('dims/x', shape=shape, chunks=chunks, dtype=x.dtype)
                dset_y = self.create_dataset('dims/y', shape=shape, chunks=chunks, dtype=y.dtype)
                dset_z = self.create_dataset('dims/z', shape=shape, chunks=chunks, dtype=z.dtype)
                
                chunk_kb_ = np.prod(dset_x.chunks)*8 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (z,y,x)','%s'%str(dset_x.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi: 
                    with dset_x.collective:
                        #dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x.T
                else:
                    dset_x[:,:,:] = x.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_y.collective:
                        #dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y.T
                else:
                    dset_y[:,:,:] = y.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_z.collective:
                        #dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z.T
                else:
                    dset_z[:,:,:] = z.T
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                #data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                data_gb = 3 * 8 * nx * ny * nz / 1024**3
                
                if verbose:
                    even_print('write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                # === write a preliminary time array --> e.g. for baseflow
                
                if ('dims/t' in self):
                    del self['dims/t']
                dset = self.create_dataset( 'dims/t', data=hf_eas4.t )
        
        self.x = np.copy(x)
        self.y = np.copy(y)
        self.z = np.copy(z)
        
        if verbose: print(72*'-')
        self.get_header(verbose=True, read_grid=False)
        if verbose: print(72*'-')
        
        return
    
    def init_from_cgd(self, fn_cgd, **kwargs):
        '''
        initialize an CGD from an CGD (copy over header data & coordinate data)
        '''
        
        t_info = kwargs.get('t_info',True)
        copy_grid = kwargs.get('copy_grid',True)
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        
        verbose = kwargs.get('verbose',True)
        if (self.rank!=0):
            verbose=False
        
        ## set default attributes
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'cgd'
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        with cgd(fn_cgd, 'r', driver=self.driver, comm=self.comm) as hf_ref:
            
            ## copy over fsubtype
            if hasattr(hf_ref,'fsubtype'):
                self.attrs['fsubtype'] = hf_ref.fsubtype
            
            # === copy over header info if needed
            
            if all([('header/udef_real' in self),('header/udef_char' in self)]):
                raise ValueError('udef already present')
            else:
                udef         = hf_ref.udef
                udef_real    = list(udef.values())
                udef_char    = list(udef.keys())
                udef_real_h5 = np.array(udef_real, dtype=np.float64)
                udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
                
                self.create_dataset('header/udef_real', data=udef_real_h5, maxshape=np.shape(udef_real_h5), dtype=np.float64)
                self.create_dataset('header/udef_char', data=udef_char_h5, maxshape=np.shape(udef_char_h5), dtype='S128')
                self.udef      = udef
                self.udef_real = udef_real
                self.udef_char = udef_char
            
            # === copy over dims info
            
            # if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
            #     raise ValueError('dims/x, dims/y, dims/z already in CGD file')
            
            ## make sure source CGD file has grid
            if ('dims/x' not in hf_ref):
                raise ValueError(f'{hf_ref.fname} does not have dset dims/x')
            if ('dims/y' not in hf_ref):
                raise ValueError(f'{hf_ref.fname} does not have dset dims/y')
            if ('dims/z' not in hf_ref):
                raise ValueError(f'{hf_ref.fname} does not have dset dims/z')
            
            ## make sure source CGD grid is 3D
            if (hf_ref['dims/x'].ndim != 3):
                raise ValueError('CGD file x grid has ndim!=3')
            if (hf_ref['dims/y'].ndim != 3):
                raise ValueError('CGD file y grid has ndim!=3')
            if (hf_ref['dims/z'].ndim != 3):
                raise ValueError('CGD file z grid has ndim!=3')
            
            ## no!
            #x   = self.x   = hf_ref.x
            #y   = self.y   = hf_ref.y
            #z   = self.z   = hf_ref.z
            
            nx  = self.nx  = hf_ref.nx
            ny  = self.ny  = hf_ref.ny
            nz  = self.nz  = hf_ref.nz
            ngp = self.ngp = hf_ref.ngp
            
            # === rank 3D grid ranges
            
            if self.usingmpi:
                
                comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                t4d = comm4d.Get_coords(self.rank)
                
                rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
                ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
                rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
                #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
                
                rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                
                rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
            
            else:
                
                nxr = self.nx
                nyr = self.ny
                nzr = self.nz
                #ntr = self.nt
            
            # ===
            
            if copy_grid:
                
                if ('dims/x' in self):
                    del self['dims/x']
                if ('dims/y' in self):
                    del self['dims/y']
                if ('dims/z' in self):
                    del self['dims/z']
                
                ## 1D
                #self.create_dataset('dims/x', data=x)
                #self.create_dataset('dims/y', data=y)
                #self.create_dataset('dims/z', data=z)
                
                ## 3D
                shape  = (nz,ny,nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=4, itemsize=8)
                
                ## if (x is not None) and (y is not None) and (z is not None):
                ##     if self.usingmpi: self.comm.Barrier()
                ##     t_start = timeit.default_timer()
                ##     dset = self.create_dataset('dims/x', data=x.T, shape=shape, chunks=chunks)
                ##     dset = self.create_dataset('dims/y', data=y.T, shape=shape, chunks=chunks)
                ##     dset = self.create_dataset('dims/z', data=z.T, shape=shape, chunks=chunks)
                ##     if self.usingmpi: self.comm.Barrier()
                ##     t_delta = timeit.default_timer() - t_start
                ##     data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                ##     if verbose:
                ##         even_print('write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                ## collective 3D grid read
                dset = hf_ref['dims/x']
                if self.usingmpi:
                    with dset.collective:
                        x = np.copy(dset[rz1:rz2,ry1:ry2,rx1:rx2].T)
                else:
                    x = np.copy(dset[()].T)
                
                dset = hf_ref['dims/y']
                if self.usingmpi:
                    with dset.collective:
                        y = np.copy(dset[rz1:rz2,ry1:ry2,rx1:rx2].T)
                else:
                    y = np.copy(dset[()].T)
                
                dset = hf_ref['dims/z']
                if self.usingmpi:
                    with dset.collective:
                        z = np.copy(dset[rz1:rz2,ry1:ry2,rx1:rx2].T)
                else:
                    z = np.copy(dset[()].T)
                
                ## initialize datasets for write
                dset_x = self.create_dataset('dims/x', shape=shape, chunks=chunks, dtype=np.float64) # dtype=x.dtype)
                dset_y = self.create_dataset('dims/y', shape=shape, chunks=chunks, dtype=np.float64)
                dset_z = self.create_dataset('dims/z', shape=shape, chunks=chunks, dtype=np.float64)
                
                chunk_kb_ = np.prod(dset_x.chunks)*8 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (z,y,x)','%s'%str(dset_x.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi: 
                    with dset_x.collective:
                        #dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x.T
                else:
                    dset_x[:,:,:] = x.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_y.collective:
                        #dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y.T
                else:
                    dset_y[:,:,:] = y.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_z.collective:
                        #dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z.T
                else:
                    dset_z[:,:,:] = z.T
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                #data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                data_gb = 3 * 8 * nx * ny * nz / 1024**3
                
                if verbose:
                    even_print('write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            # === copy over temporal dim info
            
            if t_info:
                self.t  = hf_ref.t
                self.nt = self.t.size
                self.create_dataset('dims/t', data=hf_ref.t)
            else:
                t = np.array([0.], dtype=np.float64)
                if ('dims/t' in self):
                    del self['dims/t']
                self.create_dataset('dims/t', data=t)
            
            # === 
            
            ## add additional [dims/<>] dsets
            for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                if (dsn in hf_ref):
                    if (dsn in self):
                        del self[dsn]
                    data = np.copy(hf_ref[dsn][()])
                    
                    if (dsn=='dims/stang') and (data.shape!=(self.nx,)):
                        raise ValueError
                    if (dsn=='dims/snorm') and (data.shape!=(self.ny,)):
                        raise ValueError
                    
                    ds = self.create_dataset(dsn, data=data, chunks=None)
                    #if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## add additional [csys/<>] dsets
            for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                if (dsn in hf_ref):
                    if (dsn in self):
                        del self[dsn]
                    data = np.copy(hf_ref[dsn][()])
                    
                    if (data.ndim!=3):
                        raise ValueError
                    if (data.shape!=(self.nx,self.ny,2)):
                        raise ValueError
                    
                    ds = self.create_dataset(dsn, data=data, chunks=None)
                    #if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## copy over [data_dim/<>] dsets if present
            if ('data_dim' in hf_ref):
                for dsn in hf_ref['data_dim'].keys():
                    data = np.copy( hf_ref[f'data_dim/{dsn}'][()] ) 
                    self.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                    if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False)
        return
    
    def import_eas4(self, fn_eas4_list, **kwargs):
        '''
        import data from a series of EAS4 files to a CGD
        '''
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        if verbose: print('\n'+'cgd.import_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        tt_min = kwargs.get('tt_min',None)
        tt_max = kwargs.get('tt_max',None)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        # === check for an often made mistake
        ts_min = kwargs.get('ts_min',None)
        ts_max = kwargs.get('ts_max',None)
        if (ts_min is not None):
            raise AssertionError('ts_min is not an option --> did you mean ti_min or tt_min?')
        if (ts_max is not None):
            raise AssertionError('ts_max is not an option --> did you mean ti_max or tt_max?')
        
        # === check that iterable of EAS4 files is OK
        if not hasattr(fn_eas4_list, '__iter__'):
            raise AssertionError('first arg \'fn_eas4_list\' must be iterable')
        for fn_eas4 in fn_eas4_list:
            if not os.path.isfile(fn_eas4):
                raise FileNotFoundError('%s not found!'%fn_eas4)
        
        # === ranks
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        ## skip dimensions --> spatial skips done in init_from_XXX()
        # sx = kwargs.get('sx',1)
        # sy = kwargs.get('sy',1)
        # sz = kwargs.get('sz',1)
        st = kwargs.get('st',1)
        
        ## update this CGD's header and attributes
        self.get_header(verbose=False)
        
        # === get all time info & check
        
        comm_eas4 = MPI.COMM_WORLD
        t = np.array([], dtype=np.float64)
        for fn_eas4 in fn_eas4_list:
            with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                t = np.concatenate((t, hf_eas4.t))
        comm_eas4.Barrier()
        
        if verbose: even_print('n EAS4 files','%i'%len(fn_eas4_list))
        if verbose: even_print('nt all files','%i'%t.size)
        
        if (t.size>1):
            
            ## check no zero distance elements
            if (np.diff(t).size - np.count_nonzero(np.diff(t))) != 0.:
                raise AssertionError('t arr has zero-distance elements')
            else:
                if verbose: even_print('check: Δt!=0','passed')
            
            ## check monotonically increasing
            if not np.all(np.diff(t) > 0.):
                raise AssertionError('t arr not monotonically increasing')
            else:
                if verbose: even_print('check: t mono increasing','passed')
            
            ## check constant Δt
            dt0 = np.diff(t)[0]
            if not np.all(np.isclose(np.diff(t), dt0, rtol=1e-3)):
                if (self.rank==0): print(np.diff(t))
                raise AssertionError('t arr not uniformly spaced')
            else:
                if verbose: even_print('check: constant Δt','passed')
        
        # === get all grid info & check
        
        # TODO : compare coordinate arrays for series of EAS4 files
        
        # === resolution filter (skip every n timesteps)
        tfi = self.tfi = np.arange(t.size, dtype=np.int64)
        if (st!=1):
            if verbose: even_print('st', '%i'%st)
            #print('>>> st : %i'%st)
            tfi = self.tfi = tfi[::st]
        
        # === get doRead vector
        doRead = np.full((t.size,), True, dtype=bool)
        
        ## skip filter
        if hasattr(self, 'tfi'):
            doRead[np.isin(np.arange(t.size),self.tfi,invert=True)] = False
        
        ## min/max index filter
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
            doRead[:ti_min] = False
        if (ti_max is not None):
            if not isinstance(ti_max, int):
                raise TypeError('ti_max must be type int')
            doRead[ti_max:] = False
        
        if (tt_min is not None):
            if (tt_min>=0.):
                doRead[np.where((t-t.min())<tt_min)] = False
            elif (tt_min<0.):
                doRead[np.where((t-t.max())<tt_min)] = False
        
        if (tt_max is not None):
            if (tt_max>=0.):
                doRead[np.where((t-t.min())>tt_max)] = False
            elif (tt_max<0.):
                doRead[np.where((t-t.max())>tt_max)] = False
        
        # === CGD times
        self.t  = np.copy(t[doRead])
        self.nt = self.t.size
        self.ti = np.arange(self.nt, dtype=np.int64)
        
        # === write back self.t to file
        if ('dims/t' in self):
            del self['dims/t']
        self.create_dataset('dims/t', data=self.t)
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === determine CGD scalars (from EAS4 scalars)
        if not hasattr(self, 'scalars') or (len(self.scalars)==0):
            with eas4(fn_eas4_list[0], 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                self.scalars   = hf_eas4.scalars
                self.n_scalars = len(self.scalars)
                
                ## decide dtypes
                for scalar in hf_eas4.scalars:
                    
                    domainName = hf_eas4.domainName
                    ti = 0
                    dset_path = 'Data/%s/ts_%06d/par_%06d'%(domainName,ti,hf_eas4.scalar_n_map[scalar])
                    dset = hf_eas4[dset_path]
                    dtype = dset.dtype
                    
                    ## if (prec=='same'):
                    ##     self.scalars_dtypes_dict[scalar] = dtype
                    ## elif (prec=='single'):
                    ##     if (dtype!=np.float32) and (dtype!=np.float64): ## make sure its either a single or double float
                    ##         raise ValueError
                    ##     self.scalars_dtypes_dict[scalar] = np.dtype(np.float32)
                    ## else:
                    ##     raise ValueError
                    
                    self.scalars_dtypes_dict[scalar] = np.dtype(np.float32)
        
        if self.usingmpi: comm_eas4.Barrier()
        
        data_gb = 4*self.nt*self.nz*self.ny*self.nx / 1024**3
        
        # === initialize datasets
        for scalar in self.scalars:
            
            dtype = self.scalars_dtypes_dict[scalar]
            float_bytes = dtype.itemsize
            data_gb = float_bytes*self.nt*self.nz*self.ny*self.nx / 1024**3
            
            if verbose:
                even_print('initializing data/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            
            shape  = (self.nt,self.nz,self.ny,self.nx)
            #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
            chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
            
            dset = self.create_dataset('data/%s'%scalar, 
                                       shape=shape, 
                                       dtype=dtype,
                                       chunks=chunks)
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === report size of CGD after initialization
        if verbose: tqdm.write(even_print(os.path.basename(self.fname), '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3), s=True))
        if verbose: print(72*'-')
        
        # === open EAS4s, read, write to CGD
        
        if verbose:
            progress_bar = tqdm(total=(self.nt*self.n_scalars), ncols=100, desc='import', leave=False, file=sys.stdout)
        
        data_gb_read  = 0.
        data_gb_write = 0.
        t_read  = 0.
        t_write = 0.
        
        tii  = -1 ## counter full series
        tiii = -1 ## counter CGD-local
        for fn_eas4 in fn_eas4_list:
            with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                
                #if verbose: tqdm.write( even_print( os.path.basename(fn_eas4)              , '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3), s=True))
                if verbose: tqdm.write( even_print( str(Path(fn_eas4).relative_to(Path())) , '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3), s=True))
                
                # if verbose: tqdm.write(even_print('gmode_dim1' , '%i'%hf_eas4.gmode_dim1  , s=True))
                # if verbose: tqdm.write(even_print('gmode_dim2' , '%i'%hf_eas4.gmode_dim2  , s=True))
                # if verbose: tqdm.write(even_print('gmode_dim3' , '%i'%hf_eas4.gmode_dim3  , s=True))
                
                if verbose: tqdm.write(even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1_orig, gmode_dict[hf_eas4.gmode_dim1_orig] ), s=True ))
                if verbose: tqdm.write(even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2_orig, gmode_dict[hf_eas4.gmode_dim2_orig] ), s=True ))
                if verbose: tqdm.write(even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3_orig, gmode_dict[hf_eas4.gmode_dim3_orig] ), s=True ))
                
                if verbose: tqdm.write(even_print('duration'   , '%0.2f'%hf_eas4.duration , s=True))
                
                # === write buffer
                
                # ## 5D [scalar][x,y,z,t] structured array
                # buff = np.zeros(shape=(nxr, nyr, nzr, bt), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
                
                # ===
                
                #domainName = 'DOMAIN_000000' ## only one domain supported
                domainName = hf_eas4.domainName
                
                for ti in range(hf_eas4.nt):
                    tii += 1 ## EAS4 series counter
                    if doRead[tii]:
                        tiii += 1 ## CGD counter
                        for scalar in hf_eas4.scalars:
                            if (scalar in self.scalars):
                                
                                # === collective read
                                
                                dset_path = 'Data/%s/ts_%06d/par_%06d'%(domainName,ti,hf_eas4.scalar_n_map[scalar])
                                dset = hf_eas4[dset_path]
                                
                                if hf_eas4.usingmpi: comm_eas4.Barrier()
                                t_start = timeit.default_timer()
                                if hf_eas4.usingmpi: 
                                    with dset.collective:
                                        data = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                                else:
                                    data = np.copy( dset[()] )
                                if hf_eas4.usingmpi: comm_eas4.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                data_gb       = data.nbytes / 1024**3
                                t_read       += t_delta
                                data_gb_read += data_gb
                                
                                if False:
                                    if verbose:
                                        txt = even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                        tqdm.write(txt)
                                
                                # === reduce precision (e.g. for restart which is usually double precision)
                                
                                if (data.dtype == np.float64):
                                    data = np.copy( data.astype(np.float32) )
                                
                                data_gb = data.nbytes / 1024**3
                                
                                # === collective write
                                
                                dset = self['data/%s'%scalar]
                                
                                if self.usingmpi: self.comm.Barrier()
                                t_start = timeit.default_timer()
                                if self.usingmpi:
                                    with dset.collective:
                                        dset[tiii,rz1:rz2,ry1:ry2,rx1:rx2] = data.T
                                else:
                                    
                                    if self.hasGridFilter:
                                        data = data[self.xfi[:,np.newaxis,np.newaxis],
                                                    self.yfi[np.newaxis,:,np.newaxis],
                                                    self.zfi[np.newaxis,np.newaxis,:]]
                                    
                                    dset[tiii,:,:,:] = data.T
                                
                                if self.usingmpi: self.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if False:
                                    if verbose:
                                        txt = even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                        tqdm.write(txt)
                                
                                if verbose: progress_bar.update()
                if hf_eas4.usingmpi: comm_eas4.Barrier()
        
        if verbose: progress_bar.close()
        
        if hf_eas4.usingmpi: comm_eas4.Barrier()
        if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False, read_grid=False)
        
        ## get read read/write totals all ranks
        if self.usingmpi:
            G = self.comm.gather([data_gb_read, data_gb_write, self.rank], root=0)
            G = self.comm.bcast(G, root=0)
            data_gb_read  = sum([x[0] for x in G])
            data_gb_write = sum([x[1] for x in G])
        
        if verbose: print(72*'-')
        if verbose: even_print('nt',       '%i'%self.nt )
        if verbose: even_print('dt',       '%0.6f'%self.dt )
        if verbose: even_print('duration', '%0.2f'%self.duration )
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        #if verbose: print('\n'+72*'-')
        if verbose: print('total time : cgd.import_eas4() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_csys_vecs_xy(self, fn_dat, xi=None, yi=None, **kwargs):
        '''
        add csys data (from pickled .dat, usually from tgg)
        - csys/vtang : (nx,ny,2)
        - csys/vnorm : (nx,ny,2)
        '''
        
        verbose = kwargs.get('verbose',True, **kwargs)
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        ## check xi
        if (xi is not None):
            if not isinstance(xi, np.ndarray ):
                raise ValueError('xi is not numpy array')
            if (xi.ndim!=1):
                raise ValueError('xi.ndim!=1')
            if (xi.dtype!=np.dtype(np.int32)) and (xi.dtype!=np.dtype(np.int64)):
                raise ValueError('xi.dtype not int32 or int64')
        
        ## check yi
        if (yi is not None):
            if not isinstance(yi, np.ndarray ):
                raise ValueError('yi is not numpy array')
            if (yi.ndim!=1):
                raise ValueError('yi.ndim!=1')
            if (yi.dtype!=np.dtype(np.int32)) and (yi.dtype!=np.dtype(np.int64)):
                raise ValueError('yi.dtype not int32 or int64')
        
        if verbose: print('\n'+'cgd.add_csys_vecs_xy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_dat):
            raise FileNotFoundError('file does not exist: %s'%str(fn_dat))
        
        with open(fn_dat,'rb') as f:
            dd = pickle.load(f)
        
        if ('xy2d' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain xy2d')
        #if ('vtang' not in dd.keys()):
        #    raise ValueError(f'file {fn_dat} does not contain vtang')
        #if ('vnorm' not in dd.keys()):
        #    raise ValueError(f'file {fn_dat} does not contain vnorm')
        
        ## grid from CGD
        x2d = np.copy(self['dims/x'][0,:,:]).T
        y2d = np.copy(self['dims/y'][0,:,:]).T
        
        ## check if consistent with [xy] grid in ref file
        xy2d = np.copy( dd['xy2d'] )
        
        if (xi is None) and (yi is None):
            np.testing.assert_allclose(xy2d[:,:,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,:,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is not None) and (yi is None):
            np.testing.assert_allclose(xy2d[xi,:,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[xi,:,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is None) and (yi is not None):
            np.testing.assert_allclose(xy2d[:,yi,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,yi,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is not None) and (yi is not None):
            xxi,yyi = np.ix_(xi,yi)
            np.testing.assert_allclose(xy2d[xxi,yyi,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[xxi,yyi,1], y2d, rtol=1e-14, atol=1e-14)
        else:
            raise ValueError
        
        ## key names for 'csys' group
        kn_csys = ['vtang','vnorm'] 
        for k in kn_csys:
            if (k in dd.keys()):
                dsn = f'csys/{k}'
                data = np.copy(dd[k])
                data_shape_orig = data.shape
                if (xi is not None):
                    data = np.copy(data[xi,:,:])
                if (yi is not None):
                    data = np.copy(data[:,yi,:])
                if (dsn in self):
                    del self[dsn]
                ds = self.create_dataset(dsn, data=data, chunks=None)
                if verbose: even_print(dsn,'%s --> %s'%(str(data_shape_orig),str(data.shape)))
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_csys_vecs_xy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_geom_data(self, fn_dat, xi=None, yi=None, **kwargs):
        '''
        add geom data (from pickled .dat, usually from tgg)
        - dims/stang : (nx,)
        - dims/snorm : (ny,)
        - dims/crv_R : (nx,)
        '''
        
        verbose = kwargs.get('verbose',True, **kwargs)
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        if verbose: print('\n'+'cgd.add_geom_data()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_dat):
            raise FileNotFoundError('file does not exist: %s'%str(fn_dat))
        
        ## open data file from tgg
        with open(fn_dat,'rb') as f:
            dd = pickle.load(f)
        
        if ('xy2d' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain xy2d')
        #if ('stang' not in dd.keys()):
        #    raise ValueError(f'file {fn_dat} does not contain stang')
        #if ('snorm' not in dd.keys()):
        #    raise ValueError(f'file {fn_dat} does not contain snorm')
        
        ## grid from CGD
        x2d = np.copy(self['dims/x'][0,:,:]).T
        y2d = np.copy(self['dims/y'][0,:,:]).T
        
        ## check if consistent with [xy] grid in ref file
        xy2d = np.copy( dd['xy2d'] )
        
        if (xi is None) and (yi is None):
            np.testing.assert_allclose(xy2d[:,:,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,:,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is not None) and (yi is None):
            np.testing.assert_allclose(xy2d[xi,:,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[xi,:,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is None) and (yi is not None):
            np.testing.assert_allclose(xy2d[:,yi,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,yi,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is not None) and (yi is not None):
            xxi,yyi = np.ix_(xi,yi)
            np.testing.assert_allclose(xy2d[xxi,yyi,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[xxi,yyi,1], y2d, rtol=1e-14, atol=1e-14)
        else:
            raise ValueError
        
        ## key names for 'dims' group
        ## these are all 1D with shape either (nx,) or (ny,)
        kn_dims = ['stang', 'snorm', 'crv_R'] 
        for k in kn_dims:
            if (k in dd.keys()):
                dsn = f'dims/{k}'
                #data = np.copy(dd[k])
                data = dd[k]
                if not isinstance(data, np.ndarray):
                    continue
                data_shape_orig = data.shape
                if (xi is not None):
                    if (data.shape[0]==xy2d.shape[0]):
                        data = np.copy(data[xi])
                if (yi is not None):
                    if (data.shape[0]==xy2d.shape[1]):
                        data = np.copy(data[yi])
                if (dsn in self):
                    del self[dsn]
                ds = self.create_dataset(dsn, data=data, chunks=None)
                if verbose: even_print(dsn,'%s --> %s'%(str(data_shape_orig),str(data.shape)))
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_geom_data() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    @staticmethod
    def copy(fn_cgd_src, fn_cgd_tgt, **kwargs):
        '''
        copy header info, selected scalars, and [x,y,z,t] range to new CGD file
        --> this currently does NOT work in serial mode
        '''
        
        #comm    = MPI.COMM_WORLD
        rank    = MPI.COMM_WORLD.Get_rank()
        n_ranks = MPI.COMM_WORLD.Get_size()
        
        if (rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.copy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        rt       = kwargs.get('rt',1)
        force    = kwargs.get('force',False) ## overwrite or raise error if exists
        
        ti_min   = kwargs.get('ti_min',None)
        ti_max   = kwargs.get('ti_max',None)
        scalars  = kwargs.get('scalars',None)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 32 ) ## for initializing CGD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8  )
        
        xi_min = kwargs.get('xi_min',None) ## 4D coordinate 
        xi_max = kwargs.get('xi_max',None)
        yi_min = kwargs.get('yi_min',None)
        yi_max = kwargs.get('yi_max',None)
        zi_min = kwargs.get('zi_min',None)
        zi_max = kwargs.get('zi_max',None)
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        
        ct = kwargs.get('ct',1) ## 'chunks' in time
        
        xi_step = kwargs.get('xi_step',1)
        yi_step = kwargs.get('yi_step',1)
        zi_step = kwargs.get('zi_step',1)
        
        prec_coords = kwargs.get('prec_coords',None)
        if (prec_coords is None):
            prec_coords = 'same'
        elif (prec_coords=='single'):
            pass
        elif (prec_coords=='same'):
            pass
        else:
            raise ValueError('prec_coords not set correctly')
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz!=n_ranks):
            raise AssertionError('rx*ry*rz!=n_ranks')
        if not os.path.isfile(fn_cgd_src):
            raise FileNotFoundError('%s not found!'%fn_cgd_src)
        if os.path.isfile(fn_cgd_tgt) and not force:
            raise FileExistsError('%s already exists. delete it or use \'force=True\' kwarg'%fn_cgd_tgt)
        
        # ===
        
        with cgd(fn_cgd_src, 'r', comm=MPI.COMM_WORLD, driver='mpio') as hf_src:
            with cgd(fn_cgd_tgt, 'w', comm=MPI.COMM_WORLD, driver='mpio', force=force, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_tgt:
                
                ## copy over header info
                hf_tgt.init_from_cgd(fn_cgd_src, rx=rx,ry=ry,rz=rz, copy_grid=False)
                
                if (scalars is None):
                    scalars = hf_src.scalars
                
                if verbose:
                    even_print('fn_cgd_src' , fn_cgd_src )
                    even_print('nx' , '%i'%hf_src.nx )
                    even_print('ny' , '%i'%hf_src.ny )
                    even_print('nz' , '%i'%hf_src.nz )
                    even_print('nt' , '%i'%hf_src.nt )
                    if verbose: print(72*'-')
                
                if (rx>hf_src.nx):
                    raise AssertionError('rx>nx')
                if (ry>hf_src.ny):
                    raise AssertionError('ry>ny')
                if (rz>hf_src.nz):
                    raise AssertionError('rz>nz')
                if (rt>hf_src.nt):
                    raise AssertionError('rt>nt')
                
                ## in general, cannot just load full grid data for CGD on every rank, gets too big
                # x  = np.copy( hf_src.x )
                # y  = np.copy( hf_src.y )
                # z  = np.copy( hf_src.z )
                t  = np.copy( hf_src.t )
                
                # xi  = np.arange(x.shape[0],dtype=np.int64) ## arange index vector, doesnt get touched!
                # yi  = np.arange(y.shape[0],dtype=np.int64)
                # zi  = np.arange(z.shape[0],dtype=np.int64)
                ti  = np.arange(t.shape[0],dtype=np.int64)
                
                # xfi = np.arange(x.shape[0],dtype=np.int64) ## gets clipped depending on x/y/z/t_min/max opts
                # yfi = np.arange(y.shape[0],dtype=np.int64)
                # zfi = np.arange(z.shape[0],dtype=np.int64)
                tfi = np.arange(t.shape[0],dtype=np.int64)
                
                xi  = np.arange(hf_src.nx, dtype=np.int64) ## arange index vector, doesnt get touched!
                yi  = np.arange(hf_src.ny, dtype=np.int64)
                zi  = np.arange(hf_src.nz, dtype=np.int64)
                ti  = np.arange(hf_src.nt, dtype=np.int64)
                
                xfi = np.arange(hf_src.nx, dtype=np.int64) ## gets clipped (and then stepped) depending on x/y/z/t_min/max opts
                yfi = np.arange(hf_src.ny, dtype=np.int64)
                zfi = np.arange(hf_src.nz, dtype=np.int64)
                tfi = np.arange(hf_src.nt, dtype=np.int64)
                
                # === global clip (coordinate index) --> supports negative indexing!
                
                if True: ## code folding
                    
                    if (xi_min is not None):
                        xfi_ = []
                        if verbose:
                            if (xi_min<0):
                                even_print('xi_min', '%i / %i'%(xi_min,xi[xi_min]))
                            else:
                                even_print('xi_min', '%i'%(xi_min,))
                        for c in xfi:
                            if (xi_min<0) and (c>=(hf_src.nx+xi_min)):
                                xfi_.append(c)
                            elif (xi_min>=0) and (c>=xi_min):
                                xfi_.append(c)
                        xfi=np.array(xfi_, dtype=np.int64)
                    else:
                        xi_min = 0
                    
                    if (xi_max is not None):
                        xfi_ = []
                        if verbose:
                            if (xi_max<0):
                                even_print('xi_max', '%i / %i'%(xi_max,xi[xi_max]))
                            else:
                                even_print('xi_max', '%i'%(xi_max,))
                        for c in xfi:
                            if (xi_max<0) and (c<=(hf_src.nx+xi_max)):
                                xfi_.append(c)
                            elif (xi_max>=0) and (c<=xi_max):
                                xfi_.append(c)
                        xfi=np.array(xfi_, dtype=np.int64)
                    else:
                        xi_max = xi[-1]
                    
                    ## check x
                    if ((xi[xi_max]-xi[xi_min]+1)<1):
                        raise ValueError('invalid xi range requested')
                    if (rx>(xi[xi_max]-xi[xi_min]+1)):
                        raise ValueError('more ranks than grid points in x')
                    
                    if (yi_min is not None):
                        yfi_ = []
                        if verbose:
                            if (yi_min<0):
                                even_print('yi_min', '%i / %i'%(yi_min,yi[yi_min]))
                            else:
                                even_print('yi_min', '%i'%(yi_min,))
                        for c in yfi:
                            if (yi_min<0) and (c>=(hf_src.ny+yi_min)):
                                yfi_.append(c)
                            elif (yi_min>=0) and (c>=yi_min):
                                yfi_.append(c)
                        yfi=np.array(yfi_, dtype=np.int64)
                    else:
                        yi_min = 0
                    
                    if (yi_max is not None):
                        yfi_ = []
                        if verbose:
                            if (yi_max<0):
                                even_print('yi_max', '%i / %i'%(yi_max,yi[yi_max]))
                            else:
                                even_print('yi_max', '%i'%(yi_max,))
                        for c in yfi:
                            if (yi_max<0) and (c<=(hf_src.ny+yi_max)):
                                yfi_.append(c)
                            elif (yi_max>=0) and (c<=yi_max):
                                yfi_.append(c)
                        yfi=np.array(yfi_, dtype=np.int64)
                    else:
                        yi_max = yi[-1]
                    
                    ## check y
                    if ((yi[yi_max]-yi[yi_min]+1)<1):
                        raise ValueError('invalid yi range requested')
                    if (ry>(yi[yi_max]-yi[yi_min]+1)):
                        raise ValueError('more ranks than grid points in y')
                    
                    if (zi_min is not None):
                        zfi_ = []
                        if verbose:
                            if (zi_min<0):
                                even_print('zi_min', '%i / %i'%(zi_min,zi[zi_min]))
                            else:
                                even_print('zi_min', '%i'%(zi_min,))
                        for c in zfi:
                            if (zi_min<0) and (c>=(hf_src.nz+zi_min)):
                                zfi_.append(c)
                            elif (zi_min>=0) and (c>=zi_min):
                                zfi_.append(c)
                        zfi=np.array(zfi_, dtype=np.int64)
                    else:
                        zi_min = 0
                    
                    if (zi_max is not None):
                        zfi_ = []
                        if verbose:
                            if (zi_max<0):
                                even_print('zi_max', '%i / %i'%(zi_max,zi[zi_max]))
                            else:
                                even_print('zi_max', '%i'%(zi_max,))
                        for c in zfi:
                            if (zi_max<0) and (c<=(hf_src.nz+zi_max)):
                                zfi_.append(c)
                            elif (zi_max>=0) and (c<=zi_max):
                                zfi_.append(c)
                        zfi=np.array(zfi_, dtype=np.int64)
                    else:
                        zi_max = zi[-1]
                    
                    ## check z
                    if ((zi[zi_max]-zi[zi_min]+1)<1):
                        raise ValueError('invalid zi range requested')
                    if (rz>(zi[zi_max]-zi[zi_min]+1)):
                        raise ValueError('more ranks than grid points in z')
                    
                    if (ti_min is not None):
                        tfi_ = []
                        if verbose:
                            if (ti_min<0):
                                even_print('ti_min', '%i / %i'%(ti_min,ti[ti_min]))
                            else:
                                even_print('ti_min', '%i'%(ti_min,))
                        for c in tfi:
                            if (ti_min<0) and (c>=(hf_src.nt+ti_min)):
                                tfi_.append(c)
                            elif (ti_min>=0) and (c>=ti_min):
                                tfi_.append(c)
                        tfi=np.array(tfi_, dtype=np.int64)
                    else:
                        ti_min = 0
                    
                    if (ti_max is not None):
                        tfi_ = []
                        if verbose:
                            if (ti_max<0):
                                even_print('ti_max', '%i / %i'%(ti_max,ti[ti_max]))
                            else:
                                even_print('ti_max', '%i'%(ti_max,))
                        for c in tfi:
                            if (ti_max<0) and (c<=(hf_src.nt+ti_max)):
                                tfi_.append(c)
                            elif (ti_max>=0) and (c<=ti_max):
                                tfi_.append(c)
                        tfi=np.array(tfi_, dtype=np.int64)
                    else:
                        ti_max = ti[-1]
                    
                    ## check t
                    if ((ti[ti_max]-ti[ti_min]+1)<1):
                        raise ValueError('invalid ti range requested')
                    if (ct>(ti[ti_max]-ti[ti_min]+1)):
                        raise ValueError('more chunks than timesteps')
                
                # === 3D/4D communicator
                
                comm4d = hf_src.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                t4d = comm4d.Get_coords(rank)
                
                rxl_ = np.array_split(xfi,rx)
                ryl_ = np.array_split(yfi,ry)
                rzl_ = np.array_split(zfi,rz)
                #rtl_ = np.array_split(tfi,rt)
                
                rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                
                ## the rank-local bounds for READ --> takes into acct clip but not step!
                rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
                
                ## the global dim sizes for READ
                nx_read = xfi.shape[0]
                ny_read = yfi.shape[0]
                nz_read = zfi.shape[0]
                
                # === global step
                
                ## take every nth index (of the already bounds-clipped) index-to-take vector
                xfi = np.copy(xfi[::xi_step])
                yfi = np.copy(yfi[::yi_step])
                zfi = np.copy(zfi[::zi_step])
                
                ## the global dim sizes for WRITE
                nx = xfi.shape[0]
                ny = yfi.shape[0]
                nz = zfi.shape[0]
                
                # ===
                
                ## grid for target file (rectilinear case)
                # x = np.copy(x[xfi]) 
                # y = np.copy(y[yfi])
                # z = np.copy(z[zfi])
                # nx = x.shape[0]
                # ny = y.shape[0]
                # nz = z.shape[0]
                
                t = np.copy(t[tfi])
                nt = t.shape[0]
                
                # ===
                
                if verbose:
                    even_print('fn_cgd_tgt' , fn_cgd_tgt )
                    even_print('nx' , '%i'%nx )
                    even_print('ny' , '%i'%ny )
                    even_print('nz' , '%i'%nz )
                    even_print('nt' , '%i'%nt )
                    print(72*'-')
                
                if ('dims/x' in hf_tgt):
                    del hf_tgt['dims/x']
                if ('dims/y' in hf_tgt):
                    del hf_tgt['dims/y']
                if ('dims/z' in hf_tgt):
                    del hf_tgt['dims/z']
                if ('dims/t' in hf_tgt):
                    del hf_tgt['dims/t']
                
                dset = hf_src['dims/x']
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                
                if (prec_coords=='same'):
                    pass
                elif (prec_coords=='single'):
                    if (dtype!=np.float32) and (dtype!=np.float64):
                        raise ValueError
                    dtype = np.dtype(np.float32)
                    float_bytes = dtype.itemsize
                else:
                    raise ValueError
                
                shape  = (nz,ny,nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=2, itemsize=float_bytes)
                
                for dd in ['x','y','z']:
                    
                    dsn = f'dims/{dd}'
                    data_gb = float_bytes * nx * ny * nz / 1024**3
                    
                    if verbose:
                        even_print(f'initializing {dsn}','%0.1f [GB]'%(data_gb,))
                    dset = hf_tgt.create_dataset( dsn, 
                                                  shape=shape, 
                                                  dtype=dtype,
                                                  chunks=chunks )
                    
                    chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                    if verbose:
                        even_print('chunk shape (z,y,x)','%s'%str(dset.chunks))
                        even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if verbose: print(72*'-')
                
                # === clip and update [dims/<>] & [csys/<>] dsets
                
                if True:
                    
                    dsn = 'dims/stang'
                    if (dsn in hf_tgt):
                        data = np.copy(hf_tgt[dsn][()])
                        shape_orig = data.shape
                        ni = data.shape[0]
                        if (ni!=hf_src.nx):
                            print(f'cgd.copy() : something is wrong with {dsn}')
                            MPI.COMM_WORLD.Abort(1)
                        del hf_tgt[dsn]
                        data = np.copy(data[xfi])
                        hf_tgt.comm.Barrier()
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,f'{str(shape_orig)} --> {str(data.shape)}')
                    
                    dsn = 'dims/snorm'
                    if (dsn in hf_tgt):
                        data = np.copy(hf_tgt[dsn][()])
                        shape_orig = data.shape
                        ni = data.shape[0]
                        if (ni!=hf_src.ny):
                            print(f'cgd.copy() : something is wrong with {dsn}')
                            MPI.COMM_WORLD.Abort(1)
                        del hf_tgt[dsn]
                        data = np.copy(data[yfi])
                        hf_tgt.comm.Barrier()
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,f'{str(shape_orig)} --> {str(data.shape)}')
                    
                    dsn = 'csys/vtang'
                    if (dsn in hf_tgt):
                        data = np.copy(hf_tgt[dsn][()])
                        shape_orig = data.shape
                        if (data.shape!=(hf_src.nx,hf_src.ny,2)):
                            print(f'cgd.copy() : something is wrong with {dsn}')
                            MPI.COMM_WORLD.Abort(1)
                        del hf_tgt[dsn]
                        data = np.copy( data[ np.ix_(xfi,yfi,np.arange(2,dtype=np.int64)) ] )
                        hf_tgt.comm.Barrier()
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,f'{str(shape_orig)} --> {str(data.shape)}')
                    
                    dsn = 'csys/vnorm'
                    if (dsn in hf_tgt):
                        data = np.copy(hf_tgt[dsn][()])
                        shape_orig = data.shape
                        if (data.shape!=(hf_src.nx,hf_src.ny,2)):
                            print(f'cgd.copy() : something is wrong with {dsn}')
                            MPI.COMM_WORLD.Abort(1)
                        del hf_tgt[dsn]
                        data = np.copy( data[ np.ix_(xfi,yfi,np.arange(2,dtype=np.int64)) ] )
                        hf_tgt.comm.Barrier()
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,f'{str(shape_orig)} --> {str(data.shape)}')
                    
                    # ## copy over [data_dim/<>] dsets if present
                    # if ('data_dim' in hf_tgt):
                    #     for dsn in hf_tgt['data_dim'].keys():
                    #         print(f'WARNING: {dsn} has not been clipped')
                    
                    hf_tgt.flush()
                    if verbose: print(72*'-')
                    
                    hf_tgt.comm.Barrier()
                
                # === bounds for outfile WRITE
                
                xiw     = np.array( [ i for i in xfi if all([(i>=rx1),(i<rx2)]) ], dtype=np.int32 ) ## the global indices in my local rank, taking into acct clip AND step
                nxiw    = xiw.shape[0]
                xiw_off = len([ i for i in xfi if (i<rx1) ]) ## this rank's left offset in the OUTFILE context
                rx1w    = xiw_off
                rx2w    = xiw_off + nxiw
                
                yiw     = np.array( [ i for i in yfi if all([(i>=ry1),(i<ry2)]) ], dtype=np.int32 )
                nyiw    = yiw.shape[0]
                yiw_off = len([ i for i in yfi if (i<ry1) ])
                ry1w    = yiw_off
                ry2w    = yiw_off + nyiw
                
                ziw     = np.array( [ i for i in zfi if all([(i>=rz1),(i<rz2)]) ], dtype=np.int32 )
                nziw    = ziw.shape[0]
                ziw_off = len([ i for i in zfi if (i<rz1) ])
                rz1w    = ziw_off
                rz2w    = ziw_off + nziw
                
                ## xiw,yiw,ziw are used to 'filter' the rank-local data that is read in
                ## xiw,yiw,ziw are currently in the global context, so we need to subtract off the left READ bound
                ## which is NOT just the min xiw
                xiw -= rx1
                yiw -= ry1
                ziw -= rz1
                
                # ===
                
                ## time 'chunks' split (number of timesteps to read / write at a time)
                ctl_ = np.array_split(tfi,ct)
                ctl = [[b[0],b[-1]+1] for b in ctl_ ]
                
                shape  = (nt,nz,ny,nx) ## target
                hf_tgt.scalars = []
                
                ## initialize scalar datasets
                t_start = timeit.default_timer()
                for scalar in hf_src.scalars:
                    
                    dtype = hf_src.scalars_dtypes_dict[scalar]
                    float_bytes = dtype.itemsize
                    #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=float_bytes)
                    chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                    data_gb = float_bytes * nx * ny * nz * nt / 1024**3
                    
                    if (scalar in scalars):
                        if verbose:
                            even_print('initializing data/%s'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        dset = hf_tgt.create_dataset('data/%s'%scalar,
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_tgt.scalars.append(scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                t_initialize = timeit.default_timer() - t_start
                if verbose:
                    even_print('time initialize',format_time_string(t_initialize))
                    print(72*'-')
                
                # ===
                
                data_gb_read  = 0.
                data_gb_write = 0.
                t_read  = 0.
                t_write = 0.
                
                # === copy over grid (requires collective for CGD)
                
                for scalar in ['x','y','z']:
                    
                    dset_src = hf_src[f'dims/{scalar}']
                    dset_tgt = hf_tgt[f'dims/{scalar}']
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    ## read
                    hf_src.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset_src.collective:
                        data = np.copy( dset_src[rz1:rz2,ry1:ry2,rx1:rx2].T )
                    hf_src.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                    data_gb = float_bytes * nx_read * ny_read * nz_read / 1024**3
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    if verbose:
                        tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    try:
                        data_out = np.copy( data[ np.ix_(xiw,yiw,ziw) ] )
                        #data_out = np.copy( data[ xiw[:,np.newaxis,np.newaxis], yiw[np.newaxis,:,np.newaxis], ziw[np.newaxis,np.newaxis,:] ] )
                    except:
                        print('cgd.copy() : error in xiw,yiw,ziw')
                        MPI.COMM_WORLD.Abort(1)
                    
                    dtype = dset_tgt.dtype
                    float_bytes = dtype.itemsize
                    
                    if ( dset_tgt.dtype != data_out.dtype):
                        data_out = np.copy( data_out.astype(dset_tgt.dtype) )
                    
                    ## write
                    hf_tgt.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset_tgt.collective:
                        dset_tgt[rz1w:rz2w,ry1w:ry2w,rx1w:rx2w] = data_out.T
                    hf_tgt.flush()
                    hf_tgt.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                    data_gb = float_bytes*nx*ny*nz / 1024**3
                    
                    t_write       += t_delta
                    data_gb_write += data_gb
                    
                    if verbose:
                        even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                # === copy over [t] (independent is fine)
                
                hf_tgt.create_dataset('dims/t', data=t, dtype=np.float64, chunks=None)
                
                # ===
                
                hf_tgt.n_scalars = len(hf_tgt.scalars)
                
                # === copy scalar datasets
                
                if verbose:
                    progress_bar = tqdm(total=len(ctl)*hf_tgt.n_scalars, ncols=100, desc='copy', leave=False, file=sys.stdout)
                
                for scalar in hf_tgt.scalars:
                    dset_src = hf_src[f'data/{scalar}']
                    dset_tgt = hf_tgt[f'data/{scalar}']
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    for ctl_ in ctl:
                        
                        ct1, ct2 = ctl_
                        
                        ct1_ = ct1 - ti[ti_min] ## coords in target file
                        ct2_ = ct2 - ti[ti_min]
                        
                        ## read
                        hf_src.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_src.collective:
                            data = dset_src[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                        hf_src.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                        data_gb = float_bytes * nx_read * ny_read * nz_read * (ct2-ct1) / 1024**3
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        if verbose:
                            tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        try:
                            data_out = np.copy( data[ np.ix_(xiw,yiw,ziw) ] )
                            #data_out = np.copy( data[ xiw[:,np.newaxis,np.newaxis], yiw[np.newaxis,:,np.newaxis], ziw[np.newaxis,np.newaxis,:] ] )
                        except:
                            print('cgd.copy() : error in xiw,yiw,ziw')
                            MPI.COMM_WORLD.Abort(1)
                        
                        ## write
                        hf_tgt.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_tgt.collective:
                            dset_tgt[ct1_:ct2_,rz1w:rz2w,ry1w:ry2w,rx1w:rx2w] = data_out.T
                        hf_tgt.flush()
                        hf_tgt.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                        data_gb = float_bytes*nx*ny*nz*(ct2-ct1) / 1024**3
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            tqdm.write(even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        if verbose:
                            progress_bar.update()
                
                if verbose:
                    progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: even_print('time initialize',format_time_string(t_initialize))
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_cgd_src, '%0.2f [GB]'%(os.path.getsize(fn_cgd_src)/1024**3))
        if verbose: even_print(fn_cgd_tgt, '%0.2f [GB]'%(os.path.getsize(fn_cgd_tgt)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.copy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def read(self,**kwargs):
        '''
        read data from file & return structured array
        '''
        
        verbose_master = kwargs.get('verbose',True)
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose:
            verbose = verbose_master
        
        if verbose: print('\n'+'cgd.read()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print(self.fname,'%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: print(72*'-')
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        rt       = kwargs.get('rt',1)
        scalars_to_read = kwargs.get('scalars',None)
        
        if (rx*ry*rz*rt!=self.n_ranks):
            raise AssertionError('rx*ry*rz*rt!=self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if (scalars_to_read is None):
            scalars_to_read = self.scalars
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d    = comm4d.Get_coords(self.rank)
            
            rxl_   = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_   = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_   = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            rtl_   = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            rxl    = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl    = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl    = [[b[0],b[-1]+1] for b in rzl_ ]
            rtl    = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            ntr = self.nt
        
        t_read = 0.
        data_gb_read = 0.
        
        data_gb = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        
        # ===
        
        names   = [ s for s in scalars_to_read if s in self.scalars ]
        formats = [ self.scalars_dtypes_dict[n] for n in names ]
        
        ## 5D [scalar][x,y,z,t] structured array
        data = np.zeros(shape=(nxr,nyr,nzr,ntr), dtype={'names':names, 'formats':formats})
        
        for scalar in names:
            
            # === collective read
            dset = self['data/%s'%scalar]
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi: 
                with dset.collective:
                    data[scalar] = dset[rt1:rt2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                data[scalar] = dset[()].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            
            if verbose:
                even_print( 'read: %s'%scalar , '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)) )
            
            t_read       += t_delta
            data_gb_read += data_gb
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('cgd.read()', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: print(72*'-')
        
        return data
    
    # === test data populators
    
    def make_test_file(self, **kwargs):
        '''
        make a test CGD file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.make_test_file()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        self.nx = nx = kwargs.get('nx',128)
        self.ny = ny = kwargs.get('ny',128)
        self.nz = nz = kwargs.get('nz',128)
        self.nt = nt = kwargs.get('nt',128)
        
        data_gb = 3 * 4*nx*ny*nz*nt / 1024.**3
        #data_gb = 1 * 4*nx*ny*nz*nt / 1024.**3
        
        if verbose: even_print(self.fname, '%0.2f [GB]'%(data_gb,))
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d    = comm4d.Get_coords(self.rank)
            
            rxl_   = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_   = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_   = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            rtl_   = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl    = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl    = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl    = [[b[0],b[-1]+1] for b in rzl_ ]
            rtl    = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]
            ry1, ry2 = ryl[t4d[1]]
            rz1, rz2 = rzl[t4d[2]]
            rt1, rt2 = rtl[t4d[3]]
            
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
            ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            ntr = self.nt
        
        # ===
        
        ## 1D coordinates
        x = np.linspace(0., 2*np.pi, nx, dtype=np.float64)
        y = np.linspace(0., 2*np.pi, ny, dtype=np.float64)
        z = np.linspace(0., 2*np.pi, nz, dtype=np.float64)
        t = 0.1 * np.arange(nt, dtype=np.float64)
        
        ## 3D coordinates
        x,y,z = np.meshgrid(x,y,z, indexing='ij')
        
        ## per-rank dim range
        if self.usingmpi:
            xr = np.copy( x[rx1:rx2,ry1:ry2,rz1:rz2] )
            yr = np.copy( y[rx1:rx2,ry1:ry2,rz1:rz2] )
            zr = np.copy( z[rx1:rx2,ry1:ry2,rz1:rz2] )
            #tr = t[rt1:rt2]
            tr = np.copy(t)
        else:
            xr = np.copy(x)
            yr = np.copy(y)
            zr = np.copy(z)
            tr = np.copy(t)
        
        # === apply deformation to mesh
        
        x_ = np.copy(x)
        y_ = np.copy(y)
        z_ = np.copy(z)
        
        if False: ## distort grid in [x,y] (2D)
            
            ## xy
            x += 0.2*np.sin(1*y_)
            #x += 0.2*np.sin(1*z_)
            
            ## yz
            #y += 0.2*np.sin(1*z_)
            y += 0.2*np.sin(1*x_)
            
            ## zy
            #z += 0.2*np.sin(1*x_)
            #z += 0.2*np.sin(1*y_)
        
        if True: ## distort grid in [x,y,z] (3D)
            
            ## xy
            x += 0.2*np.sin(1*y_)
            x += 0.2*np.sin(1*z_)
            
            ## yz
            y += 0.2*np.sin(1*z_)
            y += 0.2*np.sin(1*x_)
            
            ## zy
            z += 0.2*np.sin(1*x_)
            z += 0.2*np.sin(1*y_)
        
        x_ = None; del x_
        y_ = None; del y_
        z_ = None; del z_
        
        # self.x = x
        # self.y = y
        # self.z = z
        
        ## chunk info for coord arrays
        shape  = (nz,ny,nx)
        chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=2, itemsize=8)
        
        ## initialize coordinate datasets
        data_gb = 4*nx*ny*nz / 1024.**3
        for scalar in ['x','y','z']:
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            if verbose:
                even_print('initializing dims/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            dset = self.create_dataset('dims/%s'%scalar, 
                                        shape=shape,
                                        dtype=np.float64,
                                        chunks=chunks )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === write coordinate datasets (collective)
        
        data_gb = 8*nx*ny*nz / 1024.**3
        
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['dims/x']
        if self.usingmpi:
            with ds.collective:
                #ds[rz1:rz2,ry1:ry2,rx1:rx2] = x[rx1:rx2,ry1:ry2,rz1:rz2].T
                ds[rz1:rz2,ry1:ry2,rx1:rx2] = xr.T
        else:
            ds[:,:,:] = x.T
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: dims/x','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['dims/y']
        if self.usingmpi:
            with ds.collective:
                #ds[rz1:rz2,ry1:ry2,rx1:rx2] = y[rx1:rx2,ry1:ry2,rz1:rz2].T
                ds[rz1:rz2,ry1:ry2,rx1:rx2] = yr.T
        else:
            ds[:,:,:] = y.T
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: dims/y','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['dims/z']
        if self.usingmpi:
            with ds.collective:
                #ds[rz1:rz2,ry1:ry2,rx1:rx2] = z[rx1:rx2,ry1:ry2,rz1:rz2].T
                ds[rz1:rz2,ry1:ry2,rx1:rx2] = zr.T
        else:
            ds[:,:,:] = z.T
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: dims/z','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        if verbose: print(72*'-')
        
        # ===
        
        ## chunk info for scalar arrays
        shape  = (self.nt,self.nz,self.ny,self.nx)
        float_bytes = 4
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        self.scalars = ['u','v','w']
        #self.scalars = ['u']
        self.scalars_dtypes = [ np.float32 for s in self.scalars ]
        
        ## initialize datasets
        data_gb = 4*nx*ny*nz*nt / 1024.**3
        self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        for scalar in self.scalars:
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            if verbose:
                even_print('initializing data/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            dset = self.create_dataset('data/%s'%scalar, 
                                        shape=shape,
                                        dtype=np.float32,
                                        chunks=chunks )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        self.usingmpi: self.comm.Barrier()
        t_initialize = timeit.default_timer() - t_start
        
        if verbose: print(72*'-')
        
        # ===
        
        ## 5D [scalar][x,y,z,t] structured array --> data buffer
        data = np.zeros(shape=(nxr,nyr,nzr,ntr), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
        
        self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        
        if False: ## ABC Flow
            
            A = np.sqrt(3)
            B = np.sqrt(2)
            C = 1.
            na = np.newaxis
            data['u'] = (A + 0.5 * tr[na,na,na,:] * np.sin(np.pi*tr[na,na,na,:])) * np.sin(zr[:,:,:,na]) + \
                        B * np.cos(yr[:,:,:,na]) + \
                        0.*xr[:,:,:,na]
            data['v'] = B * np.sin(xr[:,:,:,na]) + \
                        C * np.cos(zr[:,:,:,na]) + \
                        0.*yr[:,:,:,na] + \
                        0.*tr[na,na,na,:]
            data['w'] = C * np.sin(yr[:,:,:,na]) + \
                        (A + 0.5 * tr[na,na,na,:] * np.sin(np.pi*tr[na,na,na,:])) * np.cos(xr[:,:,:,na]) + \
                        0.*zr[:,:,:,na]
        
        if True: ## random data
            
            rng = np.random.default_rng(seed=self.rank)
            for scalar in self.scalars:
                data[scalar] = rng.uniform(-1, +1, size=(nxr,nyr,nzr,ntr)).astype(np.float32)
        
        self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('gen data','%0.3f [s]'%(t_delta,))
        
        ## write data
        data_gb_write = 0.
        t_write = 0.
        for scalar in self.scalars:
            ds = self['data/%s'%scalar]
            self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with ds.collective:
                    ds[rt1:rt2,rz1:rz2,ry1:ry2,rx1:rx2] = data[scalar].T
            else:
                ds[:,:,:,:] = data[scalar].T
            self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4*nx*ny*nz*nt / 1024**3
            
            t_write       += t_delta
            data_gb_write += data_gb
            
            if verbose:
                even_print('write: %s'%(scalar,), '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('time initialize',format_time_string(t_initialize))
        if verbose: even_print('write total', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: print(72*'-')
        if verbose: print('total time :  cgd.make_test_file() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing
    
    def get_mean_legacy(self, **kwargs):
        '''
        get mean in [t] --> leaves [x,y,z,1]
        --> save to new CGD file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.get_mean()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        #rt = 1
        
        fn_cgd_mean  = kwargs.get('fn_cgd_mean',None)
        #sfm         = kwargs.get('scalars',None) ## scalars to take (for mean)
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === mean file name (for writing)
        if (fn_cgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_cgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_cgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_cgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if verbose: even_print('fn_cgd'       , self.fname   )
        if verbose: even_print('fn_cgd_mean'  , fn_cgd_mean  )
        #if verbose: even_print('fn_cgd_prime' , fn_cgd_prime )
        if verbose: even_print('do Favre avg' , str(favre)   )
        if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        t_avg_start  = self.t[ti_for_avg[0]]
        t_avg_end    = self.t[ti_for_avg[-1]]
        duration_avg = t_avg_end - t_avg_start
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        if verbose: even_print('Δt','%0.2f [-]'%(dt0,))
        if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        itemsize = dtype.itemsize
        
        #data_gb      = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        data_gb      = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1      / 1024**3
        
        scalars_re = [ 'u','v','w','p','T','rho' ] #, 'utang','unorm', 'vort_x','vort_y','vort_z','vort_tang' ]
        scalars_fv = [ 'u','v','w','p','T','rho' ]
        
        #with cgd(fn_cgd_mean, 'w', force=force, driver='mpio', comm=MPI.COMM_WORLD) as hf_mean:
        with cgd(fn_cgd_mean, 'w', force=force, driver=self.driver, comm=self.comm) as hf_mean:
            
            ## initialize the mean file from the opened unsteady cgd file
            hf_mean.init_from_cgd(self.fname, rx=rx,ry=ry,rz=rz, chunk_kb=chunk_kb)
            
            ## set some top-level attributes
            hf_mean.attrs['duration_avg'] = duration_avg ## duration of mean
            #hf_mean.attrs['duration_avg'] = self.duration
            hf_mean.attrs['dt'] = dt0
            #hf_mean.attrs['fclass'] = 'cgd'
            hf_mean.attrs['fsubtype'] = 'mean'
            
            if verbose: print(72*'-')
            
            # === initialize mean datasets
            for scalar in self.scalars:
                
                shape  = (1,self.nz,self.ny,self.nx)
                #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        if ('data/%s'%scalar in hf_mean):
                            del hf_mean['data/%s'%scalar]
                        if (self.rank==0):
                            even_print('initializing data/%s'%(scalar,),'%0.3f [GB]'%(data_gb_mean,))
                        dset = hf_mean.create_dataset('data/%s'%scalar,
                                                    shape=shape,
                                                    dtype=dtype,
                                                    chunks=chunks,
                                                    )
                        hf_mean.scalars.append('data/%s'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    
                    if (scalar in scalars_fv):
                        if ('data/%s_fv'%scalar in hf_mean):
                            del hf_mean['data/%s_fv'%scalar]
                        if (self.rank==0):
                            even_print('initializing data/%s_fv'%(scalar,),'%0.3f [GB]'%(data_gb_mean,))
                        dset = hf_mean.create_dataset('data/%s_fv'%scalar,
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s_fv'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            # === read rho
            if favre:
                
                dset = self['data/rho']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi: 
                    with dset.collective:
                        #rho = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                        rho = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    #rho = dset[()].T
                    rho = dset[ti_min:,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                
                t_delta = timeit.default_timer() - t_start
                if (self.rank==0):
                    txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                    tqdm.write(txt)
                
                t_read       += t_delta
                data_gb_read += data_gb
                
                ## mean ρ in [t] --> leave [x,y,z]
                rho_mean = np.mean(rho, axis=-1, keepdims=True, dtype=np.float64).astype(np.float32)
            
            # === read, do mean, write
            for scalar in self.scalars:
                
                # === collective read
                dset = self['data/%s'%scalar]
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        #data = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                        data = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    #data = dset[()].T
                    data = dset[ti_min:,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                if (self.rank==0):
                    txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                    tqdm.write(txt)
                
                t_read       += t_delta
                data_gb_read += data_gb
                
                # === do mean in [t]
                if reynolds:
                    data_mean    = np.mean(data,     axis=-1, keepdims=True, dtype=np.float64).astype(np.float32)
                if favre:
                    data_mean_fv = np.mean(data*rho, axis=-1, keepdims=True, dtype=np.float64).astype(np.float32) / rho_mean
                
                # === write
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        dset = hf_mean['data/%s'%scalar]
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_mean.T
                        else:
                            dset[:,:,:,:] = data_mean.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if (self.rank==0):
                            txt = even_print('write: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
                
                if favre:
                    if scalar in scalars_fv:
                        
                        dset = hf_mean['data/%s_fv'%scalar]
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_mean_fv.T
                        else:
                            dset[:,:,:,:] = data_mean_fv.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if (self.rank==0):
                            txt = even_print('write: %s_fv'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
            
            if self.usingmpi: self.comm.Barrier()
            
            # === replace dims/t array --> take last time of series
            t = np.array([self.t[-1]],dtype=np.float64)
            if ('dims/t' in hf_mean):
                del hf_mean['dims/t']
            hf_mean.create_dataset('dims/t', data=t)
            
            if hasattr(hf_mean, 'duration_avg'):
                if verbose: even_print('duration avg', '%0.2f [-]'%hf_mean.duration_avg)
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_cgd_mean, '%0.2f [GB]'%(os.path.getsize(fn_cgd_mean)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.get_mean() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def get_mean(self, **kwargs):
        '''
        get mean in [t] --> leaves [x,y,z,1]
        --> save to new CGD file
        -----
        - this version uses accumulator buffers and does *(1/n) at end to calculate mean
        - this allows for low RAM usage, as the time dim can be sub-chunked (ct=N)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.get_mean()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_cgd_mean  = kwargs.get('fn_cgd_mean',None)
        #sfm         = kwargs.get('scalars',None) ## scalars to take (for mean)
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        
        ct           = kwargs.get('ct',1)
        
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 32 ) ## for initializing mean file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8  )
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === mean file name (for writing)
        if (fn_cgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_cgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_cgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_cgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if verbose: even_print('fn_cgd'       , self.fname   )
        if verbose: even_print('fn_cgd_mean'  , fn_cgd_mean  )
        #if verbose: even_print('fn_cgd_prime' , fn_cgd_prime )
        if verbose: even_print('do Favre avg' , str(favre)   )
        if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        t_avg_start  = self.t[ti_for_avg[0]]
        t_avg_end    = self.t[ti_for_avg[-1]]
        duration_avg = t_avg_end - t_avg_start
        
        #if not isinstance(ct, (int,np.int32,np.int64)):
        if not isinstance(ct, int):
            raise ValueError
        if (ct<1):
            raise ValueError
        
        ## [t] sub chunk range
        ctl_ = np.array_split( ti_for_avg, min(ct,nt_avg) )
        ctl = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## check that no sub ranges are <=1
        for a_ in [ ctl_[1]-ctl_[0] for ctl_ in ctl ]:
            if (a_ <= 1):
                raise ValueError('at least one rank sub range has zero length')
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        if verbose: even_print('Δt','%0.2f [-]'%(dt0,))
        #if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        # dtype = np.dtype(np.float32)
        # float_bytes = dtype.itemsize
        # itemsize = dtype.itemsize
        
        ##data_gb      = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        #data_gb      = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
        #data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1      / 1024**3
        
        scalars_re = [ 'u','v','w','p','T','rho' ] # 'vort_x','vort_y','vort_z','vort_tang' ]
        scalars_fv = ['u','v','w','T', 'utang','unorm' ] ## 'p','rho'
        
        ## do a loop through to get names
        scalars_mean_names  = []
        scalars_mean_dtypes = []
        for scalar in self.scalars:
            dtype = self.scalars_dtypes_dict[scalar]
            if reynolds:
                if True: ## always
                    sc_name = scalar
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
            if favre:
                if (scalar in scalars_fv):
                    sc_name = f'r_{scalar}'
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
        
        #with cgd(fn_cgd_mean, 'w', force=force, driver='mpio', comm=MPI.COMM_WORLD) as hf_mean:
        with cgd(fn_cgd_mean, 'w', force=force, driver=self.driver, comm=self.comm, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_mean:
            
            ## initialize the mean file from the opened unsteady cgd file
            hf_mean.init_from_cgd(self.fname, rx=rx,ry=ry,rz=rz, chunk_kb=chunk_kb)
            
            ## set some top-level attributes
            hf_mean.attrs['duration_avg'] = duration_avg ## duration of mean
            #hf_mean.attrs['duration_avg'] = self.duration
            hf_mean.attrs['dt'] = dt0
            #hf_mean.attrs['fclass'] = 'cgd'
            hf_mean.attrs['fsubtype'] = 'mean'
            
            if verbose: print(72*'-')
            
            # === initialize mean datasets
            for scalar in self.scalars:
                
                dtype = self.scalars_dtypes_dict[scalar]
                float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                
                data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                
                shape  = (1,self.nz,self.ny,self.nx)
                #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        if ('data/%s'%scalar in hf_mean):
                            del hf_mean['data/%s'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    
                    if (scalar in scalars_fv):
                        if ('data/%s_fv'%scalar in hf_mean):
                            del hf_mean['data/%s_fv'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}_fv' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}_fv',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s_fv'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            ## accumulator array for local rank --> initialize
            data_sum = np.zeros(shape=(nxr,nyr,nzr,1), dtype={'names':scalars_mean_names, 'formats':[np.float64 for _ in scalars_mean_names]})
            
            # === main loop
            
            if verbose:
                progress_bar = tqdm(total=ct, ncols=100, desc='mean', leave=False, file=sys.stdout)
            
            ct_counter=0
            for ctl_ in ctl:
                ct_counter += 1
                ct1, ct2 = ctl_
                ntc = ct2 - ct1
                
                if (ct>1):
                    if verbose:
                        mesg = f'[t] sub chunk {ct_counter:d}/{ct:d}'
                        tqdm.write( mesg )
                        tqdm.write( '-'*len(mesg) )
                
                # === read rho
                if favre:
                    
                    dset = self['data/rho']
                    
                    dtype = self.scalars_dtypes_dict['rho']
                    if (dtype!=dset.dtype):
                        raise ValueError
                    float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if self.usingmpi: 
                        with dset.collective:
                            ##rho = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            #rho = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            rho = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                    else:
                        #rho = dset[()].T
                        #rho = dset[ti_min:,:,:,:].T
                        rho = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                    data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                    
                    if verbose:
                        txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    # ## mean ρ in [t] --> leave [x,y,z]
                    # rho_mean = np.mean(rho, axis=-1, keepdims=True, dtype=np.float64).astype(np.float32)
                
                # === read data, sum
                for scalar in self.scalars:
                    
                    dset = self[f'data/{scalar}']
                    dtype = self.scalars_dtypes_dict[scalar]
                    if (dtype!=dset.dtype):
                        raise ValueError
                    float_bytes = dtype.itemsize
                    
                    # === collective read
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if self.usingmpi:
                        with dset.collective:
                            ##data = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            #data = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            data = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                    else:
                        ##data = dset[()].T
                        #data = dset[ti_min:,:,:,:].T
                        data = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                    data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                    
                    if verbose:
                        txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    # === do sum, add to accumulator
                    if reynolds:
                        sc_name = scalar
                        data_sum[sc_name] += np.sum(data, axis=-1, dtype=np.float64, keepdims=True)
                    if favre:
                        if (scalar in scalars_fv):
                            sc_name = f'r_{scalar}'
                            data_sum[sc_name] += np.sum(data*rho, axis=-1, dtype=np.float64, keepdims=True)
                    
                    if self.usingmpi: self.comm.Barrier()
                
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                
                if verbose: progress_bar.update()
                if verbose: tqdm.write(72*'-')
            if verbose: progress_bar.close()
            
            # ==========================================================
            # multiply accumulators by (1/n)
            # ==========================================================
            
            for scalar in self.scalars:
                if reynolds:
                    sc_name = scalar
                    data_sum[sc_name] *= (1/nt_avg)
                if favre:
                    if (scalar in scalars_fv):
                        sc_name = f'r_{scalar}'
                        data_sum[sc_name] *= (1/nt_avg)
            
            # ==========================================================
            # 'data_sum' now contains averages, not sums!
            # ==========================================================
            
            ## Favre avg : φ_tilde = avg[ρ·φ]/avg[ρ]
            rho_mean = np.copy( data_sum['rho'] )
            
            # === write
            for scalar in self.scalars:
                
                if reynolds:
                    
                    dset = hf_mean[f'data/{scalar}']
                    dtype = dset.dtype
                    float_bytes = dtype.itemsize
                    
                    if (dtype==np.float32):
                        data_out = np.copy( data_sum[scalar].astype(np.float32) )
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                    else:
                        dset[:,:,:,:] = data_out.T
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                    
                    if verbose:
                        txt = even_print(f'write: {scalar}', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_write       += t_delta
                    data_gb_write += data_gb_mean
                
                if favre:
                    if (scalar in scalars_fv):
                        
                        dset = hf_mean[f'data/{scalar}_fv']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        
                        ## φ_tilde = avg[ρ·φ]/avg[ρ]
                        data_out = np.copy( data_sum[f'r_{scalar}'] / rho_mean )
                        
                        if (dtype==np.float32):
                            data_out = np.copy( data_out.astype(np.float32) )
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                        else:
                            dset[:,:,:,:] = data_out.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                        
                        if verbose:
                            txt = even_print(f'write: {scalar}_fv', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
            
            # === replace dims/t array --> take last time of series
            t = np.array([self.t[-1]],dtype=np.float64)
            if ('dims/t' in hf_mean):
                del hf_mean['dims/t']
            hf_mean.create_dataset('dims/t', data=t)
            
            if hasattr(hf_mean, 'duration_avg'):
                if verbose: even_print('duration avg', '%0.2f [-]'%hf_mean.duration_avg)
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_cgd_mean, '%0.2f [GB]'%(os.path.getsize(fn_cgd_mean)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.get_mean() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def add_mean_dimensional_data_xpln(self, **kwargs):
        '''
        get dimensionalized mean data for [x]/[s1] plane
        --> save to existing CGD file with fsubtype=mean
        - assumes volume which is thin in [x]/[s1] direction
        - a CGD which is the output of cgd.get_mean() should be opened here
        - not parallel
        '''
        
        verbose = kwargs.get('verbose',True)
        epsilon = kwargs.get('epsilon',5e-4)
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.add_mean_dimensional_data_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## this func is not parallel
        if self.usingmpi:
            raise NotImplementedError
        
        ## 'r' and 'w' open modes are not allowed
        if not (self.open_mode=='a') or (self.open_mode=='r+'):
            raise ValueError(f'open mode is {self.open_mode}')
        
        ## assert that this is a mean flow file ( i.e. output from cgd.get_mean() )
        if (self.fsubtype!='mean'):
            print(self.fsubtype)
            raise ValueError
        
        ## assert that 'data/utang' and 'data/unorm' are present
        if not ('data/utang' in self):
            raise ValueError
        if not ('data/unorm' in self):
            raise ValueError
        
        ## assert that 'dims/stang' and 'data/snorm' are present
        if not ('dims/stang' in self):
            raise ValueError
        if not ('dims/snorm' in self):
            raise ValueError
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## read in 3D coordinate arrays, then dimensionalize [m]
        x = np.copy( self['dims/x'][()].T * self.lchar )
        y = np.copy( self['dims/y'][()].T * self.lchar )
        z = np.copy( self['dims/z'][()].T * self.lchar )
        nx = self.nx
        ny = self.ny
        nz = self.nz
        
        ## read in 1D coordinate arrays, then dimensinoalize [m]
        stang_ = np.copy(self['dims/stang'])
        stang  = np.copy( stang_ * self.lchar ) ## dimensional [m]
        
        snorm_ = np.copy(self['dims/snorm'])
        snorm  = np.copy( snorm_ * self.lchar ) ## dimensional [m]
        
        ## assert [z] is same over all [x,y]
        if (z.ndim!=3):
            raise ValueError
        z1d = np.copy(z[0,0,:])
        for i in range(nx):
            for j in range(ny):
                np.testing.assert_allclose(z1d, z[i,j,:], rtol=1e-14, atol=1e-14)
        
        ## assert [x,y] is same over all [z]
        x2d  = np.copy(x[:,:,0])
        y2d  = np.copy(y[:,:,0])
        xy2d = np.stack((x2d,y2d), axis=-1)
        for k in range(nz):
            x2d_  = np.copy(x[:,:,k])
            y2d_  = np.copy(y[:,:,k])
            xy2d_ = np.stack((x2d_,y2d_), axis=-1)
            np.testing.assert_allclose(xy2d, xy2d_, rtol=1e-14, atol=1e-14)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## check if mean cgd has attr 'dt'
        if ('dt' in self.attrs.keys()):
            dt = self.attrs['dt']
            if (dt is not None):
                dt *= self.tchar
        else:
            raise ValueError
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration_avg,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration_avg*self.tchar,))
        if verbose: print(72*'-')
        
        u   =  self.U_inf                     * np.copy( self['data/u'][()].T   )
        v   =  self.U_inf                     * np.copy( self['data/v'][()].T   )
        w   =  self.U_inf                     * np.copy( self['data/w'][()].T   )
        rho =  self.rho_inf                   * np.copy( self['data/rho'][()].T )
        p   =  (self.rho_inf * self.U_inf**2) * np.copy( self['data/p'][()].T   )
        T   =  self.T_inf                     * np.copy( self['data/T'][()].T   )
        
        utang = self.U_inf * np.copy( self['data/utang'][()].T )
        unorm = self.U_inf * np.copy( self['data/unorm'][()].T )
        
        # mu1 = (14.58e-7 * T**1.5) / ( T + 110.4 )
        # mu2 = self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * (self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth)
        # mu3 = self.C_Suth * T**(3/2) / (T + self.S_Suth)
        # np.testing.assert_allclose(mu1, mu2, rtol=1e-14, atol=1e-14)
        # np.testing.assert_allclose(mu2, mu3, rtol=1e-14, atol=1e-14)
        # mu = np.copy(mu3)
        
        mu = self.C_Suth * T**(3/2) / (T + self.S_Suth)
        nu = mu / rho
        
        # === average in [z]/[s3] --> leave 2D [x,y]/[s1,s2]
        
        u     = np.squeeze( np.mean( u     , axis=2, dtype=np.float64).astype(np.float32) )
        v     = np.squeeze( np.mean( v     , axis=2, dtype=np.float64).astype(np.float32) )
        w     = np.squeeze( np.mean( w     , axis=2, dtype=np.float64).astype(np.float32) )
        rho   = np.squeeze( np.mean( rho   , axis=2, dtype=np.float64).astype(np.float32) )
        p     = np.squeeze( np.mean( p     , axis=2, dtype=np.float64).astype(np.float32) )
        T     = np.squeeze( np.mean( T     , axis=2, dtype=np.float64).astype(np.float32) )
        utang = np.squeeze( np.mean( utang , axis=2, dtype=np.float64).astype(np.float32) )
        unorm = np.squeeze( np.mean( unorm , axis=2, dtype=np.float64).astype(np.float32) )
        mu    = np.squeeze( np.mean( mu    , axis=2, dtype=np.float64).astype(np.float32) )
        nu    = np.squeeze( np.mean( nu    , axis=2, dtype=np.float64).astype(np.float32) )
        
        # === get 2D metric tensor
        
        if (nx<3):
            raise ValueError('dds1[] not possible because nx<3')
        elif (nx>=3) and (nx<5):
            acc = 2
        elif (nx>=5) and (nx<7):
            acc = 4
        elif (nx>=7):
            acc = 6
        else:
            raise ValueError('this should never happen')
        
        edge_stencil = 'half'
        if verbose: even_print('acc','%i'%acc)
        if verbose: even_print('edge_stencil',edge_stencil)
        
        M = get_metric_tensor_2d(x2d, y2d, acc=acc, edge_stencil=edge_stencil, no_warn=True, verbose=verbose)
        
        ddx_q1 = np.copy( M[:,:,0,0] ) ## ξ_x
        ddx_q2 = np.copy( M[:,:,1,0] ) ## η_x
        ddy_q1 = np.copy( M[:,:,0,1] ) ## ξ_y
        ddy_q2 = np.copy( M[:,:,1,1] ) ## η_y
        
        if verbose: even_print('ξ_x','%s'%str(ddx_q1.shape))
        if verbose: even_print('η_x','%s'%str(ddx_q2.shape))
        if verbose: even_print('ξ_y','%s'%str(ddy_q1.shape))
        if verbose: even_print('η_y','%s'%str(ddy_q2.shape))
        
        M = None; del M
        
        ## the 'computational' grid (unit Cartesian)
        #x_comp = np.arange(nx, dtype=np.float64)
        #y_comp = np.arange(ny, dtype=np.float64)
        x_comp = 1.
        y_comp = 1.
        
        # === get gradients & vort_z
        
        ddx_u_comp = gradient(u, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddy_u_comp = gradient(u, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddx_u      = ddx_u_comp*ddx_q1 + ddy_u_comp*ddx_q2
        ddy_u      = ddx_u_comp*ddy_q1 + ddy_u_comp*ddy_q2
        
        ddx_v_comp = gradient(v, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddy_v_comp = gradient(v, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddx_v      = ddx_v_comp*ddx_q1 + ddy_v_comp*ddx_q2
        ddy_v      = ddx_v_comp*ddy_q1 + ddy_v_comp*ddy_q2
        
        ## dimensional [1/s]
        vort_z = ddx_v - ddy_u
        
        # ===
        
        s1 = np.copy( stang )
        s2 = np.copy( snorm )
        
        ## [s2] gradients --> yields 1D in [y]/[s2]
        dds2_u     = gradient(u     , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_v     = gradient(v     , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_T     = gradient(T     , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_p     = gradient(p     , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_rho   = gradient(rho   , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_utang = gradient(utang , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_unorm = gradient(unorm , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        
        ## wall quantities --> mean over [x]/[s1] --> leave 0D float
        dds2_u1_wall = float( np.mean( dds2_utang[:,0] ) )
        dds2_T       = float( np.mean( dds2_T[:,0]     ) )
        rho_wall     = float( np.mean( rho[:,0]        ) )
        nu_wall      = float( np.mean( nu[:,0]         ) )
        mu_wall      = float( np.mean( mu[:,0]         ) )
        T_wall       = float( np.mean( T[:,0]          ) )
        tau_wall     = mu_wall * dds2_u1_wall
        q_wall       = self.cp * mu_wall / self.Pr * dds2_T ### wall heat flux
        
        ## friction velocity
        u_tau  = np.sqrt(tau_wall/rho_wall)
        #y_plus = np.copy( s2 * u_tau / nu_wall )
        
        # === populate 1D & 2D arrays using calc_bl_edge_1d(), calc_d99_1d()
        
        psvel = np.zeros((nx,ny), dtype=np.float64)
        
        psvel_edge = np.zeros((nx,), dtype=np.float64)
        u1_edge    = np.zeros((nx,), dtype=np.float64)
        rho_edge   = np.zeros((nx,), dtype=np.float64)
        mu_edge    = np.zeros((nx,), dtype=np.float64)
        nu_edge    = np.zeros((nx,), dtype=np.float64)
        T_edge     = np.zeros((nx,), dtype=np.float64)
        
        s2_edge = np.zeros((nx,), dtype=np.float64)
        d99     = np.zeros((nx,), dtype=np.float64)
        
        u1_99  = np.zeros((nx,), dtype=np.float64)
        
        for i in range(nx):
            
            vort_z_ = np.copy( vort_z[i,:] )
            psvel_  = sp.integrate.cumulative_trapezoid(-1*vort_z_, s2, initial=0.)
            psvel[i,:] = psvel_
            
            ## get edge
            s2_edge_ = calc_bl_edge_1d( y=s2, psvel=psvel_, ynorm=self.lchar, acc=acc, edge_stencil=edge_stencil, epsilon=epsilon )
            #aa = ( s2_edge_ - s2.min() ) / ( s2.max() - s2.min() )
            #tqdm.write(f'{aa:0.9f}')
            s2_edge[i] = s2_edge_
            
            ## interpolate at edge
            psvel_edge_ = sp.interpolate.interp1d(s2, psvel_ , kind='cubic', bounds_error=True)(s2_edge_)
            psvel_edge_ = float(psvel_edge_)
            psvel_edge[i] = psvel_edge_
            
            u1_edge[i]  = sp.interpolate.interp1d(s2, utang[i,:] , kind='cubic', bounds_error=True)(s2_edge_)
            rho_edge[i] = sp.interpolate.interp1d(s2, rho[i,:]   , kind='cubic', bounds_error=True)(s2_edge_)
            mu_edge[i]  = sp.interpolate.interp1d(s2, mu[i,:]    , kind='cubic', bounds_error=True)(s2_edge_)
            nu_edge[i]  = sp.interpolate.interp1d(s2, nu[i,:]    , kind='cubic', bounds_error=True)(s2_edge_)
            T_edge[i]   = sp.interpolate.interp1d(s2, T[i,:]     , kind='cubic', bounds_error=True)(s2_edge_)
            
            ## get d99
            d99_ = calc_d99_1d( y=s2, y_edge=s2_edge_, psvel=psvel_, psvel_edge=psvel_edge_ )
            d99_ = float(d99_)
            d99[i] = d99_
            
            u1_99[i] = sp.interpolate.interp1d(s2, utang[i,:] , kind='cubic', bounds_error=True)(d99_)
        
        # === avg in [x]/[s1] --> leave [y]/[s2]
        
        psvel = np.mean( psvel , axis=0 )
        utang = np.mean( utang , axis=0 )
        rho   = np.mean( rho   , axis=0 )
        
        psvel_edge  = np.mean( psvel_edge )
        u1_edge     = np.mean( u1_edge    )
        rho_edge    = np.mean( rho_edge   )
        mu_edge     = np.mean( mu_edge    )
        nu_edge     = np.mean( nu_edge    )
        T_edge      = np.mean( T_edge     )
        
        s2_edge     = np.mean( s2_edge     )
        d99         = np.mean( d99        )
        
        u1_99       = np.mean( u1_99      )
        
        sc_l_out = d99
        sc_u_out = u1_99
        sc_t_out = d99/u1_99
        np.testing.assert_allclose(sc_t_out, sc_l_out/sc_u_out, rtol=1e-14, atol=1e-14)
        
        sc_u_in = u_tau
        sc_l_in = nu_wall / u_tau
        sc_t_in = nu_wall / u_tau**2
        np.testing.assert_allclose(sc_t_in, sc_l_in/sc_u_in, rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration_avg * (self.lchar / self.U_inf)
        t_eddy = t_meas / (d99/u_tau)
        
        # === get 0D BL integral quantities
        
        dd = calc_bl_integral_quantities_1d( y=s2,
                                             u=utang,
                                             rho=rho,
                                             u_tau=u_tau,
                                             d99=d99,
                                             y_edge=s2_edge,
                                             rho_edge=rho_edge,
                                             nu_edge=nu_edge,
                                             u_edge=u1_edge,
                                             nu_wall=nu_wall,
                                            )
        
        # === add to file
        
        gn = 'data_dim'
        
        ## if group already exists, delete it entirely
        if (gn in self):
            del self[gn]
        
        ## 1D
        self.create_dataset(f'{gn}/psvel' , data=psvel, chunks=None)
        self.create_dataset(f'{gn}/utang' , data=utang, chunks=None)
        self.create_dataset(f'{gn}/rho'   , data=rho,   chunks=None)
        
        self.create_dataset(f'{gn}/z1d'   , data=z1d,   chunks=None)
        
        ## 0D
        self.create_dataset(f'{gn}/dz0'        , data=dz0, chunks=None)
        self.create_dataset(f'{gn}/dt'         , data=dt, chunks=None)
        
        self.create_dataset(f'{gn}/s2_edge'    , data=s2_edge    , chunks=None)
        self.create_dataset(f'{gn}/d99'        , data=d99        , chunks=None)
        self.create_dataset(f'{gn}/u1_99'      , data=u1_99      , chunks=None)
        
        self.create_dataset(f'{gn}/psvel_edge' , data=psvel_edge , chunks=None)
        self.create_dataset(f'{gn}/u1_edge'    , data=u1_edge    , chunks=None)
        self.create_dataset(f'{gn}/rho_edge'   , data=rho_edge   , chunks=None)
        self.create_dataset(f'{gn}/mu_edge'    , data=mu_edge    , chunks=None)
        self.create_dataset(f'{gn}/nu_edge'    , data=nu_edge    , chunks=None)
        self.create_dataset(f'{gn}/T_edge'     , data=T_edge     , chunks=None)
        
        self.create_dataset(f'{gn}/u_tau'      , data=u_tau      , chunks=None)
        
        self.create_dataset(f'{gn}/dds2_u1_wall' , data=dds2_u1_wall , chunks=None )
        self.create_dataset(f'{gn}/dds2_T'       , data=dds2_T       , chunks=None )
        self.create_dataset(f'{gn}/rho_wall'     , data=rho_wall     , chunks=None )
        self.create_dataset(f'{gn}/nu_wall'      , data=nu_wall      , chunks=None )
        self.create_dataset(f'{gn}/mu_wall'      , data=mu_wall      , chunks=None )
        self.create_dataset(f'{gn}/T_wall'       , data=T_wall       , chunks=None )
        self.create_dataset(f'{gn}/tau_wall'     , data=tau_wall     , chunks=None )
        self.create_dataset(f'{gn}/q_wall'       , data=q_wall       , chunks=None )
        
        self.create_dataset(f'{gn}/sc_u_in'    , data=sc_u_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_l_in'    , data=sc_l_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_t_in'    , data=sc_t_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_u_out'   , data=sc_u_out   , chunks=None)
        self.create_dataset(f'{gn}/sc_l_out'   , data=sc_l_out   , chunks=None)
        self.create_dataset(f'{gn}/sc_t_out'   , data=sc_t_out   , chunks=None)
        
        ## add integrated quantities (all 0D)
        for key,val in dd.items():
            self.create_dataset(f'{gn}/{key}', data=val, chunks=None)
        
        ## report
        if verbose:
            print(72*'-')
            even_print('Re_τ'                      , '%0.1f'%dd['Re_tau']            )
            even_print('Re_θ'                      , '%0.1f'%dd['Re_theta']          )
            even_print('θ'                         , '%0.5e [m]'%dd['theta_cmp']     )
            even_print('δ*'                        , '%0.5e [m]'%dd['dstar_cmp']     )
            even_print('H12'                       , '%0.5f'%dd['H12']               )
            ##
            even_print('δ99'                       , '%0.5e [m]'%d99                 )
            even_print('u_τ'                       , '%0.3f [m/s]'%u_tau             )
            even_print('ν_wall'                    , '%0.5e [m²/s]'%nu_wall          )
            even_print('t_meas'                    , '%0.5e [s]'%t_meas              )
            even_print('t_meas/tchar'              , '%0.1f'%(t_meas/self.tchar)     )
            even_print('t_eddy = t_meas/(δ99/u_τ)' , '%0.2f'%t_eddy                  )
            even_print('t_meas/(δ99/u1_99)'        , '%0.2f'%(t_meas/(d99/u1_99))    )
            even_print('t_meas/(20·δ99/u1_99)'     , '%0.2f'%(t_meas/(20*d99/u1_99)) )
            print(72*'-')
            even_print('sc_u_in = u_τ'               , '%0.5e [m/s]'%(sc_u_in,)  )
            even_print('sc_l_in = δ_ν = ν_wall/u_τ'  , '%0.5e [m]'%(sc_l_in,)    )
            even_print('sc_t_in = t_ν = ν_wall/u_τ²' , '%0.5e [s]'%(sc_t_in,)    )
            even_print('sc_u_out = u1_99'            , '%0.5e [m/s]'%(sc_u_out,) )
            even_print('sc_l_out = δ99'              , '%0.5e [m]'%(sc_l_out,)   )
            even_print('sc_t_out = δ99/u1_99'        , '%0.5e [s]'%(sc_t_out,)   )
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_mean_dimensional_data_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_mean_dimensional_data_ypln(self, **kwargs):
        '''
        get dimensionalized mean data for [y] plane
        --> save to existing CGD file with fsubtype=mean
        - assumes volume which is thin in [y] direction
        - an CGD which is the output of cgd.get_mean() should be opened here
        - not parallel
        '''
        
        pass
        raise NotImplementedError
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_mean_dimensional_data_ypln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_mean_dimensional_data_zpln(self, **kwargs):
        '''
        get dimensionalized mean data for [z] plane
        --> save to existing CGD file with fsubtype=mean
        - assumes volume which is thin in [z] direction
        - an CGD which is the output of cgd.get_mean() should be opened here
        - not parallel
        '''
        
        pass
        raise NotImplementedError
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_mean_dimensional_data_zpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing (unsteady)
    
    def get_prime(self, **kwargs):
        '''
        calc mean-removed (prime) variables in [t]
        --> save to new CGD file
        -----
        XI  : Reynolds primes : mean(XI)=0
        XII : Favre primes    : mean(ρ·XII)=0 --> mean(XII)≠0 !!
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        ## assert that the opened CGD has fsubtype 'unsteady'
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        if verbose: print('\n'+'cgd.get_prime()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        fn_cgd_mean  = kwargs.get('fn_cgd_mean',None)
        fn_cgd_prime = kwargs.get('fn_cgd_prime',None)
        sfp          = kwargs.get('scalars',None) ## scalars (for prime)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 32 ) ## for initializing prime file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8  )
        
        ## start timestep index
        ti_min = kwargs.get('ti_min',None)
        
        ## if writing Favre primes, copy over ρ --> mean(ρ·XII)=0 / mean(XII)≠0 !!
        if favre:
            copy_rho = True
        else:
            copy_rho = False
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if (sfp is None):
            sfp = self.scalars
        
        # === ranks
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === get times to take for prime
        if (ti_min is not None):
            ti_for_prime = np.copy( self.ti[ti_min:] )
        else:
            ti_for_prime = np.copy( self.ti )
        
        nt_prime       = ti_for_prime.shape[0]
        t_prime_start  = self.t[ti_for_prime[0]]
        t_prime_end    = self.t[ti_for_prime[-1]]
        duration_prime = t_prime_end - t_prime_start
        
        # === chunks
        #ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl_ = np.array_split(ti_for_prime,min(ct,nt_prime))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # === mean file name (for reading)
        if (fn_cgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_cgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_cgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_cgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if not os.path.isfile(fn_cgd_mean):
            raise FileNotFoundError('%s not found!'%fn_cgd_mean)
        
        # === prime file name (for writing)
        if (fn_cgd_prime is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_prime_h5_base = fname_root+'_prime.h5'
            #fn_cgd_prime = os.path.join(fname_path, fname_prime_h5_base)
            fn_cgd_prime = str(PurePosixPath(fname_path, fname_prime_h5_base))
            #fn_cgd_prime = Path(fname_path, fname_prime_h5_base)
        
        if verbose: even_print('fn_cgd'          , self.fname    )
        if verbose: even_print('fn_cgd_mean'     , fn_cgd_mean   )
        if verbose: even_print('fn_cgd_prime'    , fn_cgd_prime  )
        if verbose: even_print('do Favre avg'    , str(favre)    )
        if verbose: even_print('do Reynolds avg' , str(reynolds) )
        if verbose: even_print('copy rho'        , str(copy_rho) )
        if verbose: even_print('ct'              , '%i'%ct       )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('n timesteps prime','%i/%i'%(nt_prime,self.nt))
        if verbose: even_print('t index prime start','%i'%(ti_for_prime[0],))
        if verbose: even_print('t index prime end','%i'%(ti_for_prime[-1],))
        if verbose: even_print('t prime start','%0.2f [-]'%(t_prime_start,))
        if verbose: even_print('t prime end','%0.2f [-]'%(t_prime_end,))
        if verbose: even_print('duration prime','%0.2f [-]'%(duration_prime,))
        if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        itemsize = dtype.itemsize
        
        #data_gb      = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        data_gb      = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1       / 1024**3
        
        scalars_re = [ 'u','v','w','T','p','rho', 'utang','unorm', 'vort_x','vort_y','vort_z','vort_tang' ]
        scalars_fv = [ 'u','v','w','T',           'utang','unorm' ] ## p'' and ρ'' are never really needed
        
        scalars_re_ = []
        for scalar in scalars_re:
            if (scalar in self.scalars) and (scalar in sfp):
                scalars_re_.append(scalar)
        scalars_re = scalars_re_
        
        scalars_fv_ = []
        for scalar in scalars_fv:
            if (scalar in self.scalars) and (scalar in sfp):
                scalars_fv_.append(scalar)
        scalars_fv = scalars_fv_
        
        # ===
        
        comm_cgd_prime = MPI.COMM_WORLD
        
        with cgd(fn_cgd_prime, 'w', force=force, driver=self.driver, comm=self.comm, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_prime:
            
            ## initialize prime cgd from cgd
            hf_prime.init_from_cgd(self.fname, rx=rx,ry=ry,rz=rz, chunk_kb=chunk_kb)
            
            ## add top-level attributes
            #hf_prime.attrs['fclass'] = 'cgd'
            hf_prime.attrs['fsubtype'] = 'prime'
            
            #shape = (self.nt,self.nz,self.ny,self.nx)
            shape = (nt_prime,self.nz,self.ny,self.nx)
            
            ## determine dtypes for prime file
            for scalar in self.scalars:
                dset = self[f'data/{scalar}']
                dtype = dset.dtype
                if reynolds and (scalar in scalars_re):
                    hf_prime.scalars_dtypes_dict[f'{scalar}I'] = dtype
                if favre and (scalar in scalars_fv):
                    hf_prime.scalars_dtypes_dict[f'{scalar}II'] = dtype
            
            ##chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
            #chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
            
            # === initialize prime datasets + rho
            
            if copy_rho:
                
                dtype = self.scalars_dtypes_dict['rho']
                float_bytes = dtype.itemsize
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                
                if verbose:
                    even_print('initializing data/rho','%0.1f [GB]'%(data_gb,))
                
                dset = hf_prime.create_dataset('data/rho',
                                               shape=shape,
                                               dtype=dtype,
                                               chunks=chunks)
                
                chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                if verbose:
                    even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            for scalar in self.scalars:
                
                if reynolds:
                    if (scalar in scalars_re):
                        
                        dtype = hf_prime.scalars_dtypes_dict[f'{scalar}I']
                        float_bytes = dtype.itemsize
                        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                        
                        ## if ('data/%sI'%scalar in hf_prime):
                        ##     del hf_prime['data/%sI'%scalar]
                        if verbose:
                            even_print('initializing data/%sI'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        
                        dset = hf_prime.create_dataset(f'data/{scalar}I',
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_prime.scalars.append(f'{scalar}I')
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    if (scalar in scalars_fv):
                        
                        dtype = hf_prime.scalars_dtypes_dict[f'{scalar}II']
                        float_bytes = dtype.itemsize
                        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                        
                        ## if ('data/%sII'%scalar in hf_prime):
                        ##     del hf_prime['data/%sII'%scalar]
                        if verbose:
                            even_print('initializing data/%sII'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        
                        dset = hf_prime.create_dataset(f'data/{scalar}II',
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_prime.scalars.append(f'{scalar}II')
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if hf_prime.usingmpi: comm_cgd_prime.Barrier()
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            # === read unsteady + mean, do difference, write
            
            n_pbar = 0
            if favre or copy_rho:
                n_pbar += 1
            for scalar in self.scalars:
                if (scalar in scalars_re) and reynolds:
                    n_pbar += 1
                if (scalar in scalars_fv) and favre:
                    n_pbar += 1
            
            comm_cgd_mean = MPI.COMM_WORLD
            
            with cgd(fn_cgd_mean, 'r', driver=self.driver, comm=self.comm) as hf_mean:
                
                ## copy over 'data_dim' from mean file
                if ('data_dim' in hf_mean):
                    grp = hf_mean['data_dim']
                    for dsn in grp.keys():
                        ds = hf_mean[f'data_dim/{dsn}']
                        data = np.copy(ds[()])
                        #if (f'data_dim/{dsn}' in hf_prime):
                        #    del hf_prime[f'data_dim/{dsn}']
                        
                        #if self.usingmpi:
                        #    with dset.collective:
                        #        hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                        #else:
                        #    hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                        
                        hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                    
                    if verbose:
                        even_print('data_dim copied to prime',str(True))
                else:
                    if verbose:
                        even_print('data_dim copied to prime',str(False))
                
                if verbose:
                    progress_bar = tqdm(total=ct*n_pbar, ncols=100, desc='prime', leave=False, file=sys.stdout)
                
                for ctl_ in ctl:
                    ct1, ct2 = ctl_
                    ntc = ct2 - ct1
                    
                    #if verbose: tqdm.write(f'ct1,ct2 = {ct1:d},{ct2:d}')
                    #if verbose: tqdm.write(f'ntc = {ntc:d}')
                    
                    ## chunk range for writing to file (offset from read if using ti_min)
                    if (ti_min is not None):
                        #ct1w,ct2w = ct1-ti_min, ct2-ti_min ## doesnt work for (-) ti_min
                        ct1w,ct2w = ct1-ti_for_prime[0], ct2-ti_for_prime[0]
                    else:
                        ct1w,ct2w = ct1,ct2
                    
                    # ## debug report
                    # if verbose: tqdm.write('ct1,ct2 = %i,%i'%(ct1,ct2))
                    # if verbose: tqdm.write('ct1w,ct2w = %i,%i'%(ct1w,ct2w))
                    
                    if favre or copy_rho:
                        
                        ## read rho
                        dset = self['data/rho']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                rho = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                        else:
                            rho = dset[ct1:ct2,:,:,:].T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if verbose:
                            txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        ## write a copy of rho to the prime file
                        dset = hf_prime['data/rho']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if hf_prime.usingmpi: hf_prime.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = rho.T
                                dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = rho.T
                        else:
                            #dset[ct1:ct2,:,:,:] = rho.T
                            dset[ct1w:ct2w,:,:,:] = rho.T
                        if hf_prime.usingmpi: hf_prime.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            txt = even_print('write: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        if verbose: progress_bar.update()
                    
                    for scalar in self.scalars:
                        
                        if (scalar in scalars_re) or (scalar in scalars_fv):
                            
                            ## read CGD data
                            dset = self['data/%s'%scalar]
                            dtype = dset.dtype
                            float_bytes = dtype.itemsize
                            data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                            
                            if self.usingmpi: self.comm.Barrier()
                            t_start = timeit.default_timer()
                            if self.usingmpi:
                                with dset.collective:
                                    data = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                            else:
                                data = dset[ct1:ct2,:,:,:].T
                            if self.usingmpi: self.comm.Barrier()
                            t_delta = timeit.default_timer() - t_start
                            
                            if verbose:
                                txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                tqdm.write(txt)
                            
                            t_read       += t_delta
                            data_gb_read += data_gb
                            
                            # === do prime Reynolds
                            
                            if (scalar in scalars_re) and reynolds:
                                
                                ## read Reynolds avg from mean file
                                dset = hf_mean['data/%s'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_mean.nx * hf_mean.ny * hf_mean.nz * 1 / 1024**3
                                
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_mean.usingmpi:
                                    with dset.collective:
                                        data_mean_re = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                                else:
                                    data_mean_re = dset[()].T
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                ## if verbose:
                                ##     txt = even_print('read: %s (Re avg)'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                ##     tqdm.write(txt)
                                
                                ## calc mean-removed Reynolds
                                data_prime_re = data - data_mean_re
                                
                                ## if False:
                                ##     data_prime_re_mean = np.mean(data_prime_re, axis=-1, dtype=np.float64, keepdims=True).astype(np.float32)
                                ##     
                                ##     ## normalize [mean(prime)] by mean
                                ##     data_prime_re_mean = np.abs(np.divide(data_prime_re_mean,
                                ##                                           data_mean_re, 
                                ##                                           out=np.zeros_like(data_prime_re_mean), 
                                ##                                           where=data_mean_re!=0))
                                ##     
                                ##     # np.testing.assert_allclose( data_prime_re_mean , 
                                ##     #                             np.zeros_like(data_prime_re_mean, dtype=np.float32), atol=1e-4)
                                ##     if verbose:
                                ##         tqdm.write('max(abs(mean(%sI)/mean(%s)))=%0.4e'%(scalar,scalar,data_prime_re_mean.max()))
                                
                                ## write Reynolds prime
                                dset = hf_prime['data/%sI'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_prime.nx * hf_prime.ny * hf_prime.nz * ntc / 1024**3
                                
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_prime.usingmpi:
                                    with dset.collective:
                                        #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_re.T
                                        dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_re.T
                                else:
                                    #dset[ct1:ct2,:,:,:] = data_prime_re.T
                                    dset[ct1w:ct2w,:,:,:] = data_prime_re.T
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if verbose:
                                    txt = even_print('write: %sI'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                    tqdm.write(txt)
                                pass
                                
                                if verbose: progress_bar.update()
                            
                            # === do prime Favre
                            
                            if (scalar in scalars_fv) and favre:
                                
                                ## read Favre avg from mean file
                                dset = hf_mean['data/%s_fv'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_mean.nx * hf_mean.ny * hf_mean.nz * 1 / 1024**3
                                
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_mean.usingmpi:
                                    with dset.collective:
                                        data_mean_fv = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                                else:
                                    data_mean_fv = dset[()].T
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                ## if verbose:
                                ##     txt = even_print('read: %s (Fv avg)'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                ##     tqdm.write(txt)
                                
                                ## calc mean-removed Favre
                                ## data_prime_fv = ( data - data_mean_fv ) * rho ## pre-multiply with ρ (has zero mean) --> better to not do this here
                                data_prime_fv = data - data_mean_fv
                                
                                ## write Favre prime
                                dset = hf_prime['data/%sII'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_prime.nx * hf_prime.ny * hf_prime.nz * ntc / 1024**3
                                
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_prime.usingmpi:
                                    with dset.collective:
                                        #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_fv.T
                                        dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_fv.T
                                else:
                                    #dset[ct1:ct2,:,:,:] = data_prime_fv.T
                                    dset[ct1w:ct2w,:,:,:] = data_prime_fv.T
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if verbose:
                                    txt = even_print('write: %sII'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                    tqdm.write(txt)
                                
                                pass
                                
                                if verbose: progress_bar.update()
                        
                        if self.usingmpi: self.comm.Barrier()
                        if hf_prime.usingmpi: comm_cgd_prime.Barrier()
                        if hf_mean.usingmpi: comm_cgd_mean.Barrier()
                    
                    ## report mem free
                    mem_avail_gb = psutil.virtual_memory().available/1024**3
                    mem_free_gb  = psutil.virtual_memory().free/1024**3
                    if verbose:
                        tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                
                if verbose:
                    progress_bar.close()
            
            # === replace dims/t array in prime file (if ti_min was given)
            if (ti_min is not None):
                t = np.copy( self.t[ti_min:] )
                if ('dims/t' in hf_prime):
                    del hf_prime['dims/t']
                hf_prime.create_dataset('dims/t', data=t)
            
            if hf_mean.usingmpi: comm_cgd_mean.Barrier()
        if hf_prime.usingmpi: comm_cgd_prime.Barrier()
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_cgd_prime, '%0.2f [GB]'%(os.path.getsize(fn_cgd_prime)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.get_prime() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_lambda2(self, **kwargs):
        '''
        calculate λ₂ & Q
        Jeong & Hussain (1996) : https://doi.org/10.1017/S0022112095000462
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx           = kwargs.get('rx',1)
        ry           = kwargs.get('ry',1)
        rz           = kwargs.get('rz',1)
        
        save_Q       = kwargs.get('save_Q',True)
        save_lambda2 = kwargs.get('save_lambda2',True)
        
        acc          = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','half')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        d = 1 ## derivative order
        stencil_npts = 2*math.floor((d+1)/2) - 1 + acc
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        
        # ===
        
        if verbose: print('\n'+'turbx.cgd.calc_lambda2()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## checks
        if all([(save_Q is False),(save_lambda2 is False)]):
            raise AssertionError('neither λ-2 nor Q set to be solved')
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('data/w' in self):
            raise ValueError('data/w not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('save_Q','%s'%save_Q)
        if verbose: even_print('save_lambda2','%s'%save_lambda2)
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        t_q_crit = 0.
        t_l2     = 0.
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # === extend the rank ranges (spatial range overlap)
        
        if self.usingmpi:
            
            n_overlap = stencil_npts_one_side + 3
            
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            zA = 0
            zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## overlap in [y]
            if (t4d[1]!=0):
                ry1, ry2 = ry1-n_overlap, ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                ry1, ry2 = ry1, ry2+n_overlap
            
            ## overlap in [z]
            if (t4d[2]!=0):
                rz1, rz2 = rz1-n_overlap, rz2
                zA += n_overlap
                zB += n_overlap
            if (t4d[2]!=rz-1):
                rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        dtype = self['data/u'].dtype
        itemsize = dtype.itemsize
        float_bytes = dtype.itemsize
        
        data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        # === initialize 4D arrays in HDF5
        
        if save_lambda2:
            
            self.scalars_dtypes_dict['lambda2'] = dtype
            
            if verbose:
                even_print('initializing data/lambda2','%0.2f [GB]'%(data_gb,))
            if ('data/lambda2' in self):
                del self['data/lambda2']
            dset = self.create_dataset('data/lambda2', 
                                        shape=shape, 
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if save_Q:
            
            self.scalars_dtypes_dict['Q'] = dtype
            
            if verbose:
                even_print('initializing data/Q','%0.2f [GB]'%(data_gb,))
            if ('data/Q' in self):
                del self['data/Q']
            dset = self.create_dataset('data/Q', 
                                        shape=shape, 
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === read grid collectively (CGD)
        
        dset = self['dims/x']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                x_ = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            x_ = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read x', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/y']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                y_ = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            y_ = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read y', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/z']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                z_ = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            z_ = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        # === get metric tensor (CGD / curvilinear)
        
        t_start = timeit.default_timer()
        
        M = get_metric_tensor_3d(x_, y_, z_, acc=acc, edge_stencil=edge_stencil, verbose=verbose)
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get metric tensor','%0.3f [s]'%(t_delta,), s=True) )
        
        # === split out the metric tensor components
        
        ddx_q1 = np.copy( M[:,:,:,0,0] ) ## ξ_x
        ddx_q2 = np.copy( M[:,:,:,1,0] ) ## η_x
        ddx_q3 = np.copy( M[:,:,:,2,0] ) ## ζ_x
        ##
        ddy_q1 = np.copy( M[:,:,:,0,1] ) ## ξ_y
        ddy_q2 = np.copy( M[:,:,:,1,1] ) ## η_y
        ddy_q3 = np.copy( M[:,:,:,2,1] ) ## ζ_y
        ##
        ddz_q1 = np.copy( M[:,:,:,0,2] ) ## ξ_z
        ddz_q2 = np.copy( M[:,:,:,1,2] ) ## η_z
        ddz_q3 = np.copy( M[:,:,:,2,2] ) ## ζ_z
        
        M = None; del M ## free memory
        
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        #if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## the 'computational' grid (unit Cartesian)
        #x_comp = np.arange(nx, dtype=np.float64)
        #y_comp = np.arange(ny, dtype=np.float64)
        #z_comp = np.arange(nz, dtype=np.float64)
        x_comp = 1.
        y_comp = 1.
        z_comp = 1.
        
        # === main loop
        
        if verbose:
            progress_bar = tqdm(total=self.nt, ncols=100, desc='calc λ2', leave=False, file=sys.stdout)
        
        for ti in self.ti:
            
            # === read u,v,w
            
            dset = self['data/u']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    u_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                u_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read u', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/v']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    v_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                v_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read v', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/w']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    w_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                w_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # ===
            
            # if self.usingmpi:
            #     x_ = np.copy(self.x[rx1:rx2])
            #     y_ = np.copy(self.y[ry1:ry2])
            #     z_ = np.copy(self.z[rz1:rz2])
            # else:
            #     x_ = np.copy(self.x)
            #     y_ = np.copy(self.y)
            #     z_ = np.copy(self.z)
            
            # ==========================================================
            # get velocity gradient (strain) tensor ∂(u,v,w)/∂(x,y,z)
            # ==========================================================
            
            t_start = timeit.default_timer()
            
            # === ∂(u)/∂(x,y,z)
            
            ddx_u_comp = gradient(u_, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_u_comp = gradient(u_, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_u_comp = gradient(u_, z_comp, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            u_ = None; del u_ ## free memory
            
            ## get physical grid gradient elements by taking
            ##  inner product with metric tensor
            ddx_u = np.copy( ddx_u_comp * ddx_q1 +
                             ddy_u_comp * ddx_q2 +
                             ddz_u_comp * ddx_q3 )
            
            ddy_u = np.copy( ddx_u_comp * ddy_q1 +
                             ddy_u_comp * ddy_q2 +
                             ddz_u_comp * ddy_q3 )
            
            ddz_u = np.copy( ddx_u_comp * ddz_q1 +
                             ddy_u_comp * ddz_q2 +
                             ddz_u_comp * ddz_q3 )
            
            ddx_u_comp = None; del ddx_u_comp ## free memory
            ddy_u_comp = None; del ddy_u_comp ## free memory
            ddz_u_comp = None; del ddz_u_comp ## free memory
            
            # === ∂(v)/∂(x,y,z)
            
            ddx_v_comp = gradient(v_, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_v_comp = gradient(v_, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_v_comp = gradient(v_, z_comp, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            v_ = None; del v_ ## free memory
            
            ddx_v = np.copy( ddx_v_comp * ddx_q1 + \
                             ddy_v_comp * ddx_q2 + \
                             ddz_v_comp * ddx_q3 )
            
            ddy_v = np.copy( ddx_v_comp * ddy_q1 + \
                             ddy_v_comp * ddy_q2 + \
                             ddz_v_comp * ddy_q3 )
            
            ddz_v = np.copy( ddx_v_comp * ddz_q1 + \
                             ddy_v_comp * ddz_q2 + \
                             ddz_v_comp * ddz_q3 )
            
            ddx_v_comp = None; del ddx_v_comp ## free memory
            ddy_v_comp = None; del ddy_v_comp ## free memory
            ddz_v_comp = None; del ddz_v_comp ## free memory
            
            # === ∂(w)/∂(x,y,z)
            
            ddx_w_comp = gradient(w_, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_w_comp = gradient(w_, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_w_comp = gradient(w_, z_comp, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            w_ = None; del w_ ## free memory
            
            ddx_w = np.copy( ddx_w_comp * ddx_q1 + \
                             ddy_w_comp * ddx_q2 + \
                             ddz_w_comp * ddx_q3 )
            
            ddy_w = np.copy( ddx_w_comp * ddy_q1 + \
                             ddy_w_comp * ddy_q2 + \
                             ddz_w_comp * ddy_q3 )
            
            ddz_w = np.copy( ddx_w_comp * ddz_q1 + \
                             ddy_w_comp * ddz_q2 + \
                             ddz_w_comp * ddz_q3 )
            
            ddx_w_comp = None; del ddx_w_comp ## free memory
            ddy_w_comp = None; del ddy_w_comp ## free memory
            ddz_w_comp = None; del ddz_w_comp ## free memory
            
            # ===
            
            ## free memory
            u_ = None; del u_
            v_ = None; del v_
            w_ = None; del w_
            gc.collect()
            
            strain = np.copy( np.stack((np.stack((ddx_u, ddy_u, ddz_u), axis=3),
                                        np.stack((ddx_v, ddy_v, ddz_v), axis=3),
                                        np.stack((ddx_w, ddy_w, ddz_w), axis=3)), axis=4) )
            
            t_delta = timeit.default_timer() - t_start
            if verbose: tqdm.write( even_print('get strain ∂(u,v,v)/∂(x,y,z)' , '%0.3f [s]'%(t_delta,), s=True) )
            
            ## free memory
            ddx_u = None; del ddx_u
            ddy_u = None; del ddy_u
            ddz_u = None; del ddz_u
            ddx_v = None; del ddx_v
            ddy_v = None; del ddy_v
            ddz_v = None; del ddz_v
            ddx_w = None; del ddx_w
            ddy_w = None; del ddy_w
            ddz_w = None; del ddz_w
            gc.collect()
            
            # === get the rate-of-strain & vorticity tensors
            
            S = np.copy( 0.5*(strain + np.transpose(strain, axes=(0,1,2,4,3))) ) ## strain rate tensor (symmetric)
            O = np.copy( 0.5*(strain - np.transpose(strain, axes=(0,1,2,4,3))) ) ## rotation rate tensor (anti-symmetric)
            # np.testing.assert_allclose(S+O, strain, atol=1.e-6)
            
            ## free memory
            strain = None; del strain
            gc.collect()
            
            # === Q : second invariant of characteristics equation: λ³ + Pλ² + Qλ + R = 0
            
            if save_Q:
                
                t_start = timeit.default_timer()
                
                O_norm  = np.linalg.norm(O, ord='fro', axis=(3,4)) ## Frobenius norm
                S_norm  = np.linalg.norm(S, ord='fro', axis=(3,4))
                Q       = 0.5*(O_norm**2 - S_norm**2)
                
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print('calc Q','%s'%format_time_string(t_delta), s=True))
                
                dset = self['data/Q']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ti,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = Q[xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ti,:,:,:] = Q.T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print('write Q','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                # === second invariant : Q --> an equivalent formulation using eigenvalues (but much slower)
                if False:
                    Q_bak = np.copy(Q)
                    t_start = timeit.default_timer()
                    eigvals = np.linalg.eigvals(strain)
                    P       = -1*np.sum(eigvals, axis=-1) ## first invariant : P
                    SijSji  = np.einsum('xyzij,xyzji->xyz', S, S)
                    OijOji  = np.einsum('xyzij,xyzji->xyz', O, O)
                    Q       = 0.5*(P**2 - SijSji - OijOji)
                    t_delta = timeit.default_timer() - t_start
                    if verbose: tqdm.write(even_print('calc Q','%s'%format_time_string(t_delta), s=True))
                    np.testing.assert_allclose(Q.imag, np.zeros_like(Q.imag, dtype=np.float32), atol=1e-6)
                    Q = np.copy(Q.real)
                    np.testing.assert_allclose(Q, Q_bak, rtol=1e-2, atol=1e-5)
                
                ## free memory
                O_norm = None; del O_norm
                S_norm = None; del S_norm
                Q = None; del Q
                gc.collect()
            
            # === λ₂
            
            if save_lambda2:
                
                t_start = timeit.default_timer()
                
                # === S² and Ω²
                SikSkj = np.einsum('xyzik,xyzkj->xyzij', S, S)
                OikOkj = np.einsum('xyzik,xyzkj->xyzij', O, O)
                #np.testing.assert_allclose(np.matmul(S,S), SikSkj, atol=1e-6)
                #np.testing.assert_allclose(np.matmul(O,O), OikOkj, atol=1e-6)
                
                ## free memory
                S = None; del S
                O = None; del O
                gc.collect()
                
                # === Eigenvalues of (S²+Ω²) --> a real symmetric (Hermitian) matrix
                eigvals            = np.linalg.eigvalsh(SikSkj+OikOkj, UPLO='L')
                #eigvals_sort_order = np.argsort(np.abs(eigvals), axis=3) ## sort order of λ --> magnitude (wrong)
                eigvals_sort_order = np.argsort(eigvals, axis=3) ## sort order of λ
                eigvals_sorted     = np.take_along_axis(eigvals, eigvals_sort_order, axis=3) ## do λ sort
                lambda2            = np.squeeze(eigvals_sorted[:,:,:,1]) ## λ2 is the second eigenvalue (index=1)
                t_delta            = timeit.default_timer() - t_start
                
                if verbose: tqdm.write(even_print('calc λ2','%s'%format_time_string(t_delta), s=True))
                
                dset = self['data/lambda2']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ti,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = lambda2[xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ti,:,:,:] = lambda2.T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print('write λ2','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## free memory
                lambda2 = None; del lambda2
                eigvals = None; del eigvals
                eigvals_sort_order = None; del eigvals_sort_order
                eigvals_sorted = None; del eigvals_sorted
                gc.collect()
            
            if verbose: progress_bar.update()
            if verbose and (ti<self.nt-1): tqdm.write( '---' )
        if verbose: progress_bar.close()
        if verbose: print(72*'-')
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.cgd.calc_lambda2() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vel_tangnorm(self, **kwargs):
        '''
        calculate velocity in wall-tangent & wall-normal coordinates [utang,unorm]=[uξ,uη]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        ct       = kwargs.get('ct',1) ## n chunks [t]
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        if verbose: print('\n'+'cgd.calc_vel_tangnorm()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('csys/vtang' in self):
            raise ValueError('csys/vtang not in hdf5')
        if not ('csys/vnorm' in self):
            raise ValueError('csys/vnorm not in hdf5')
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ct>self.nt):
            raise AssertionError('ct>self.nt')
        
        if ( self['csys/vnorm'].shape != (self.nx,self.ny,2)):
            raise ValueError(f'csys/vnorm shape != (nx,ny,2)')
        if ( self['csys/vtang'].shape != (self.nx,self.ny,2)):
            raise ValueError(f'csys/vtang shape != (nx,ny,2)')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        # === ranks
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # ===
        
        ## read rank-local tang/norm basis vectors vtang & vnorm
        dset = self['csys/vtang']
        vtang = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        dset = self['csys/vnorm']
        vnorm = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        
        #vtang = np.broadcast_to(vtang[:,:,np.newaxis,np.newaxis,:], (nxr,nyr,nzr,1,2))
        #vnorm = np.broadcast_to(vnorm[:,:,np.newaxis,np.newaxis,:], (nxr,nyr,nzr,1,2))
        
        # ===
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        
        data_gb = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        for scalar in ['utang','unorm']:
            
            self.scalars_dtypes_dict[scalar] = dtype
            
            if verbose:
                even_print(f'initializing data/{scalar}','%0.1f [GB]'%(data_gb,))
            if (f'data/{scalar}' in self):
                del self[f'data/{scalar}']
            dset = self.create_dataset(f'data/{scalar}',
                                        shape=shape,
                                        dtype=dtype,
                                        chunks=chunks )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === main loop
        
        if verbose:
            #progress_bar = tqdm(total=self.nt, ncols=100, desc='calc unorm,utang', leave=False, file=sys.stdout)
            progress_bar = tqdm(total=len(ctl), ncols=100, desc='calc unorm,utang', leave=False, file=sys.stdout)
        
        #for ti in self.ti:
        for ctl_ in ctl:
            ct1, ct2 = ctl_
            ntc = ct2 - ct1
            
            # === read u,v
            
            dset = self['data/u']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    u = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
            else:
                u = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read u', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/v']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    v = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
            else:
                v = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read v', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # === perform csys transformation
            
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            
            uv = np.stack((u,v), axis=-1)
            
            if (uv.ndim!=4+1):
                raise ValueError
            
            ## inner product of velocity vector and basis vector (csys transform)
            utang = np.einsum('xyi,xyzti->xyzt', vtang, uv)
            unorm = np.einsum('xyi,xyzti->xyzt', vnorm, uv)
            
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            if verbose: tqdm.write(even_print('csys trafo','%0.3f [s]'%(t_delta,), s=True))
            
            # === convert back to float32 before write
            
            utang = np.copy( utang.astype(np.float32) )
            unorm = np.copy( unorm.astype(np.float32) )
            
            # === write
            
            dset = self['data/utang']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = utang.T
            else:
                dset[ct1:ct2,:,:,:] = utang.T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz / 1024**3
            
            if verbose: tqdm.write(even_print('write utang','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            dset = self['data/unorm']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = unorm.T
            else:
                dset[ct1:ct2,:,:,:] = unorm.T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz / 1024**3
            
            if verbose: tqdm.write(even_print('write unorm','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_vel_tangnorm() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vorticity(self, **kwargs):
        '''
        calculate vorticity vector [ω_x,ω_y,ω_z]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx           = kwargs.get('rx',1)
        ry           = kwargs.get('ry',1)
        rz           = kwargs.get('rz',1)
        ct           = kwargs.get('ct',1) ## n chunks [t]
        
        acc          = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','half')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        d = 1 ## derivative order
        stencil_npts = 2*math.floor((d+1)/2) - 1 + acc
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        
        # ===
        
        if verbose: print('\n'+'cgd.calc_vorticity()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('data/w' in self):
            raise ValueError('data/w not in hdf5')
        #if not ('csys/vtang' in self):
        #    raise ValueError('csys/vtang not in hdf5')
        #if not ('csys/vnorm' in self):
        #    raise ValueError('csys/vnorm not in hdf5')
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ct>self.nt):
            raise AssertionError('ct>self.nt')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## time chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        # === extend the rank ranges (spatial range overlap)
        
        if self.usingmpi:
            
            n_overlap = stencil_npts_one_side + 3
            
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            zA = 0
            zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## ## overlap in [y]
            ## if (t4d[1]!=0):
            ##     ry1, ry2 = ry1-n_overlap, ry2
            ##     yA += n_overlap
            ##     yB += n_overlap
            ## if (t4d[1]!=ry-1):
            ##     ry1, ry2 = ry1, ry2+n_overlap
            
            ## overlap in [y] --> TODO: needs checking
            if (t4d[1]!=0):
                ry1 = ry1-n_overlap
                #ry2 = ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                #ry1 = ry1
                ry2 = min(ry2+n_overlap,self.ny)
            
            if (ry2>self.ny):
                #raise IndexError
                print('ry2>self.ny')
                self.comm.Abort(1)
            if (ry1<0):
                #raise IndexError
                print('ry1<0')
                self.comm.Abort(1)
            
            ## overlap in [z]
            if (t4d[2]!=0):
                rz1, rz2 = rz1-n_overlap, rz2
                zA += n_overlap
                zB += n_overlap
            if (t4d[2]!=rz-1):
                rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        # ## read rank-local tang/norm basis vectos vtang & vnorm
        # dset = self['csys/vtang']
        # vtang = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        # dset = self['csys/vnorm']
        # vnorm = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        # 
        # vtang = np.broadcast_to(vtang[:,:,np.newaxis,:], (nxr,nyr,nzr,2))
        # vnorm = np.broadcast_to(vnorm[:,:,np.newaxis,:], (nxr,nyr,nzr,2))
        
        # ===
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        
        data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        # === initialize 4D arrays in HDF5
        
        for dsn in ['vort_x','vort_y','vort_z']:
            
            self.scalars_dtypes_dict[dsn] = dtype
            
            if verbose:
                even_print(f'initializing data/{dsn}','%0.2f [GB]'%(data_gb,))
            if (f'data/{dsn}' in self):
                del self[f'data/{dsn}']
            dset = self.create_dataset( f'data/{dsn}',
                                        shape=shape,
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === read grid collectively (CGD)
        
        dset = self['dims/x']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                x = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            x = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read x', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/y']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                y = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            y = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read y', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/z']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                z = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            z = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        # === get metric tensor (CGD / curvilinear)
        
        t_start = timeit.default_timer()
        
        M = get_metric_tensor_3d(x, y, z, acc=acc, edge_stencil=edge_stencil, verbose=verbose, no_warn=True)
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get metric tensor','%0.3f [s]'%(t_delta,), s=True) )
        
        # === split out the metric tensor components
        
        ddx_q1 = np.copy( M[:,:,:,0,0] ) ## ξ_x
        ddx_q2 = np.copy( M[:,:,:,1,0] ) ## η_x
        ddx_q3 = np.copy( M[:,:,:,2,0] ) ## ζ_x
        ##
        ddy_q1 = np.copy( M[:,:,:,0,1] ) ## ξ_y
        ddy_q2 = np.copy( M[:,:,:,1,1] ) ## η_y
        ddy_q3 = np.copy( M[:,:,:,2,1] ) ## ζ_y
        ##
        ddz_q1 = np.copy( M[:,:,:,0,2] ) ## ξ_z
        ddz_q2 = np.copy( M[:,:,:,1,2] ) ## η_z
        ddz_q3 = np.copy( M[:,:,:,2,2] ) ## ζ_z
        
        M = None; del M ## free memory
        
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        #if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## the 'computational' grid (unit Cartesian)
        #x_comp = np.arange(nx, dtype=np.float64)
        #y_comp = np.arange(ny, dtype=np.float64)
        #z_comp = np.arange(nz, dtype=np.float64)
        x_comp = 1.
        y_comp = 1.
        z_comp = 1.
        
        # === main loop
        
        if verbose:
            #progress_bar = tqdm(total=self.nt, ncols=100, desc='calc ω', leave=False, file=sys.stdout)
            progress_bar = tqdm(total=len(ctl), ncols=100, desc='calc ω', leave=False, file=sys.stdout)
        
        #for ti in self.ti:
        for ctl_ in ctl:
            ct1, ct2 = ctl_
            ntc = ct2 - ct1
            
            # === read u,v,w
            
            dset = self['data/u']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    u = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                u = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read u', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/v']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    v = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                v = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read v', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/w']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    w = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                w = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # ==========================================================
            # get velocity gradient (strain) tensor ∂(u,v,w)/∂(x,y,z)
            # ==========================================================
            
            t_start = timeit.default_timer()
            
            # === ∂(u)/∂(x,y,z)
            
            ## rectilinear case
            # ddx_u = gradient(u, x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddy_u = gradient(u, y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddz_u = gradient(u, z, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            ## get computational grid gradient elements
            ddx_u_comp = gradient(u, x_comp, axis=0, acc=acc, d=1, no_warn=True)
            ddy_u_comp = gradient(u, y_comp, axis=1, acc=acc, d=1, no_warn=True)
            ddz_u_comp = gradient(u, z_comp, axis=2, acc=acc, d=1, no_warn=True)
            u = None; del u ## free memory
            
            ## get physical grid gradient elements by taking
            ##  inner product with metric tensor
            ddx_u = np.copy( ddx_u_comp * ddx_q1[:,:,:,np.newaxis] +
                             ddy_u_comp * ddx_q2[:,:,:,np.newaxis] +
                             ddz_u_comp * ddx_q3[:,:,:,np.newaxis] )
            
            ddy_u = np.copy( ddx_u_comp * ddy_q1[:,:,:,np.newaxis] +
                             ddy_u_comp * ddy_q2[:,:,:,np.newaxis] +
                             ddz_u_comp * ddy_q3[:,:,:,np.newaxis] )
            
            ddz_u = np.copy( ddx_u_comp * ddz_q1[:,:,:,np.newaxis] +
                             ddy_u_comp * ddz_q2[:,:,:,np.newaxis] +
                             ddz_u_comp * ddz_q3[:,:,:,np.newaxis] )
            
            ddx_u_comp = None; del ddx_u_comp ## free memory
            ddy_u_comp = None; del ddy_u_comp ## free memory
            ddz_u_comp = None; del ddz_u_comp ## free memory
            
            # === ∂(v)/∂(x,y,z)
            
            ## rectilinear case
            # ddx_v = gradient(v, x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddy_v = gradient(v, y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddz_v = gradient(v, z, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            ddx_v_comp = gradient(v, x_comp, axis=0, acc=acc, d=1, no_warn=True)
            ddy_v_comp = gradient(v, y_comp, axis=1, acc=acc, d=1, no_warn=True)
            ddz_v_comp = gradient(v, z_comp, axis=2, acc=acc, d=1, no_warn=True)
            v = None; del v ## free memory
            
            ddx_v = np.copy( ddx_v_comp * ddx_q1[:,:,:,np.newaxis] + \
                             ddy_v_comp * ddx_q2[:,:,:,np.newaxis] + \
                             ddz_v_comp * ddx_q3[:,:,:,np.newaxis] )
            
            ddy_v = np.copy( ddx_v_comp * ddy_q1[:,:,:,np.newaxis] + \
                             ddy_v_comp * ddy_q2[:,:,:,np.newaxis] + \
                             ddz_v_comp * ddy_q3[:,:,:,np.newaxis] )
            
            ddz_v = np.copy( ddx_v_comp * ddz_q1[:,:,:,np.newaxis] + \
                             ddy_v_comp * ddz_q2[:,:,:,np.newaxis] + \
                             ddz_v_comp * ddz_q3[:,:,:,np.newaxis] )
            
            ddx_v_comp = None; del ddx_v_comp ## free memory
            ddy_v_comp = None; del ddy_v_comp ## free memory
            ddz_v_comp = None; del ddz_v_comp ## free memory
            
            # === ∂(w)/∂(x,y,z)
            
            ## rectilinear case
            # ddx_w = gradient(w, x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddy_w = gradient(w, y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddz_w = gradient(w, z, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            ddx_w_comp = gradient(w, x_comp, axis=0, acc=acc, d=1, no_warn=True)
            ddy_w_comp = gradient(w, y_comp, axis=1, acc=acc, d=1, no_warn=True)
            ddz_w_comp = gradient(w, z_comp, axis=2, acc=acc, d=1, no_warn=True)
            w = None; del w ## free memory
            
            ddx_w = np.copy( ddx_w_comp * ddx_q1[:,:,:,np.newaxis] + \
                             ddy_w_comp * ddx_q2[:,:,:,np.newaxis] + \
                             ddz_w_comp * ddx_q3[:,:,:,np.newaxis] )
            
            ddy_w = np.copy( ddx_w_comp * ddy_q1[:,:,:,np.newaxis] + \
                             ddy_w_comp * ddy_q2[:,:,:,np.newaxis] + \
                             ddz_w_comp * ddy_q3[:,:,:,np.newaxis] )
            
            ddz_w = np.copy( ddx_w_comp * ddz_q1[:,:,:,np.newaxis] + \
                             ddy_w_comp * ddz_q2[:,:,:,np.newaxis] + \
                             ddz_w_comp * ddz_q3[:,:,:,np.newaxis] )
            
            ddx_w_comp = None; del ddx_w_comp ## free memory
            ddy_w_comp = None; del ddy_w_comp ## free memory
            ddz_w_comp = None; del ddz_w_comp ## free memory
            
            # ===
            
            ## free memory
            u_ = None; del u_
            v_ = None; del v_
            w_ = None; del w_
            gc.collect()
            
            # ===
            
            ## 4D [scalar][x,y,z] structured array
            names = ['vort_x', 'vort_y', 'vort_z']
            formats  = [ np.float32 for s in names ]
            vort = np.zeros(shape=ddx_u.shape, dtype={'names':names, 'formats':formats})
            
            vort['vort_x'] = ddy_w - ddz_v
            vort['vort_y'] = ddz_u - ddx_w
            vort['vort_z'] = ddx_v - ddy_u
            
            ## free memory
            ddx_u = None; del ddx_u
            ddy_u = None; del ddy_u
            ddz_u = None; del ddz_u
            ddx_v = None; del ddx_v
            ddy_v = None; del ddy_v
            ddz_v = None; del ddz_v
            ddx_w = None; del ddx_w
            ddy_w = None; del ddy_w
            ddz_w = None; del ddz_w
            gc.collect()
            
            for scalar in vort.dtype.names:
                dset = self[f'data/{scalar}']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ct1:ct2,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = vort[scalar][xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ct1:ct2,:,:,:] = vort[scalar].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print(f'write {scalar}','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            if verbose: progress_bar.update()
            #if verbose and (ti<self.nt-1): tqdm.write( '---' )
        if verbose: progress_bar.close()
        if verbose: print(72*'-')
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_vorticity() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vorticity_tangnorm(self, **kwargs):
        '''
        calculate vorticity vector [ω_ξ,ω_η]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx           = kwargs.get('rx',1)
        ry           = kwargs.get('ry',1)
        rz           = kwargs.get('rz',1)
        ct           = kwargs.get('ct',1) ## n chunks [t]
        
        acc          = kwargs.get('acc',6)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        d = 1 ## derivative order
        stencil_npts = 2*int(np.floor((d+1)/2)) - 1 + acc
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        
        # ===
        
        if verbose: print('\n'+'cgd.calc_vorticity_tangnorm()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('csys/vtang' in self):
            raise ValueError('csys/vtang not in hdf5')
        if not ('csys/vnorm' in self):
            raise ValueError('csys/vnorm not in hdf5')
        if not ('data/utang' in self):
            raise ValueError('data/utang not in hdf5')
        if not ('data/unorm' in self):
            raise ValueError('data/unorm not in hdf5')
        if not ('dims/snorm' in self):
            raise ValueError('dims/snorm not in hdf5')
        if not ('dims/z' in self):
            raise ValueError('dims/z not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ct>self.nt):
            raise AssertionError('ct>self.nt')
        
        if ( self['csys/vnorm'].shape != (self.nx,self.ny,2)):
            raise ValueError(f'csys/vnorm shape != (nx,ny,2)')
        if ( self['csys/vtang'].shape != (self.nx,self.ny,2)):
            raise ValueError(f'csys/vtang shape != (nx,ny,2)')
        if ( self['dims/snorm'].shape != (self.ny,)):
            raise ValueError(f'dims/snorm shape != (ny,)')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## time chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # === extend the rank ranges (spatial range overlap)
        
        if self.usingmpi:
            
            n_overlap = stencil_npts_one_side + 3
            
            ## default start:end indices for local-->global write
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            zA = 0
            zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## ## overlap in [y]
            ## if (t4d[1]!=0):
            ##     ry1, ry2 = ry1-n_overlap, ry2
            ##     yA += n_overlap
            ##     yB += n_overlap
            ## if (t4d[1]!=ry-1):
            ##     ry1, ry2 = ry1, ry2+n_overlap
            
            ## overlap in [y] --> TODO: needs checking
            if (t4d[1]!=0):
                ry1 = ry1-n_overlap
                #ry2 = ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                #ry1 = ry1
                ry2 = min(ry2+n_overlap,self.ny)
            
            if (ry2>self.ny):
                #raise IndexError
                print('ry2>self.ny')
                self.comm.Abort(1)
            if (ry1<0):
                #raise IndexError
                print('ry1<0')
                self.comm.Abort(1)
            
            ## overlap in [z]
            if (t4d[2]!=0):
                rz1, rz2 = rz1-n_overlap, rz2
                zA += n_overlap
                zB += n_overlap
            if (t4d[2]!=rz-1):
                rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        #time.sleep(1)
        #if verbose: print('')
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        dset = self['dims/snorm']
        snorm = np.copy( dset[ry1:ry2] )
        
        ## read rank-local tang/norm basis vectors vtang & vnorm
        dset = self['csys/vtang']
        vtang = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        dset = self['csys/vnorm']
        vnorm = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        
        ## convert vtang,vnorm (nx,ny,2) --> (nx,ny,nz,3)
        vtang3d = np.zeros((nxr,nyr,nzr,3), dtype=np.float64)
        vnorm3d = np.zeros((nxr,nyr,nzr,3), dtype=np.float64)
        vspan3d = np.zeros((nxr,nyr,nzr,3), dtype=np.float64)
        
        vtang3d[:,:,:,:2] = vtang[:,:,np.newaxis,:]
        vnorm3d[:,:,:,:2] = vnorm[:,:,np.newaxis,:]
        vspan3d[:,:,:,-1] = 1.
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        
        data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        # === initialize 4D arrays in HDF5
        
        for dsn in ['vort_tang']:
            
            self.scalars_dtypes_dict[dsn] = dtype
            
            if verbose:
                even_print(f'initializing data/{dsn}','%0.2f [GB]'%(data_gb,))
            if (f'data/{dsn}' in self):
                del self[f'data/{dsn}']
            dset = self.create_dataset( f'data/{dsn}',
                                        shape=shape,
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === read grid collectively (CGD)
        
        dset = self['dims/x']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                x = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            x = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read x', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/y']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                y = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            y = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read y', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/z']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                z = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            z = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        # ===
        
        ## assuming dim [z] does not require curvilinear description
        np.testing.assert_allclose(z[0,0,:], z[-1,-1,:], rtol=1e-14, atol=1e-14)
        z1d = np.copy(z[0,0,:])
        
        # === get metric tensor (CGD / curvilinear)
        
        ### t_start = timeit.default_timer()
        ### 
        ### M = get_metric_tensor_3d(x, y, z, acc=acc, edge_stencil=edge_stencil, verbose=verbose, no_warn=True)
        ### 
        ### t_delta = timeit.default_timer() - t_start
        ### if verbose: tqdm.write( even_print('get metric tensor','%0.3f [s]'%(t_delta,), s=True) )
        ### 
        ### # === split out the metric tensor components
        ### 
        ### ddx_q1 = np.copy( M[:,:,:,0,0] ) ## ξ_x
        ### ddx_q2 = np.copy( M[:,:,:,1,0] ) ## η_x
        ### ddx_q3 = np.copy( M[:,:,:,2,0] ) ## ζ_x
        ### ##
        ### ddy_q1 = np.copy( M[:,:,:,0,1] ) ## ξ_y
        ### ddy_q2 = np.copy( M[:,:,:,1,1] ) ## η_y
        ### ddy_q3 = np.copy( M[:,:,:,2,1] ) ## ζ_y
        ### ##
        ### ddz_q1 = np.copy( M[:,:,:,0,2] ) ## ξ_z
        ### ddz_q2 = np.copy( M[:,:,:,1,2] ) ## η_z
        ### ddz_q3 = np.copy( M[:,:,:,2,2] ) ## ζ_z
        ### 
        ### M = None; del M ## free memory
        ### 
        ### mem_total_gb = psutil.virtual_memory().total/1024**3
        ### mem_avail_gb = psutil.virtual_memory().available/1024**3
        ### mem_free_gb  = psutil.virtual_memory().free/1024**3
        ### if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        ### #if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        ### if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        ### if verbose: print(72*'-')
        
        ### ## the 'computational' grid (unit Cartesian)
        ### #x_comp = np.arange(nx, dtype=np.float64)
        ### #y_comp = np.arange(ny, dtype=np.float64)
        ### #z_comp = np.arange(nz, dtype=np.float64)
        ### x_comp = 1.
        ### y_comp = 1.
        ### z_comp = 1.
        
        # === main loop
        
        vort_tang_calc_method = 1 ## 1,2
        
        # option 1: calculate vort_s1 directly using [w/u3] & [unorm/u2] over coord vecs [snorm/s2] & [z/s3]
        #   - in Cartesian : vort_x = ddy_w - ddz_v
        #   - in s1,s2,s3  : vort_s1 = dds2_u3 - dds3_u2
        #
        # option 2: rotate [ω_x,ω_y,ω_z] --> [ω_s1,ω_s2,ω_s3]
        # 
        # this has been tested, the results are visually, although not numerically, identical.
        # method 1 is probably a little bit more accurate, especially for x-planes, because 
        # one doesn't have to rely on the accuracy of a 1D gradient over a very short dimension i.e. ddx_w
        # which is in vort_y = ddz_u - ddx_w
        # 
        # vort_x = ddy_w - ddz_v
        # vort_y = ddz_u - ddx_w
        # vort_z = ddx_v - ddy_u
        #
        # --> vort_s1 = vort_tang = ntang[0]·vort_x + ntang[1]·vort_y + ...
        #                                                      ------      
        
        if verbose:
            progress_bar = tqdm(total=len(ctl), ncols=100, desc='calc vort_tang', leave=False, file=sys.stdout)
        
        #for ti in self.ti:
        for ctl_ in ctl:
            ct1, ct2 = ctl_
            ntc = ct2 - ct1
            
            # === read utang, unorm, w
            
            if (vort_tang_calc_method == 1):
                
                dset = self['data/utang']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        utang = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    utang = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read utang', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                dset = self['data/unorm']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        unorm = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    unorm = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read unorm', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                dset = self['data/w']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        w = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    w = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # ===
            
            if (vort_tang_calc_method == 2):
                
                dset = self['data/vort_x']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        vort_x = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    vort_x = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read vort_x', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                dset = self['data/vort_y']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        vort_y = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    vort_y = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read vort_y', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                dset = self['data/vort_z']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        vort_z = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    vort_z = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read vort_z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # === calculate (1)
            
            if (vort_tang_calc_method == 1):
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## in Cartesian: vort_x = ddy_w - ddz_v
                ## in s1,s2,s3:  vort_s1 = dds2_u3 - dds3_u2
                dds2_w    = gradient(w,     snorm, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
                ddz_unorm = gradient(unorm, z1d,   axis=2, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
                
                vort_tang = dds2_w - ddz_unorm
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print('calc vort_tang','%0.3f [s]'%(t_delta,), s=True))
            
            # === calculate (2)
            
            if (vort_tang_calc_method == 2):
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## stack vorticities into vector
                vort = np.stack((vort_x,vort_y,vort_z), axis=-1)
                if (vort.ndim != 5):
                    raise ValueError
                
                ## rotate vorticity vector
                vort_tang = np.einsum('xyzi,xyzti->xyzt', vtang3d, vort)
                #vort_norm = np.einsum('xyzi,xyzti->xyzt', vnorm3d, vort)
                #vort_span = np.einsum('xyzi,xyzti->xyzt', vspan3d, vort)
                
                ## this fails, but understandably so
                ## rtol will never match well because the variable is centered (logarithmically) around 0
                ## atol does better but high spatial gradients make it tough to get a perfect match
                ## 
                # np.testing.assert_allclose(vort_tang, vort_tang_bak, atol=1e-3)
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print('calc vort_tang','%0.3f [s]'%(t_delta,), s=True))
            
            # === write
            
            dset = self['data/vort_tang']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    dset[ct1:ct2,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = vort_tang[xA:xB,yA:yB,zA:zB].T
            else:
                dset[ct1:ct2,:,:,:] = vort_tang.T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = ntc * 4 * self.nx * self.ny * self.nz / 1024**3
            
            if verbose: tqdm.write(even_print(f'write vort_tang','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            # dset = self['data/vort_tang_2']
            # if self.usingmpi: self.comm.Barrier()
            # t_start = timeit.default_timer()
            # if self.usingmpi:
            #     with dset.collective:
            #         dset[ct1:ct2,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = vort_tang_2[xA:xB,yA:yB,zA:zB].T
            # else:
            #     dset[ct1:ct2,:,:,:] = vort_tang_2.T
            # if self.usingmpi: self.comm.Barrier()
            # t_delta = timeit.default_timer() - t_start
            # data_gb = 4 * self.nx * self.ny * self.nz / 1024**3
            # 
            # if verbose: tqdm.write(even_print(f'write vort_tang_2','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_vorticity_tangnorm() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # ===
    
    def calc_turb_spectrum_span_xpln(self, **kwargs):
        '''
        calculate FFT in [z] (wavenumber) at every [x,y,t], avg in [x,t]
        - designed for analyzing unsteady, thin planes in [x]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.calc_turb_spectrum_span_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_dat_fft  = kwargs.get('fn_dat_fft',None)
        fn_cgd_mean = kwargs.get('fn_cgd_mean',None)
        
        ## assert that a CGD with (fsubtype=='prime') was opened
        if (self.fsubtype!='prime'):
            raise ValueError("fsubtype!='prime'")
        
        ## for now only distribute data in [y] --> allows [x,t] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # === distribute 4D data over ranks
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # # === mean file name (for reading)
        # if (fn_cgd_mean is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     if ('_prime' in fname_root):
        #         fname_root = fname_root.replace('_prime','')
        #     fname_mean_h5_base = fname_root+'_mean.h5'
        #     #fn_cgd_mean = os.path.join(fname_path, fname_mean_h5_base)
        #     fn_cgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
        #     #fn_cgd_mean = Path(fname_path, fname_mean_h5_base)
        
        # === fft file name (for writing) : dat
        if (fn_dat_fft is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_dat_base = fname_root+'_turb_spec_span.dat'
            fn_dat_fft = str(PurePosixPath(fname_path, fname_fft_dat_base))
        
        if verbose: even_print('fn_cgd_prime'    , self.fname   )
        #if verbose: even_print('fn_cgd_mean'     , fn_cgd_mean  )
        if verbose: even_print('fn_dat_fft'      , fn_dat_fft   )
        if verbose: print(72*'-')
        
        #if not os.path.isfile(fn_cgd_mean):
        #    raise FileNotFoundError('%s not found'%fn_cgd_mean)
        
        #with cgd(fn_cgd_mean, 'r', driver=self.driver, comm=self.comm) as hf_mean:
        #    if (self.fsubtype!='mean'):
        #        raise ValueError("fsubtype!='mean'")
        
        # ===
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho   = np.copy( self['data_dim/rho'][()]   )
        utang = np.copy( self['data_dim/utang'][()] )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        u1_99    = float( self['data_dim/u1_99'][()]    )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## these are recalculated and checked in next step
        z1d_ = np.copy( self['data_dim/z1d'][()] )
        dz0_ = np.copy( self['data_dim/dz0'][()] )
        dt_  = np.copy( self['data_dim/dt'][()]  )
        
        # ===
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 3D coordinate arrays, then dimensionalize [m]
        ## every ranks gets full grid
        x = np.copy( self['dims/x'][()].T * self.lchar )
        y = np.copy( self['dims/y'][()].T * self.lchar )
        z = np.copy( self['dims/z'][()].T * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        ## read in 1D coordinate arrays, then dimensinoalize [m]
        stang_ = np.copy(self['dims/stang'])
        stang  = np.copy( stang_ * self.lchar ) ## dimensional [m]
        data['stang'] = stang
        
        snorm_ = np.copy(self['dims/snorm'])
        snorm  = np.copy( snorm_ * self.lchar ) ## dimensional [m]
        data['snorm'] = snorm
        
        ## assert [z] is same over all [x,y]
        if (z.ndim!=3):
            raise ValueError
        z1d = np.copy(z[0,0,:])
        for i in range(nx):
            for j in range(ny):
                np.testing.assert_allclose(z1d, z[i,j,:], rtol=1e-14, atol=1e-14)
        
        ## assert [x,y] is same over all [z]
        x2d  = np.copy(x[:,:,0])
        y2d  = np.copy(y[:,:,0])
        xy2d = np.stack((x2d,y2d), axis=-1)
        for k in range(nz):
            x2d_  = np.copy(x[:,:,k])
            y2d_  = np.copy(y[:,:,k])
            xy2d_ = np.stack((x2d_,y2d_), axis=-1)
            np.testing.assert_allclose(xy2d, xy2d_, rtol=1e-14, atol=1e-14)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z1d.max() - z1d.min()
        
        #data['x'] = x
        #data['y'] = y
        #data['z'] = z
        data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        # === report
        if verbose:
            #even_print('nx'     , '%i'        %nx     )
            #even_print('ny'     , '%i'        %ny     )
            #even_print('nz'     , '%i'        %nz     )
            #even_print('nt'     , '%i'        %nt     )
            #even_print('dt'     , '%0.5e [s]' %dt     )
            #even_print('t_meas' , '%0.5e [s]' %t_meas )
            even_print('dz0'    , '%0.5e [m]' %dz0     )
            even_print('zrange' , '%0.5e [m]' %zrange  )
            print(72*'-')
        
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+' , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'Δt+' , '%0.3f'%(dt/sc_t_in) )
            print(72*'-')
        
        ## get spanwise wavenumber [kz] vector
        kz_full = sp.fft.fftfreq(n=nz, d=dz0) * ( 2 * np.pi )
        kzp     = np.where(kz_full>0)
        kz      = np.copy(kz_full[kzp])
        dkz     = kz[1]-kz[0]
        nkz     = kz.size
        
        data['kz']  = kz
        data['dkz'] = dkz
        data['nkz'] = nkz
        
        if verbose:
            even_print('kz min','%0.1f [1/m]'%kz.min())
            even_print('kz max','%0.1f [1/m]'%kz.max())
            even_print('dkz','%0.1f [1/m]'%dkz)
            even_print('nkz','%i'%nkz)
            
            kz_inner = np.copy( kz * sc_l_in  )
            kz_outer = np.copy( kz * sc_l_out )
            
            even_print('(kz·δ_ν) min' , '%0.5e [-]'%kz_inner.min())
            even_print('(kz·δ_ν) max' , '%0.5f [-]'%kz_inner.max())
            even_print('(kz·δ99) min' , '%0.5f [-]'%kz_outer.min())
            even_print('(kz·δ99) max' , '%0.5f [-]'%kz_outer.max())
            
            print(72*'-')
        
        # === read in data (prime) --> still dimless (char)
        
        #scalars = [ 'uI','vI','wI' , 'rho','uII','vII','wII' ]
        scalars = [ 'uI','vI','wI', 'utangI', 'unormI' ]
        scalars_dtypes = [self.scalars_dtypes_dict[s] for s in scalars]
        
        ## [var1, var2, density_scaling]
        fft_combis = [
                     # [ 'uI'  , 'uI'  , False ],
                     # [ 'vI'  , 'vI'  , False ],
                     # [ 'wI'  , 'wI'  , False ],
                     # [ 'uI'  , 'vI'  , False ],
                     # [ 'uII' , 'uII' , True  ],
                     # [ 'vII' , 'vII' , True  ],
                     # [ 'wII' , 'wII' , True  ],
                     # [ 'uII' , 'vII' , True  ],
                     ##
                     [ 'utangI' , 'utangI' , False ],
                     [ 'unormI' , 'unormI' , False ],
                     [ 'wI'     , 'wI'     , False ],
                     [ 'utangI' , 'unormI' , False ],
                     [ 'utangI' , 'wI'     , False ],
                     ]
        
        scalars_dtypes = [ self.scalars_dtypes_dict[s] for s in scalars ]
        
        ## dtype of prime data (currently must be all same dtype)
        if np.all( [ (dtp==np.float32) for dtp in scalars_dtypes ] ):
            dtype_primes = np.float32
        elif np.all( [ (dtp==np.float64) for dtp in scalars_dtypes ] ):
            dtype_primes = np.float64
        else:
            raise NotImplementedError
        
        ## 5D [scalar][x,y,z,t] structured array
        data_prime = np.zeros(shape=(self.nx, nyr, self.nz, self.nt), dtype={'names':scalars, 'formats':scalars_dtypes})
        
        for scalar in scalars:
            
            dset = self[f'data/{scalar}']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            
            self.comm.Barrier()
            t_start = timeit.default_timer()
            with dset.collective:
                data_prime[scalar] = np.copy( dset[:,:,ry1:ry2,:].T )
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            
            data_gb = float_bytes * self.nx * self.ny * self.nz * self.nt / 1024**3
            
            if verbose:
                even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        # === redimensionalize prime data
        
        for var in data_prime.dtype.names:
            if var in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII', 'utangI','unormI', 'utangII','unormII']:
                data_prime[var] *= U_inf
            elif var in ['r_uII','r_vII','r_wII']:
                data_prime[var] *= (U_inf*rho_inf)
            elif var in ['T','TI','TII']:
                data_prime[var] *= T_inf
            elif var in ['r_TII']:
                data_prime[var] *= (T_inf*rho_inf)
            elif var in ['rho','rhoI']:
                data_prime[var] *= rho_inf
            elif var in ['p','pI','pII']:
                data_prime[var] *= (rho_inf * U_inf**2)
            else:
                raise ValueError('condition needed for redimensionalizing \'%s\''%var)
        
        ## force the ∫PSD == (co)variance
        ## this usually represents about a 1-2% power correction which comes about due to 
        ##   non-stationarity of windowed data
        normalize_psd_by_cov = False
        
        ## initialize buffers
        Euu_scalars         = [ '%s%s'%(cc[0],cc[1]) for cc in fft_combis ]
        Euu_scalars_dtypes  = [ dtype_primes for s in Euu_scalars ]
        Euu                 = np.zeros(shape=(self.nx, nyr, self.nt, nkz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg            = np.zeros(shape=(self.nx, nyr, self.nt)      , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        if normalize_psd_by_cov:
            energy_norm_fac_arr = np.zeros(shape=(self.nx, nyr, self.nt) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes}) ## just for monitoring
        
        ## check memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose:
            even_print('mem total',     '%0.1f [GB]'%(mem_total_gb,))
            even_print('mem available', '%0.1f [GB] / %0.1f[%%]'%(mem_avail_gb,(100*mem_avail_gb/mem_total_gb)))
            even_print('mem free',      '%0.1f [GB] / %0.1f[%%]'%(mem_free_gb,(100*mem_free_gb/mem_total_gb)))
        
        ## for spanwise mean, no overlapping windows are applied, so win_len = [z] vec length
        win_len = nz
        
        ## the window function
        window_type = 'tukey'
        if (window_type=='tukey'):
            window = sp.signal.windows.tukey(win_len, alpha=0.1)
        elif (window_type is None):
            window = np.ones(win_len, dtype=np.float64)
        if verbose:
            even_print('window type', '\'%s\''%str(window_type))
        
        ## sum of sqrt of window: needed for power normalization
        sum_sqrt_win = np.sum(np.sqrt(window))
        
        # === main loop
        
        self.comm.Barrier()
        if verbose: progress_bar = tqdm(total=self.nx*nyr*self.nt, ncols=100, desc='fft', leave=False)
        for xi in range(self.nx):
            for yi in range(nyr):
                for ti in range(self.nt):
                    for cci,cc in enumerate(fft_combis):
                        
                        tag = Euu_scalars[cci]
                        ccL,ccR,density_scaling = cc
                        
                        uL = np.copy( data_prime[ccL][xi,yi,:,ti] )
                        uR = np.copy( data_prime[ccR][xi,yi,:,ti] )
                        
                        if ('rho' in data_prime.dtype.names):
                            rho     = np.copy( data_prime['rho'][xi,yi,:,ti] )
                            rho_avg = np.mean( rho, dtype=np.float64 )
                        else:
                            rho     = None
                            rho_avg = None
                        
                        # ===
                        
                        if density_scaling:
                            uIuI_avg_ijk = np.mean(uL*uR*rho, dtype=np.float64) / rho_avg
                        else:
                            uIuI_avg_ijk = np.mean(uL*uR,     dtype=np.float64)
                        
                        uIuI_avg[tag][xi,yi,ti] = uIuI_avg_ijk
                        
                        if density_scaling:
                            ui = np.copy( uL * rho )
                            uj = np.copy( uR * rho )
                        else:
                            ui = np.copy( uL )
                            uj = np.copy( uR )
                        
                        n     = ui.size
                        #A_ui = sp.fft.fft(ui)[fp] / n
                        #A_uj = sp.fft.fft(uj)[fp] / n
                        ui   *= window
                        uj   *= window
                        #ui  -= np.mean(ui) ## de-trend
                        #uj  -= np.mean(uj)
                        A_ui    = sp.fft.fft(ui)[kzp] / sum_sqrt_win
                        A_uj    = sp.fft.fft(uj)[kzp] / sum_sqrt_win
                        Euu_ijk = 2 * np.real(A_ui*np.conj(A_uj)) / dkz
                        
                        ## divide off mean mass density
                        if density_scaling:
                            Euu_ijk /= rho_avg**2
                        
                        ## normalize such that ∫PSD=(co)variance
                        if normalize_psd_by_cov:
                            if (uIuI_avg_ijk!=0.):
                                energy_norm_fac = np.sum(dkz*Euu_ijk) / uIuI_avg_ijk
                            else:
                                energy_norm_fac = 1.
                            Euu_ijk /= energy_norm_fac
                            energy_norm_fac_arr[tag][xi,yi,ti] = energy_norm_fac
                        
                        ## write
                        Euu[tag][xi,yi,ti,:] = Euu_ijk
                    
                    if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        ## report energy normalization factors --> tmp,only rank 0 currently!
        if normalize_psd_by_cov:
            if verbose: print(72*'-')
            for tag in Euu_scalars:
                energy_norm_fac_min = energy_norm_fac_arr[tag].min()
                energy_norm_fac_max = energy_norm_fac_arr[tag].max()
                energy_norm_fac_avg = np.mean(energy_norm_fac_arr[tag], axis=(0,1,2), dtype=np.float64)
                if verbose:
                    even_print('energy norm min/max/avg : %s'%tag, '%0.4f / %0.4f / %0.4f'%(energy_norm_fac_min,energy_norm_fac_max,energy_norm_fac_avg))
            energy_norm_fac_arr = None ; del energy_norm_fac_arr
        
        # === average in [x,t] --> leave [y,kz]
        
        Euu_      = np.zeros(shape=(nyr,nkz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg_ = np.zeros(shape=(nyr,)    , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        self.comm.Barrier()
        
        for tag in Euu_scalars:
            Euu_[tag]      = np.mean( Euu[tag]      , axis=(0,2) , dtype=np.float64) #.astype(np.float32) ## avg in [x,t] --> leave [y,kz]
            uIuI_avg_[tag] = np.mean( uIuI_avg[tag] , axis=(0,2) , dtype=np.float64) #.astype(np.float32) ## avg in [x,t] --> leave [y]
        Euu      = np.copy( Euu_ )
        uIuI_avg = np.copy( uIuI_avg_ )
        self.comm.Barrier()
        
        # === gather all results
        # --> arrays are very small at this point, just do 'lazy' gather/bcast, dont worry about buffers etc
        
        G = self.comm.gather([self.rank, 
                              Euu, uIuI_avg ], root=0)
        G = self.comm.bcast(G, root=0)
        
        Euu      = np.zeros( (ny,nkz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg = np.zeros( (ny,)    , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        
        for ri in range(self.n_ranks):
            j = ri
            for GG in G:
                if (GG[0]==ri):
                    for tag in Euu_scalars:
                        Euu[tag][ryl[j][0]:ryl[j][1],:]    = GG[1][tag]
                        uIuI_avg[tag][ryl[j][0]:ryl[j][1]] = GG[2][tag]
                else:
                    pass
        if verbose: print(72*'-')
        
        # === save results
        if (self.rank==0):
            
            data['Euu']      = Euu
            data['uIuI_avg'] = uIuI_avg
            
            with open(fn_dat_fft,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_fft,os.path.getsize(fn_dat_fft)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_turb_spectrum_span_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_turb_spectrum_time_xpln(self, **kwargs):
        raise NotImplementedError
        return
    
    # === polydata
    
    def export_polydata_wall(self, fn_spd=None, **kwargs):
        '''
        get 2D [x,z] wall quantities, export structured polydata (SPD) file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        force    = kwargs.get('force',False)
        
        acc = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(None,None,1)) ## the 'constraint' parameter for sizing h5 chunks (i,j,t)
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 32 ) ## for initializing SPD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8  )
        
        if verbose: print('\n'+'cgd.export_polydata_wall()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/utang' in self) and self.curvilinear:
            raise ValueError('data/utang not in hdf5')
        if not ('data/unorm' in self) and self.curvilinear:
            raise ValueError('data/unorm not in hdf5')
        if not ('dims/snorm' in self) and self.curvilinear:
            raise ValueError('dims/snorm not in hdf5')
        if not ('dims/stang' in self) and self.curvilinear:
            raise ValueError('dims/stang not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        #if (ry>self.ny):
        #    raise AssertionError('ry>self.ny')
        if (ry!=1):
            raise AssertionError('ry!=1')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## polydata file name (for writing)
        if (fn_spd is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_spd_h5_base = fname_root+'_polydata_wall.h5'
            #fn_spd = os.path.join(fname_path, fname_spd_h5_base)
            fn_spd = str(PurePosixPath(fname_path, fname_spd_h5_base))
            #fn_spd = Path(fname_path, fname_spd_h5_base)
        
        # === ranks
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## [t] chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # ===
        
        ## only take first N wall-normal points
        nyr = min(20,self.ny)
        ry1, ry2 = 0, 0+nyr
        
        data_gb = nyr * (self.nx * self.nz) * 4 / 1024**3
        
        ## get [snorm] or [y]
        if ('dims/snorm' in self):
            dset = self['dims/snorm']
            snorm = np.copy( dset[ry1:ry2] )
        else:
            if self.rectilinear:
                dset = self['dims/y']
                snorm = np.copy( dset[0,ry1:ry2,0].T )
            else:
                raise ValueError
        
        ## 3D polydata grid coordinates : shape (nx,nz,3)
        xyz_wall = np.zeros((self.nx, self.nz, 3))
        xyz_wall[:,:,0] = np.copy(self['dims/x'][:,0,:]).T
        xyz_wall[:,:,1] = np.copy(self['dims/y'][:,0,:]).T
        xyz_wall[:,:,2] = np.copy(self['dims/z'][:,0,:]).T
        
        ## open & initialize polydata (SPD) file
        if verbose: even_print('surface polydata (spd) .h5', fn_spd)
        with spd(fn_spd, 'w',
                 force=force,
                 driver=self.driver,
                 comm=MPI.COMM_WORLD,
                 stripe_count=stripe_count,
                 stripe_size_mb=stripe_size_mb) as hfspd:
            
            ## add attributes from CGD to SPD
            for key in self.udef:
                hfspd.attrs[key] = self.udef[key]
            #for key in self.udef_deriv:
            #    hfspd.attrs[key] = self.udef_deriv[key]
            
            ## add time vector from CGD to SPD
            dsn = f'dims/t'
            if (dsn in hfspd):
                del hfspd[dsn]
            data = np.copy(self[dsn][()])
            ds = hfspd.create_dataset(dsn, data=data, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## add 3D polydata grid coordinates : shape (nx,nz,3)
            dsn = f'dims/xyz'
            if (dsn in hfspd):
                del hfspd[dsn]
            ds = hfspd.create_dataset(dsn, data=xyz_wall, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            hfspd.attrs['ni'] = self.nx
            hfspd.attrs['nj'] = self.nz
            hfspd.attrs['n_quads'] = ( self.nx - 1 ) * ( self.nz - 1 )
            hfspd.attrs['n_pts'] = self.nx * self.nz
            hfspd.attrs['nt'] = self.nt
            
            hfspd.get_header(verbose=verbose)
            
            ## add additional [dims/<>] dsets
            for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][()])
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            ## add additional [csys/<>] dsets
            for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][:,0,:]) ## take only vectors at wall [i,j==0,:]
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            if verbose: print(72*'-')
            
            ## initialize datasets
            data_gb = (self.nx * self.nz) * 4 / 1024**3
            shape  = (self.nx,self.nz,self.nt)
            chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=4)
            
            scalars_spd = [ 'tau_u1_s2', 'tau_u3_s2', 'T','rho','mu','nu','u_tau','p' ]
            
            for scalar in scalars_spd:
                
                if verbose:
                    even_print(f'initializing data/{scalar}','%0.1f [GB]'%(data_gb,))
                if (f'data/{scalar}' in hfspd):
                    del hfspd[f'data/{scalar}']
                dset = hfspd.create_dataset(f'data/{scalar}',
                                            shape=shape,
                                            dtype=np.float32,
                                            chunks=chunks )
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    #even_print('chunk shape (t,z,x)','%s'%str(dset.chunks))
                    even_print('chunk shape (x,z,t)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === main loop
        
        ## re-dimensionalize snorm
        snorm *= self.lchar
        
        with spd(fn_spd, 'a', driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
            
            if verbose:
                progress_bar = tqdm(total=ct, ncols=100, desc='calc τ_w', leave=False, file=sys.stdout)
            
            for ctl_ in ctl:
                ct1, ct2 = ctl_
                ntc = ct2 - ct1
                
                if ('data/utang' in self):
                    dset = self['data/utang']
                else:
                    if self.rectilinear:
                        dset = self['data/u']
                    else:
                        raise ValueError
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        utang = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    utang = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read utang', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                
                dset = self['data/w']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        w = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    w = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                
                dset = self['data/rho']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        rho = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    rho = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read rho', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                
                dset = self['data/T']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        T = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    T = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read T', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                
                dset = self['data/p']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        p = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    p = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read p', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                # === re-dimensionalize
                
                T     *= self.T_inf
                utang *= self.U_inf
                w     *= self.U_inf
                rho   *= self.rho_inf
                p     *= self.rho_inf * self.U_inf**2
                
                # ===
                
                # mu1 = (14.58e-7 * T**1.5) / ( T + 110.4 )
                # mu2 = self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * (self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth)
                # mu3 = self.C_Suth * T**(3/2) / (T + self.S_Suth)
                # np.testing.assert_allclose(mu1, mu2, rtol=1e-14, atol=1e-14)
                # np.testing.assert_allclose(mu2, mu3, rtol=1e-14, atol=1e-14)
                # mu = np.copy(mu3)
                
                mu = self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * (self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth)
                nu = mu / rho
                
                # ===
                
                ## ddy[u] equivalent : ddn[utang]
                ddn_utang = gradient(utang, snorm, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
                ddn_utang_wall = np.copy( ddn_utang[:,0,:,:] )
                
                ## ddy[w] equivalent : ddn[w]
                ddn_w = gradient(w, snorm, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
                ddn_w_wall = np.copy( ddn_w[:,0,:,:] )
                
                #tau = np.copy( mu * ddn_utang )
                #tau_wall = np.copy( tau[:,0,:] )
                
                ## dimensional!
                rho_wall = np.copy( rho[:,0,:,:] )
                mu_wall  = np.copy( mu[:,0,:,:]  )
                nu_wall  = np.copy( nu[:,0,:,:]  )
                T_wall   = np.copy( T[:,0,:,:]   )
                p_wall   = np.copy( p[:,0,:,:]   )
                
                tau_wall = np.copy( mu_wall * ddn_utang_wall )
                
                #u_tau = np.sqrt(tau_wall/rho_wall)
                arg = np.copy(tau_wall/rho_wall)
                u_tau = np.sqrt(np.abs(arg)) * np.sign(arg)
                
                tau_u3_s2 = np.copy( mu_wall * ddn_w_wall )
                
                # ===
                
                ## 3D [scalar][x,z] structured array
                dtypes_spd  = [ np.float32 for s in scalars_spd ]
                data4spd = np.zeros(shape=(nxr,nzr,ntc), dtype={'names':scalars_spd, 'formats':dtypes_spd})
                data4spd['tau_u1_s2'][:,:,:] = tau_wall  / ( self.rho_inf * self.U_inf**2 )
                data4spd['tau_u3_s2'][:,:,:] = tau_u3_s2 / ( self.rho_inf * self.U_inf**2 )
                data4spd['T'][:,:,:]         = T_wall    / self.T_inf
                data4spd['rho'][:,:,:]       = rho_wall  / self.rho_inf
                data4spd['p'][:,:,:]         = p_wall    / ( self.rho_inf * self.U_inf**2 )
                data4spd['mu'][:,:,:]        = mu_wall   / self.mu_inf
                data4spd['nu'][:,:,:]        = nu_wall   / self.nu_inf
                data4spd['u_tau'][:,:,:]     = u_tau     / self.U_inf
                
                for scalar in data4spd.dtype.names:
                    
                    dset = hfspd[f'data/{scalar}']
                    if hfspd.usingmpi: hfspd.comm.Barrier()
                    t_start = timeit.default_timer()
                    if hfspd.usingmpi:
                        with dset.collective:
                            #dset[ct1:ct2,rz1:rz2,rx1:rx2] = data4spd[scalar]
                            dset[rx1:rx2,rz1:rz2,ct1:ct2] = data4spd[scalar]
                    else:
                        dset[:,:,ct1:ct2] = data4spd[scalar]
                    if hfspd.usingmpi: hfspd.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = 4 * ntc * 1 * self.nx * self.nz / 1024**3
                    
                    if verbose: tqdm.write(even_print(f'write {scalar}','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.export_polydata_wall() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def export_polydata_xpln(self, fn_spd=None, **kwargs):
        '''
        get 2D [y,z] quantities of a reduced I/O volume which is thin in the [x]/[s1] direction
        and export structured polydata (SPD) file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        force    = kwargs.get('force',False)
        
        acc = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        if verbose: print('\n'+'cgd.export_polydata_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/utang' in self) and self.curvilinear:
            raise ValueError('data/utang not in hdf5')
        if not ('data/unorm' in self) and self.curvilinear:
            raise ValueError('data/unorm not in hdf5')
        if not ('dims/snorm' in self) and self.curvilinear:
            raise ValueError('dims/snorm not in hdf5')
        if not ('dims/stang' in self) and self.curvilinear:
            raise ValueError('dims/stang not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## polydata file name (for writing)
        if (fn_spd is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_spd_h5_base = fname_root+'_polydata.h5'
            #fn_spd = os.path.join(fname_path, fname_spd_h5_base)
            fn_spd = str(PurePosixPath(fname_path, fname_spd_h5_base))
            #fn_spd = Path(fname_path, fname_spd_h5_base)
        
        # === ranks
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ri1 = ry1
        ri2 = ry2
        rj1 = rz1
        rj2 = rz2
        
        ## [t] chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        data_gb = (self.ny * self.nz) * 4 / 1024**3
        
        ## get [snorm] or [y]
        if ('dims/snorm' in self):
            dset = self['dims/snorm']
            snorm = np.copy( dset[ry1:ry2] )
        else:
            if self.rectilinear:
                dset = self['dims/y']
                snorm = np.copy( dset[0,ry1:ry2,0].T )
            else:
                raise ValueError
        
        ## take middle-most x index
        xi = self.nx // 2
        
        ## 3D polydata grid coordinates : shape (ny,nz,3)
        xyz_wall = np.zeros((self.ny, self.nz, 3))
        xyz_wall[:,:,0] = np.copy(self['dims/x'][:,:,xi]).T
        xyz_wall[:,:,1] = np.copy(self['dims/y'][:,:,xi]).T
        xyz_wall[:,:,2] = np.copy(self['dims/z'][:,:,xi]).T
        
        ## open & initialize polydata (SPD) file
        if verbose: even_print('surface polydata (spd) .h5', fn_spd)
        with spd(fn_spd, 'w', force=force, driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
            
            ## add attributes from CGD to SPD
            ## this doesnt work because CGD does not use attributes for its metadata as of writing this func
            # for key,val in self.attrs.items():
            #     hfspd.attrs[key] = val
            
            ## add attributes from CGD to SPD
            for key in self.udef:
                hfspd.attrs[key] = self.udef[key]
            #for key in self.udef_deriv:
            #    hfspd.attrs[key] = self.udef_deriv[key]
            
            ## add time vector from CGD to SPD
            dsn = f'dims/t'
            if (dsn in hfspd):
                del hfspd[dsn]
            data = np.copy(self[dsn][()])
            ds = hfspd.create_dataset(dsn, data=data, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## add 3D polydata grid coordinates : shape (nx,nz,3)
            dsn = f'dims/xyz'
            if (dsn in hfspd):
                del hfspd[dsn]
            ds = hfspd.create_dataset(dsn, data=xyz_wall, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ni = self.ny
            nj = self.nz
            nt = self.nt
            hfspd.attrs['ni'] = ni
            hfspd.attrs['nj'] = nj
            hfspd.attrs['n_quads'] = ( ni - 1 ) * ( nj - 1 )
            hfspd.attrs['n_pts'] = ni * nj
            hfspd.attrs['nt'] = nt
            
            if verbose: print(72*'-')
            hfspd.get_header(verbose=verbose)
            if verbose: print(72*'-')
            
            ## add additional [dims/<>] dsets
            for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][()])
                    
                    if (data.shape==(self.nx,)):
                        data = np.array( [data[xi] ], dtype=data.dtype )
                        if (data.ndim!=1):
                            raise ValueError
                    
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            ## add additional [csys/<>] dsets
            for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][xi,:,:])
                    
                    if (data.ndim!=3):
                        data = data[np.newaxis,:,:]
                    if (data.ndim!=3):
                        raise ValueError
                    
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            if verbose: print(72*'-')
            
            ## initialize datasets
            for scalar in self.scalars:
                
                dsn = f'data/{scalar}'
                dset = self[dsn]
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                
                #shape  = (self.ny,self.nz,self.nt)
                shape  = (ni,nj,nt)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,1), size_kb=chunk_kb, base=4, itemsize=float_bytes)
                
                data_gb = np.prod(shape) * float_bytes / 1024**3
                
                if verbose:
                    even_print(f'initializing {dsn}','%0.1f [GB]'%(data_gb,))
                if (dsn in hfspd):
                    del hfspd[dsn]
                dset = hfspd.create_dataset( dsn,
                                             shape=shape,
                                             dtype=dtype,
                                             chunks=chunks )
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (i,j,t)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === main loop
        
        data_gb_read  = 0.
        data_gb_write = 0.
        t_read  = 0.
        t_write = 0.
        
        with spd(fn_spd, 'a', driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
            
            if verbose:
                progress_bar = tqdm(total=ct*self.n_scalars, ncols=100, desc='export spd', leave=False, file=sys.stdout)
            
            for scalar in self.scalars:
                
                dsn = f'data/{scalar}'
                
                dset_src = self[dsn]
                dset_tgt = hfspd[dsn]
                
                dtype = dset_src.dtype
                float_bytes = dtype.itemsize
                
                for ctl_ in ctl:
                    ct1, ct2 = ctl_
                    ntc = ct2 - ct1
                    
                    ## read
                    self.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset_src.collective:
                        data = np.copy( dset_src[ct1:ct2,rz1:rz2,ry1:ry2,xi].T )
                    self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = float_bytes * ni * nj * ntc / 1024**3
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    if verbose:
                        tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    ## write
                    hfspd.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset_tgt.collective:
                        dset_tgt[ri1:ri2,rj1:rj2,ct1:ct2] = data
                    hfspd.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = float_bytes * ni * nj * ntc / 1024**3
                    
                    t_write       += t_delta
                    data_gb_write += data_gb
                    
                    if verbose:
                        tqdm.write(even_print(f'write: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    if verbose: progress_bar.update()
            
            if verbose: progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.export_polydata_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def export_polydata_zpln(self, fn_spd=None, **kwargs):
        '''
        get 2D [x,y] quantities of a reduced I/O volume which is thin in the [z]/[s3] direction
        and export structured polydata (SPD) file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        force    = kwargs.get('force',False)
        
        acc = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(None,None,1)) ## the 'constraint' parameter for sizing h5 chunks (i,j,t)
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 32 ) ## for initializing SPD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8  )
        
        d = 1 ## derivative order
        stencil_npts = 2*math.floor((d+1)/2) - 1 + acc ## central explicit FD kernel width
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        n_overlap             = stencil_npts_one_side + 3
        
        if verbose: print('\n'+'cgd.export_polydata_zpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/utang' in self) and self.curvilinear:
            raise ValueError('data/utang not in hdf5')
        if not ('data/unorm' in self) and self.curvilinear:
            raise ValueError('data/unorm not in hdf5')
        if not ('dims/snorm' in self) and self.curvilinear:
            raise ValueError('dims/snorm not in hdf5')
        if not ('dims/stang' in self) and self.curvilinear:
            raise ValueError('dims/stang not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## polydata file name (for writing)
        if (fn_spd is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_spd_h5_base = fname_root+'_polydata.h5'
            #fn_spd = os.path.join(fname_path, fname_spd_h5_base)
            fn_spd = str(PurePosixPath(fname_path, fname_spd_h5_base))
            #fn_spd = Path(fname_path, fname_spd_h5_base)
        
        # === ranks
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # ===
        
        ri1 = rx1
        ri2 = rx2
        rj1 = ry1
        rj2 = ry2
        
        ## [t] chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## get [snorm] or [y]
        if ('dims/snorm' in self):
            dset = self['dims/snorm']
            snorm = np.copy( dset[ry1:ry2] )
        else:
            if self.rectilinear:
                dset = self['dims/y']
                snorm = np.copy( dset[0,ry1:ry2,0].T )
            else:
                raise ValueError
        
        ## get [stang] or [x]
        if ('dims/stang' in self):
            dset = self['dims/stang']
            stang = np.copy( dset[rx1:rx2] )
        else:
            if self.rectilinear:
                dset = self['dims/x']
                stang = np.copy( dset[rx1:rx2,ry1:ry2,0].T )
            else:
                raise ValueError
        
        ## get middle-most [z] index
        zi = self.nz // 2
        
        ## 3D polydata grid coordinates : shape (nx,ny,3)
        xyz = np.zeros((self.nx, self.ny, 3))
        xyz[:,:,0] = np.copy(self['dims/x'][zi,:,:]).T
        xyz[:,:,1] = np.copy(self['dims/y'][zi,:,:]).T
        xyz[:,:,2] = np.copy(self['dims/z'][zi,:,:]).T
        
        ## open & initialize polydata (SPD) file
        if verbose: even_print('surface polydata (spd) .h5', fn_spd)
        with spd(fn_spd, 'w',
                 force=force,
                 driver=self.driver,
                 comm=MPI.COMM_WORLD,
                 stripe_count=stripe_count,
                 stripe_size_mb=stripe_size_mb) as hfspd:
            
            ## add attributes from CGD to SPD
            ## this doesnt work because CGD does not use attributes for its metadata as of writing this func
            # for key,val in self.attrs.items():
            #     hfspd.attrs[key] = val
            
            ## add attributes from CGD to SPD
            for key in self.udef:
                hfspd.attrs[key] = self.udef[key]
            #for key in self.udef_deriv:
            #    hfspd.attrs[key] = self.udef_deriv[key]
            
            ## add time vector from CGD to SPD
            dsn = f'dims/t'
            if (dsn in hfspd):
                del hfspd[dsn]
            data = np.copy(self[dsn][()])
            ds = hfspd.create_dataset(dsn, data=data, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## add 3D polydata grid coordinates : shape (nx,nz,3)
            dsn = f'dims/xyz'
            if (dsn in hfspd):
                del hfspd[dsn]
            ds = hfspd.create_dataset(dsn, data=xyz, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ni = self.nx
            nj = self.ny
            nt = self.nt
            hfspd.attrs['ni'] = ni
            hfspd.attrs['nj'] = nj
            hfspd.attrs['n_quads'] = ( ni - 1 ) * ( nj - 1 )
            hfspd.attrs['n_pts'] = ni * nj
            hfspd.attrs['nt'] = nt
            
            if verbose: print(72*'-')
            hfspd.get_header(verbose=verbose)
            if verbose: print(72*'-')
            
            ## add additional [dims/<>] dsets
            for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][()])
                    
                    ## this would clip the additional datasets in [x], but doing nothing for now
                    if (data.shape==(self.nx,)):
                        #data = np.array( [data[xi]], dtype=data.dtype )
                        if (data.ndim!=1):
                            raise ValueError
                    
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            ## add additional [csys/<>] dsets
            for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    
                    #data = np.copy(self[dsn][xi,:,:]) ## clip in [x]
                    data = np.copy(self[dsn][()])
                    
                    if (data.ndim!=3):
                        data = data[np.newaxis,:,:]
                    if (data.ndim!=3):
                        raise ValueError
                    
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            if verbose: print(72*'-')
            
            ## initialize datasets
            #shape   = (self.nx,self.ny,self.nt)
            shape  = (ni,nj,nt)
            
            scalars_spd = self.scalars
            
            # scalars_spd += ['umag',
            #                 'ddx_u', 'ddy_u', 'ddz_u',
            #                 'ddx_v', 'ddy_v', 'ddz_v',
            #                 'ddx_w', 'ddy_w', 'ddz_w',
            #                 'mag_grad_vel',
            #                 'ddx_rho', 'ddy_rho', 'ddz_rho',
            #                 'mag_grad_rho',
            #                 'ddx_p', 'ddy_p', 'ddz_p',
            #                 'mag_grad_p',
            #                 'ddx_T', 'ddy_T', 'ddz_T',
            #                 'mag_grad_T',
            #                 ]
            
            for scalar in scalars_spd:
                
                dsn = f'data/{scalar}'
                
                if (dsn in self):
                    dset = self[dsn]
                    dtype = dset.dtype
                    float_bytes = dtype.itemsize
                else:
                    dset = None
                    dtype = np.dtype(np.float32)
                    float_bytes = dtype.itemsize
                
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                data_gb = np.prod(shape) * float_bytes / 1024**3
                
                if verbose:
                    even_print(f'initializing {dsn}','%0.1f [GB]'%(data_gb,))
                if (dsn in hfspd):
                    del hfspd[dsn]
                dset = hfspd.create_dataset( dsn,
                                             shape=shape,
                                             dtype=dtype,
                                             chunks=chunks,
                                             )
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (i,j,t)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === main loop : copy over [u,v,w,ρ,T,p]
        
        data_gb_read  = 0.
        data_gb_write = 0.
        t_read  = 0.
        t_write = 0.
        
        with spd(fn_spd, 'a', driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
            
            if verbose:
                progress_bar = tqdm(total=ct*self.n_scalars, ncols=100, desc='export spd', leave=False, file=sys.stdout)
            
            for scalar in scalars_spd:
                
                dsn = f'data/{scalar}'
                
                if (dsn in self):
                    
                    dset_src = self[dsn]
                    dset_tgt = hfspd[dsn]
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    for ctl_ in ctl:
                        ct1, ct2 = ctl_
                        ntc = ct2 - ct1
                        
                        ## read
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_src.collective:
                            data = np.copy( dset_src[ct1:ct2,zi,ry1:ry2,rx1:rx2].T )
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * ni * nj * ntc / 1024**3
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        if verbose:
                            tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        ## write
                        hfspd.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_tgt.collective:
                            dset_tgt[ri1:ri2,rj1:rj2,ct1:ct2] = data
                        hfspd.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * ni * nj * ntc / 1024**3
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            tqdm.write(even_print(f'write: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        if verbose: progress_bar.update()
            
            if verbose: progress_bar.close()
        
        ## === [u,v,w,ρ,T,p] have been copied, now calculate extra variables 
        
        # === extend the rank ranges (spatial range overlap)
        
        ## backup non-overlapped bounds
        rx1_orig, rx2_orig = rx1, rx2
        ry1_orig, ry2_orig = ry1, ry2
        #rz1_orig, rz2_orig = rz1, rz2
        
        if self.usingmpi: 
            
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            #zA = 0
            #zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            #rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## overlap in [y]
            if (t4d[1]!=0):
                ry1, ry2 = ry1-n_overlap, ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                ry1, ry2 = ry1, ry2+n_overlap
            
            # ## overlap in [z]
            # if (t4d[2]!=0):
            #     rz1, rz2 = rz1-n_overlap, rz2
            #     zA += n_overlap
            #     zB += n_overlap
            # if (t4d[2]!=rz-1):
            #     rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            #nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        ## not finished!
        
        ## with spd(fn_spd, 'a', driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
        ##     
        ##     # if verbose:
        ##     #     progress_bar = tqdm(total=ct*self.n_scalars, ncols=100, desc='export spd', leave=False, file=sys.stdout)
        ##     
        ##     for scalar in scalars_spd:
        ##         
        ##         dsn = f'data/{scalar}'
        ##         
        ##         if (dsn in self):
        ##             
        ##             dset_src = self[dsn]
        ##             dset_tgt = hfspd[dsn]
        ##             
        ##             dtype = dset_src.dtype
        ##             float_bytes = dtype.itemsize
        ##             
        ##             for ctl_ in ctl:
        ##                 ct1, ct2 = ctl_
        ##                 ntc = ct2 - ct1
        ##                 
        ##                 ## read
        ##                 self.comm.Barrier()
        ##                 t_start = timeit.default_timer()
        ##                 with dset_src.collective:
        ##                     data = np.copy( dset_src[ct1:ct2,zi,ry1:ry2,rx1:rx2].T )
        ##                 self.comm.Barrier()
        ##                 t_delta = timeit.default_timer() - t_start
        ##                 data_gb = float_bytes * ni * nj * ntc / 1024**3
        ##                 
        ##                 t_read       += t_delta
        ##                 data_gb_read += data_gb
        ##                 
        ##                 if verbose:
        ##                     tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
        ##                 
        ##                 ## write
        ##                 hfspd.comm.Barrier()
        ##                 t_start = timeit.default_timer()
        ##                 with dset_tgt.collective:
        ##                     dset_tgt[ri1:ri2,rj1:rj2,ct1:ct2] = data[xA:xB,yA:yB,:]
        ##                 hfspd.comm.Barrier()
        ##                 t_delta = timeit.default_timer() - t_start
        ##                 data_gb = float_bytes * ni * nj * ntc / 1024**3
        ##                 
        ##                 t_write       += t_delta
        ##                 data_gb_write += data_gb
        ##                 
        ##                 if verbose:
        ##                     tqdm.write(even_print(f'write: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
        ##                 
        ##                 # if verbose: progress_bar.update()
        ##     
        ##     # if verbose: progress_bar.close()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.export_polydata_zpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # ===
    
    def check_grid_validity_z(self,):
        '''
        check grid validity for CGD file (in [z] direction)
        - monotonically increasing
        - no Δ=0
        '''
        
        verbose = True
        
        if verbose: print('\n'+'cgd.check_grid_validity_z()'+'\n'+72*'-')
        
        # x = np.copy(self.x)
        # y = np.copy(self.y)
        # z = np.copy(self.z)
        # t = np.copy(self.t)
        
        z = np.copy(self['dims/z']).T
        
        nx,ny,nz = z.shape
        
        z_ref = np.copy(z[0,0,:])
        
        ## check no zero distance elements
        if (np.diff(z_ref).size - np.count_nonzero(np.diff(z_ref))) != 0.:
            if verbose: even_print('check: Δz!=0','failed')
        else:
            if verbose: even_print('check: Δz!=0','passed')
        
        ## check monotonically increasing
        if not np.all(np.diff(z_ref) > 0.):
            if verbose: even_print('check: z mono increasing','failed')
        else:
            if verbose: even_print('check: z mono increasing','passed')
        
        if verbose: print(72*'-')
        
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from CGD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        makeVectors = kwargs.get('makeVectors',True) ## write vectors (e.g. velocity, vorticity) to XDMF
        makeTensors = kwargs.get('makeTensors',True) ## write 3x3 tensors (e.g. stress, strain) to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if verbose: print('\n'+'cgd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for scalar in ['x','y','z']:
            if ('dims/'+scalar in self):
                data = self['dims/'+scalar]
                
                txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
                if verbose: even_print(scalar, txt)
                
                dataset_precision_dict[scalar] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[scalar] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[scalar] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        # scalar names dict
        # --> labels for Paraview could be customized (e.g. units could be added) using a dict
        # --> the block below shows one such example dict, though it is currently inactive
        
        if False:
            units = 'dimless'
            if (units=='SI') or (units=='si'): ## m,s,kg,K
                scalar_names = {'x':'x [m]',
                                'y':'y [m]',
                                'z':'z [m]', 
                                'u':'u [m/s]',
                                'v':'v [m/s]',
                                'w':'w [m/s]', 
                                'T':'T [K]',
                                'rho':'rho [kg/m^3]',
                                'p':'p [Pa]'}
            elif (units=='dimless') or (units=='dimensionless'):
                scalar_names = {'x':'x [dimless]',
                                'y':'y [dimless]',
                                'z':'z [dimless]', 
                                'u':'u [dimless]',
                                'v':'v [dimless]',
                                'w':'w [dimless]',
                                'T':'T [dimless]',
                                'rho':'rho [dimless]',
                                'p':'p [dimless]'}
            else:
                raise ValueError('choice of units not recognized : %s --> options are : %s / %s'%(units,'SI','dimless'))
        else:
            scalar_names = {} ## dummy/empty 
        
        ## refresh header
        self.get_header(verbose=False, read_grid=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')
        
        # === write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                ## <Topology TopologyType="3DRectMesh" NumberOfElements="{self.nz:d} {self.ny:d} {self.nx:d}"/>
                ## <Geometry GeometryType="VxVyVz">
                
                xdmf_str=f'''
                         <Topology TopologyType="3DSMesh" NumberOfElements="{self.nz:d} {self.ny:d} {self.nx:d}"/>
                         <Geometry GeometryType="X_Y_Z">
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                             {fname_base}:/dims/{'x'}
                           </DataItem>
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                             {fname_base}:/dims/{'y'}
                           </DataItem>
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['z']}" Precision="{dataset_precision_dict['z']:d}" Format="HDF">
                             {fname_base}:/dims/{'z'}
                           </DataItem>
                         </Geometry>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # =====
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===== .xdmf : <Grid> per 3D coordinate array
                    
                    for scalar in ['x','y','z']:
                        
                        dset_hf_path = 'dims/%s'%scalar
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        if scalar in scalar_names:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        xdmf_str=f'''
                                 <!-- ===== scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                     {fname_base}:/{dset_hf_path}
                                   </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # ===== .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        
                        dset_hf_path = 'data/%s'%scalar
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        if scalar in scalar_names:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        xdmf_str=f'''
                                 <!-- ===== scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                     <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                       {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                       {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                       {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                     </DataItem>
                                     <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                       {fname_base}:/{dset_hf_path}
                                     </DataItem>
                                   </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    if makeVectors:
                        
                        # === .xdmf : <Grid> per vector : velocity vector
                        
                        if ('u' in self.scalars) and ('v' in self.scalars) and ('w' in self.scalars):
                            
                            scalar_name    = 'velocity'
                            dset_hf_path_i = 'data/u'
                            dset_hf_path_j = 'data/v'
                            dset_hf_path_k = 'data/w'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['u']}" Precision="{dataset_precision_dict['u']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['v']}" Precision="{dataset_precision_dict['v']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['w']}" Precision="{dataset_precision_dict['w']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                        
                        # === .xdmf : <Grid> per vector : velocity vector
                        
                        if ('uI' in self.scalars) and ('vI' in self.scalars) and ('wI' in self.scalars):
                            
                            scalar_name    = 'velocityI'
                            dset_hf_path_i = 'data/uI'
                            dset_hf_path_j = 'data/vI'
                            dset_hf_path_k = 'data/wI'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['uI']}" Precision="{dataset_precision_dict['uI']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vI']}" Precision="{dataset_precision_dict['vI']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['wI']}" Precision="{dataset_precision_dict['wI']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                        
                        # === .xdmf : <Grid> per vector : vorticity vector
                        
                        if ('vort_x' in self.scalars) and ('vort_y' in self.scalars) and ('vort_z' in self.scalars):
                            
                            scalar_name    = 'vorticity'
                            dset_hf_path_i = 'data/vort_x'
                            dset_hf_path_j = 'data/vort_y'
                            dset_hf_path_k = 'data/vort_z'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_x']}" Precision="{dataset_precision_dict['vort_x']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_y']}" Precision="{dataset_precision_dict['vort_y']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_z']}" Precision="{dataset_precision_dict['vort_z']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    if makeTensors:
                        if all([('dudx' in self.scalars),('dvdx' in self.scalars),('dwdx' in self.scalars),
                                ('dudy' in self.scalars),('dvdy' in self.scalars),('dwdy' in self.scalars),
                                ('dudz' in self.scalars),('dvdz' in self.scalars),('dwdz' in self.scalars)]):
                            pass
                            pass ## TODO
                            pass
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if verbose: print('--w-> %s'%fname_xdmf_base)
        return

class rgd(h5py.File):
    '''
    Rectilinear Grid Data (RGD)
    ---------------------------
    - super()'ed h5py.File class
    - 4D dataset storage
    - dimension coordinates are 4x 1D arrays defining [x,y,z,t] 
    
    to clear:
    ---------
    > os.system('h5clear -s tmp.h5')
    > hf = h5py.File('tmp.h5', 'r', libver='latest')
    > hf.close()
    
    Structure
    ---------
    
    rgd.h5
    │
    ├── header/
    │   └── udef_char
    │   └── udef_real
    │
    ├── dims/ --> 1D
    │   └── x
    │   └── y
    │   └── z
    │   └── t
    │
    └-─ data/<<scalar>> --> 4D [t,z,y,x]
    
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.rgd()')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## rgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 32    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8     )
        perms          = kwargs.pop('perms'          , '640' )
        
        if not isinstance(stripe_count, int):
            raise ValueError
        if not isinstance(stripe_size_mb, int):
            raise ValueError
        if not isinstance(perms, str):
            raise ValueError
        if not len(perms)==3:
            raise ValueError
        if not re.fullmatch(r'\d{3}',perms):
            raise ValueError
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using romio --> causes crash
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        
        ## set ROMIO hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(16*1024**2))) ) ## 16 [MB]
                ##
                #mpi_info.Set('romio_no_indep_rw' , 'true' ) ## deferred open + only collective I/O 
                #mpi_info.Set('cb_nodes' , str(int(round(1*self.n_ranks))) )
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        ## | rdcc_nbytes:
        ## | ------------
        ## | Integer setting the total size of the raw data chunk cache for this dataset in bytes.
        ## | In most cases increasing this number will improve performance, as long as you have 
        ## | enough free memory. The default size is 1 MB
        
        ## --> gets passed to H5Pset_chunk_cache
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        ## | rdcc_nslots:
        ## | ------------
        ## | Integer defining the number of chunk slots in the raw data chunk cache for this dataset.
        
        ## if ('rdcc_nslots' not in kwargs):
        ##     kwargs['rdcc_nslots'] = 521
        
        ## rgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with rgd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(0.1)
                Path(self.fname).touch() ## touch a new file
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        ## call actual h5py.File.__init__()
        super(rgd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(rgd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed RGD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(rgd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of RGD class instance
        --> this gets called automatically upon opening the file
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if (self.rank!=0):
            verbose=False
        
        # === attrs
        if ('duration_avg' in self.attrs.keys()):
            self.duration_avg = self.attrs['duration_avg']
        # if ('rectilinear' in self.attrs.keys()):
        #     self.rectilinear = self.attrs['rectilinear']
        # if ('curvilinear' in self.attrs.keys()):
        #     self.curvilinear = self.attrs['curvilinear']
        
        ## these should be set in the (init_from_() funcs)
        if ('fclass' in self.attrs.keys()):
            self.fclass = self.attrs['fclass'] ## 'rgd','cgd',...
        if ('fsubtype' in self.attrs.keys()):
            self.fsubtype = self.attrs['fsubtype'] ## 'unsteady','mean','prime',...
        
        # === udef
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## just make udef_real a dict with udef_char as keys
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.C_Suth      = self.udef['C_Suth']
            self.S_Suth      = self.udef['S_Suth']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            
            # === characteristic values : derived
            
            rho_inf = self.rho_inf = self.p_inf/(self.R * self.T_inf)
            mu_inf  = self.mu_inf  = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            nu_inf  = self.nu_inf  = self.mu_inf/self.rho_inf
            a_inf   = self.a_inf   = np.sqrt(self.kappa*self.R*self.T_inf)
            U_inf   = self.U_inf   = self.Ma*self.a_inf
            cp      = self.cp      = self.R*self.kappa/(self.kappa-1.)
            cv      = self.cv      = self.cp/self.kappa
            r       = self.r       = self.Pr**(1/3)
            Tw      = self.Tw      = self.T_inf
            Taw     = self.Taw     = self.T_inf + self.r*self.U_inf**2/(2*self.cp)
            lchar   = self.lchar   = self.Re*self.nu_inf/self.U_inf
            
            tchar   = self.tchar = self.lchar / self.U_inf
            uchar   = self.uchar = self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf' , '%0.3f [kg/m³]'    % self.rho_inf )
            if verbose: even_print('mu_inf'  , '%0.6E [kg/(m·s)]' % self.mu_inf  )
            if verbose: even_print('nu_inf'  , '%0.6E [m²/s]'     % self.nu_inf  )
            if verbose: even_print('a_inf'   , '%0.6f [m/s]'      % self.a_inf   )
            if verbose: even_print('U_inf'   , '%0.6f [m/s]'      % self.U_inf   )
            if verbose: even_print('cp'      , '%0.3f [J/(kg·K)]' % self.cp      )
            if verbose: even_print('cv'      , '%0.3f [J/(kg·K)]' % self.cv      )
            if verbose: even_print('r'       , '%0.6f [-]'        % self.r       )
            if verbose: even_print('Tw'      , '%0.3f [K]'        % self.Tw      )
            if verbose: even_print('Taw'     , '%0.3f [K]'        % self.Taw     )
            if verbose: even_print('lchar'   , '%0.6E [m]'        % self.lchar   )
            if verbose: even_print('tchar'   , '%0.6E [s]'        % self.tchar   )
            if verbose: print(72*'-')
            #if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the RGD instance
            udef_char_deriv = ['rho_inf', 'mu_inf', 'nu_inf', 'a_inf', 'U_inf', 'cp', 'cv', 'r', 'Tw', 'Taw', 'lchar']
            udef_real_deriv = [ rho_inf,   mu_inf,   nu_inf,   a_inf,   U_inf,   cp,   cv,   r,   Tw,   Taw,   lchar ]
            self.udef_deriv = dict(zip(udef_char_deriv, udef_real_deriv))
        
        else:
            pass
        
        # === coordinate vectors
        
        if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
            
            x   = self.x   = np.copy(self['dims/x'][:])
            y   = self.y   = np.copy(self['dims/y'][:])
            z   = self.z   = np.copy(self['dims/z'][:])
            nx  = self.nx  = x.size
            ny  = self.ny  = y.size
            nz  = self.nz  = z.size
            ngp = self.ngp = nx*ny*nz
            
            #if verbose: print(72*'-')
            if verbose: even_print('nx', '%i'%nx )
            if verbose: even_print('ny', '%i'%ny )
            if verbose: even_print('nz', '%i'%nz )
            if verbose: even_print('ngp', '%i'%ngp )
            if verbose: print(72*'-')
            
            if verbose: even_print('x_min', '%0.2f'%x.min())
            if verbose: even_print('x_max', '%0.2f'%x.max())
            if (self.nx>2):
                if verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (x[1]-x[0]), (x[-1]-x[-2]) ))
            if verbose: even_print('y_min', '%0.2f'%y.min())
            if verbose: even_print('y_max', '%0.2f'%y.max())
            if (self.ny>2):
                if verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (y[1]-y[0]), (y[-1]-y[-2]) ))
            if verbose: even_print('z_min', '%0.2f'%z.min())
            if verbose: even_print('z_max', '%0.2f'%z.max())
            if (self.nz>2):
                if verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (z[1]-z[0]), (z[-1]-z[-2]) ))
            #if verbose: print(72*'-'+'\n')
            if verbose: print(72*'-')
        
        else:
            pass
        
        # === 1D grid filters
        
        self.hasGridFilter=False
        if ('dims/xfi' in self):
            xfi = np.copy(self['dims/xfi'][:])
            self.xfi = xfi
            if not np.array_equal(xfi, np.array(range(nx), dtype=np.int64)):
                self.hasGridFilter=True
        if ('dims/yfi' in self):
            yfi = np.copy(self['dims/yfi'][:])
            self.yfi = yfi
            if not np.array_equal(yfi, np.array(range(ny), dtype=np.int64)):
                self.hasGridFilter=True
        if ('dims/zfi' in self):
            zfi = np.copy(self['dims/zfi'][:])
            self.zfi = zfi
            if not np.array_equal(xfi, np.array(range(nz), dtype=np.int64)):
                self.hasGridFilter=True
        
        # === time vector
        
        if ('dims/t' in self):
            self.t = np.copy(self['dims/t'][:])
            
            if ('data' in self): ## check t dim and data arr agree
                nt,_,_,_ = self['data/%s'%list(self['data'].keys())[0]].shape ## 4D
                if (nt!=self.t.size):
                    raise AssertionError('nt!=self.t.size : %i!=%i'%(nt,self.t.size))
            
            nt = self.t.size
            
            try:
                self.dt = self.t[1] - self.t[0]
            except IndexError:
                self.dt = 0.
            
            self.nt       = nt       = self.t.size
            self.duration = duration = self.t[-1] - self.t[0]
            self.ti       = ti       = np.array(range(self.nt), dtype=np.int64)
        
        elif all([('data' in self),('dims/t' not in self)]): ## data but no time
            self.scalars = list(self['data'].keys())
            nt,_,_,_ = self['data/%s'%self.scalars[0]].shape
            self.nt  = nt
            self.t   =      np.array(range(self.nt), dtype=np.float64)
            self.ti  = ti = np.array(range(self.nt), dtype=np.int64)
            self.dt  = 1.
            self.duration = duration = self.t[-1]-self.t[0]
        
        else:
            self.t  = np.array([], dtype=np.float64)
            self.ti = np.array([], dtype=np.int64)
            self.nt = nt = 0
            self.dt = 0.
            self.duration = duration = 0.
        
        #if verbose: print(72*'-')
        if verbose: even_print('nt', '%i'%self.nt )
        if verbose: even_print('dt', '%0.6f'%self.dt)
        if verbose: even_print('duration', '%0.2f'%self.duration )
        if hasattr(self, 'duration_avg'):
            if verbose: even_print('duration_avg', '%0.2f'%self.duration_avg )
        #if verbose: print(72*'-'+'\n')
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys()) ## 4D : string names of scalars : ['u','v','w'],...
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self[f'data/{scalar}'].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    # === I/O
    
    def init_from_eas4(self, fn_eas4, **kwargs):
        '''
        initialize an RGD from an EAS4 (NS3D output format)
        -----
        - x_min/max xi_min/max : min/max coord/index
        - stride filters (sx,sy,sz)
        '''
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        verbose = kwargs.get('verbose',True)
        if (self.rank!=0):
            verbose=False
        
        # === spatial resolution filter : take every nth grid point
        sx = kwargs.get('sx',1)
        sy = kwargs.get('sy',1)
        sz = kwargs.get('sz',1)
        #st = kwargs.get('st',1)
        
        # === spatial resolution filter : set x/y/z bounds
        x_min = kwargs.get('x_min',None)
        y_min = kwargs.get('y_min',None)
        z_min = kwargs.get('z_min',None)
        
        x_max = kwargs.get('x_max',None)
        y_max = kwargs.get('y_max',None)
        z_max = kwargs.get('z_max',None)
        
        xi_min = kwargs.get('xi_min',None)
        yi_min = kwargs.get('yi_min',None)
        zi_min = kwargs.get('zi_min',None)
        
        xi_max = kwargs.get('xi_max',None)
        yi_max = kwargs.get('yi_max',None)
        zi_max = kwargs.get('zi_max',None)
        
        ## set default attributes
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'rgd'
        
        if verbose: print('\n'+'rgd.init_from_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        # even_print('infile', os.path.basename(fn_eas4))
        # even_print('infile size', '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3))
        # even_print('outfile', self.fname)
        
        if verbose: even_print('infile', os.path.basename(fn_eas4))
        if verbose: even_print('infile size', '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3))
        if verbose: even_print('outfile', self.fname)
        
        with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=self.comm) as hf_eas4:
            
            if verbose: even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1_orig, gmode_dict[hf_eas4.gmode_dim1_orig] ) )
            if verbose: even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2_orig, gmode_dict[hf_eas4.gmode_dim2_orig] ) )
            if verbose: even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3_orig, gmode_dict[hf_eas4.gmode_dim3_orig] ) )
            
            # === check gmode (RGD should not have more than ALL_G)
            if (hf_eas4.gmode_dim1_orig > 4):
                raise ValueError('turbx.rgd cannot handle gmode > 4 (EAS4 gmode_dim1=%i)'%hf_eas4.gmode_dim1_orig)
            if (hf_eas4.gmode_dim2_orig > 4):
                raise ValueError('turbx.rgd cannot handle gmode > 4 (EAS4 gmode_dim2=%i)'%hf_eas4.gmode_dim2_orig)
            if (hf_eas4.gmode_dim3_orig > 4):
                raise ValueError('turbx.rgd cannot handle gmode > 4 (EAS4 gmode_dim3=%i)'%hf_eas4.gmode_dim3_orig)
            
            # === copy over header info if needed
            if all([('header/udef_real' in self),('header/udef_char' in self)]):
                raise ValueError('udef already present')
            else:
                udef         = hf_eas4.udef
                udef_real    = list(udef.values())
                udef_char    = list(udef.keys())
                udef_real_h5 = np.array(udef_real, dtype=np.float64)
                udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
                
                self.create_dataset('header/udef_real', data=udef_real_h5, dtype=np.float64)
                self.create_dataset('header/udef_char', data=udef_char_h5, dtype='S128')
                self.udef      = udef
                self.udef_real = udef_real
                self.udef_char = udef_char
             
            # === copy over dims info
            if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
                pass
                ## future: 2D/3D handling here
            else:
                x, y, z = hf_eas4.x, hf_eas4.y, hf_eas4.z
                nx  = x.size
                ny  = y.size
                nz  = z.size
                ngp = nx*ny*nz
                # nt = hf_eas4.nt --> no time data available yet
                
                if any([(xi_min is not None),\
                        (xi_max is not None),\
                        (yi_min is not None),\
                        (yi_max is not None),\
                        (zi_min is not None),\
                        (zi_max is not None),\
                        (x_min is not None),\
                        (x_max is not None),\
                        (y_min is not None),\
                        (y_max is not None),\
                        (z_min is not None),\
                        (z_max is not None),\
                        (sx!=1),\
                        (sy!=1),\
                        (sz!=1)]):
                            hasFilters=True
                            if verbose: print('filtered dim info\n'+72*'-')
                else:
                    hasFilters=False
                
                # === index arrays along each axis
                xfi = np.array(range(nx),  dtype=np.int64)
                yfi = np.array(range(ny),  dtype=np.int64)
                zfi = np.array(range(nz),  dtype=np.int64)
                #tfi = np.array(range(nt), dtype=np.int64) --> no time data available yet
                # === total bounds clip (physical nondimensional distance)
                if (x_min is not None):
                    xfi = np.array([i for i in xfi if (x[i] >= x_min)])
                    if verbose: even_print('x_min', '%0.3f'%x_min)
                if (x_max is not None):
                    xfi = np.array([i for i in xfi if (x[i] <= x_max)])
                    if verbose: even_print('x_max', '%0.3f'%x_max)
                if (y_min is not None):
                    yfi = np.array([i for i in yfi if (y[i] >= y_min)])
                    if verbose: even_print('y_min', '%0.3f'%y_min)
                if (y_max is not None):
                    yfi = np.array([i for i in yfi if (y[i] <= y_max)])
                    if verbose: even_print('y_max', '%0.3f'%y_max)
                if (z_min is not None):
                    zfi = np.array([i for i in zfi if (z[i] >= z_min)])
                    if verbose: even_print('z_min', '%0.3f'%z_min)
                if (z_max is not None):
                    zfi = np.array([i for i in zfi if (z[i] <= z_max)])
                    if verbose: even_print('z_max', '%0.3f'%z_max)
                
                # === total bounds clip (coordinate index)
                if (xi_min is not None):
                    
                    xfi_ = []
                    if verbose: even_print('xi_min', '%i'%xi_min)
                    for c in xfi:
                        if (xi_min<0) and (c>=(nx+xi_min)): ## support negative indexing
                            xfi_.append(c)
                        elif (xi_min>=0) and (c>=xi_min):
                            xfi_.append(c)
                    xfi=np.array(xfi_, dtype=np.int64)
                
                if (xi_max is not None):
                    
                    xfi_ = []
                    if verbose: even_print('xi_max', '%i'%xi_max)
                    for c in xfi:
                        if (xi_max<0) and (c<=(nx+xi_max)): ## support negative indexing
                            xfi_.append(c)
                        elif (xi_max>=0) and (c<=xi_max):
                            xfi_.append(c)
                    xfi=np.array(xfi_, dtype=np.int64)
                
                if (yi_min is not None):
                    
                    yfi_ = []
                    if verbose: even_print('yi_min', '%i'%yi_min)
                    for c in yfi:
                        if (yi_min<0) and (c>=(ny+yi_min)): ## support negative indexing
                            yfi_.append(c)
                        elif (yi_min>=0) and (c>=yi_min):
                            yfi_.append(c)
                    yfi=np.array(yfi_, dtype=np.int64)
                
                if (yi_max is not None):
                    
                    yfi_ = []
                    if verbose: even_print('yi_max', '%i'%yi_max)
                    for c in yfi:
                        if (yi_max<0) and (c<=(ny+yi_max)): ## support negative indexing
                            yfi_.append(c)
                        elif (yi_max>=0) and (c<=yi_max):
                            yfi_.append(c)
                    yfi=np.array(yfi_, dtype=np.int64)
                
                if (zi_min is not None):
                    
                    zfi_ = []
                    if verbose: even_print('zi_min', '%i'%zi_min)
                    for c in zfi:
                        if (zi_min<0) and (c>=(nz+zi_min)): ## support negative indexing
                            zfi_.append(c)
                        elif (zi_min>=0) and (c>=zi_min):
                            zfi_.append(c)
                    zfi=np.array(zfi_, dtype=np.int64)
                
                if (zi_max is not None):
                    
                    zfi_ = []
                    if verbose: even_print('zi_max', '%i'%zi_max)
                    for c in zfi:
                        if (zi_max<0) and (c<=(nz+zi_max)): ## support negative indexing
                            zfi_.append(c)
                        elif (zi_max>=0) and (c<=zi_max):
                            zfi_.append(c)
                    zfi=np.array(zfi_, dtype=np.int64)
                
                # === resolution filter (skip every n grid points in each direction)
                if (sx!=1):
                    if verbose: even_print('sx', '%i'%sx)
                    xfi = xfi[::sx]
                if (sy!=1):
                    if verbose: even_print('sy', '%i'%sy)
                    yfi = yfi[::sy]
                if (sz!=1):
                    if verbose: even_print('sz', '%i'%sz)
                    zfi = zfi[::sz]
                # if (st!=1):
                #     even_print('st', '%i'%st)
                #     tfi = tfi[::st]
                # ===
                self.xfi = xfi
                self.yfi = yfi
                self.zfi = zfi
                #self.tfi = tfi
                # ===
                self.create_dataset('dims/xfi', data=xfi)
                self.create_dataset('dims/yfi', data=yfi)
                self.create_dataset('dims/zfi', data=zfi)
                #self.create_dataset('dims/tfi', data=tfi, maxshape=np.shape(tfi))
                
                if (xfi.size==0):
                    raise ValueError('x grid filter is empty... check!')
                if (yfi.size==0):
                    raise ValueError('y grid filter is empty... check!')
                if (zfi.size==0):
                    raise ValueError('z grid filter is empty... check!')
                #if (tfi.size==0):
                #    raise ValueError('t grid filter is empty... check!')
                
                # === determine if gridFilter is active
                self.hasGridFilter=False
                if not np.array_equal(xfi, np.array(range(nx), dtype=np.int64)):
                    self.hasGridFilter=True
                if not np.array_equal(yfi, np.array(range(ny), dtype=np.int64)):
                    self.hasGridFilter=True
                if not np.array_equal(zfi, np.array(range(nz), dtype=np.int64)):
                    self.hasGridFilter=True
                #if not np.array_equal(tfi, np.array(range(nt), dtype=np.int64)):
                #    self.hasGridFilter=True
                # === overwrite & write 1D filter arrays
                x = np.copy(x[xfi])
                y = np.copy(y[yfi])
                z = np.copy(z[zfi])
                #t = np.copy(t[tfi])
                
                ### if self.hasGridFilter:
                ###     print('>>> grid filter is present')
                ### else:
                ###     print('>>> no grid filter present')
                
                # === (over)write coord vecs if filter present
                if self.hasGridFilter:
                    
                    nx = x.size
                    ny = y.size
                    nz = z.size
                    ngp = nx*ny*nz
                    #nt = t.size
                    
                    if verbose: even_print('nx',  '%i'%nx  )
                    if verbose: even_print('ny',  '%i'%ny  )
                    if verbose: even_print('nz',  '%i'%nz  )
                    if verbose: even_print('ngp', '%i'%ngp )
                    #if verbose: even_print('nt', '%i'%nt )
                    
                    self.nx  = nx
                    self.ny  = ny
                    self.nz  = nz
                    self.ngp = ngp
                    #self.nt = nt
                
                # === write 1D coord arrays
                if ('dims/x' in self):
                    del self['dims/x']
                self.create_dataset('dims/x', data=x)
                if ('dims/y' in self):
                    del self['dims/y']
                self.create_dataset('dims/y', data=y)
                if ('dims/z' in self):
                    del self['dims/z']
                self.create_dataset('dims/z', data=z)
                # if ('dims/t' in self):
                #     del self['dims/t']
                # self.create_dataset('dims/t', data=t, maxshape=t.shape)
                
                # === write 3D coord arrays
                if False:
                    size3DCoordArrays = 3*ngp*8/1024**2 ## [MB]
                    #print('>>> size of 3D coord arrays : %0.2f [GB]'%size3DCoordArrays)
                    if (size3DCoordArrays < 100): ## if less than 100 [MB]
                        xxx, yyy, zzz = np.meshgrid(x, y, z, indexing='ij')
                        self.create_dataset('dims/xxx', data=xxx.T)
                        self.create_dataset('dims/yyy', data=yyy.T)
                        self.create_dataset('dims/zzz', data=zzz.T)
        
        if verbose: print(72*'-')
        self.get_header(verbose=True)
        if verbose: print(72*'-')
        
        return
    
    def init_from_rgd(self, fn_rgd, **kwargs):
        '''
        initialize an RGD from an RGD (copy over header data & coordinate data)
        '''
        
        t_info = kwargs.get('t_info',True)
        #chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        
        verbose = kwargs.get('verbose',True)
        if (self.rank!=0):
            verbose=False
        
        ## set default attributes: fsubtype, fclass
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'rgd'
        
        with rgd(fn_rgd, 'r', driver=self.driver, comm=self.comm) as hf_ref:
            
            ## copy over fsubtype
            if hasattr(hf_ref,'fsubtype'):
                self.attrs['fsubtype'] = hf_ref.fsubtype
            
            # === copy over header info if needed
            
            if all([('header/udef_real' in self),('header/udef_char' in self)]):
                raise ValueError('udef already present')
            else:
                udef         = hf_ref.udef
                udef_real    = list(udef.values())
                udef_char    = list(udef.keys())
                udef_real_h5 = np.array(udef_real, dtype=np.float64)
                udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
                
                self.create_dataset('header/udef_real', data=udef_real_h5, maxshape=np.shape(udef_real_h5), dtype=np.float64)
                self.create_dataset('header/udef_char', data=udef_char_h5, maxshape=np.shape(udef_char_h5), dtype='S128')
                self.udef      = udef
                self.udef_real = udef_real
                self.udef_char = udef_char
            
            # === copy over spatial dim info
            
            x, y, z = hf_ref.x, hf_ref.y, hf_ref.z
            nx  = self.nx  = x.size
            ny  = self.ny  = y.size
            nz  = self.nz  = z.size
            ngp = self.ngp = nx*ny*nz
            if ('dims/x' in self):
                del self['dims/x']
            if ('dims/y' in self):
                del self['dims/y']
            if ('dims/z' in self):
                del self['dims/z']
            
            self.create_dataset('dims/x', data=x)
            self.create_dataset('dims/y', data=y)
            self.create_dataset('dims/z', data=z)
            
            # === copy over temporal dim info
            
            if t_info:
                self.t  = hf_ref.t
                self.nt = self.t.size
                self.create_dataset('dims/t', data=hf_ref.t)
            else:
                t = np.array([0.], dtype=np.float64)
                if ('dims/t' in self):
                    del self['dims/t']
                self.create_dataset('dims/t', data=t)
            
            # ===
            
            ## copy over [data_dim/<>] dsets if present
            if ('data_dim' in hf_ref):
                for dsn in hf_ref['data_dim'].keys():
                    data = np.copy( hf_ref[f'data_dim/{dsn}'][()] ) 
                    self.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                    if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False)
        return
    
    def import_eas4(self, fn_eas4_list, **kwargs):
        '''
        import data from a series of EAS4 files to a RGD
        '''
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        if verbose: print('\n'+'rgd.import_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        tt_min = kwargs.get('tt_min',None)
        tt_max = kwargs.get('tt_max',None)
        
        ## dont actually copy over data, just initialize datasets with 0's
        init_dsets_only = kwargs.get('init_dsets_only',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        ## float precision when copying
        ## default is 'single' i.e. cast data to single
        ## 'same' will preserve the floating point precision from the EAS4 file
        prec = kwargs.get('prec',None)
        if (prec is None):
            prec = 'single'
        elif (prec=='single'):
            pass
        elif (prec=='same'):
            pass
        else:
            raise ValueError('prec not set correctly')
        
        ## HDF5 chunk parameters (t,z,y,x)
        if (chunk_constraint is None):
            chunk_constraint = (1,None,None,None) ## single [t] convention
            #chunk_constraint = (None,-1,1,-1) ## single [y] convention
        if (chunk_base is None):
            chunk_base = 2
        
        ## check for an often made mistake
        ts_min = kwargs.get('ts_min',None)
        ts_max = kwargs.get('ts_max',None)
        if (ts_min is not None):
            raise AssertionError('ts_min is not an option --> did you mean ti_min or tt_min?')
        if (ts_max is not None):
            raise AssertionError('ts_max is not an option --> did you mean ti_max or tt_max?')
        
        ## check that the passed iterable of EAS4 files is OK
        if not hasattr(fn_eas4_list, '__iter__'):
            raise AssertionError('first arg \'fn_eas4_list\' must be iterable')
        for fn_eas4 in fn_eas4_list:
            if not os.path.isfile(fn_eas4):
                raise FileNotFoundError('%s not found!'%fn_eas4)
        
        ## ranks
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        ## skip dimensions --> spatial skips done in init_from_XXX()
        # sx = kwargs.get('sx',1)
        # sy = kwargs.get('sy',1)
        # sz = kwargs.get('sz',1)
        st = kwargs.get('st',1)
        
        ## update this RGD's header and attributes
        self.get_header(verbose=False)
        
        # === get all time info & check
        
        comm_eas4 = MPI.COMM_WORLD
        t = np.array([], dtype=np.float64)
        for fn_eas4 in fn_eas4_list:
            with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                t = np.concatenate((t, hf_eas4.t))
        comm_eas4.Barrier()
        
        if verbose: even_print('n EAS4 files','%i'%len(fn_eas4_list))
        if verbose: even_print('nt all files','%i'%t.size)
        
        if (t.size>1):
            
            ## check no zero distance elements
            if (np.diff(t).size - np.count_nonzero(np.diff(t))) != 0.:
                raise AssertionError('t arr has zero-distance elements')
            else:
                if verbose: even_print('check: Δt!=0','passed')
            
            ## check monotonically increasing
            if not np.all(np.diff(t) > 0.):
                raise AssertionError('t arr not monotonically increasing')
            else:
                if verbose: even_print('check: t mono increasing','passed')
            
            ## check constant Δt
            dt0 = np.diff(t)[0]
            if not np.all(np.isclose(np.diff(t), dt0, rtol=1e-3)):
                if (self.rank==0): print(np.diff(t))
                raise AssertionError('t arr not uniformly spaced')
            else:
                if verbose: even_print('check: constant Δt','passed')
        
        # === get all grid info & check
        
        if ( len(fn_eas4_list) > 1 ):
            
            comm_eas4 = MPI.COMM_WORLD
            eas4_x_arr = []
            eas4_y_arr = []
            eas4_z_arr = []
            for fn_eas4 in fn_eas4_list:
                with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                    eas4_x_arr.append( hf_eas4.x )
                    eas4_y_arr.append( hf_eas4.y )
                    eas4_z_arr.append( hf_eas4.z )
            comm_eas4.Barrier()
            
            ## check coordinate vectors are same
            if not np.all([np.allclose(eas4_z_arr[i],eas4_z_arr[0],rtol=1e-8) for i in range(len(fn_eas4_list))]):
                raise AssertionError('EAS4 files do not have the same z coordinates')
            else:
                if verbose: even_print('check: z coordinate vectors equal','passed')
            if not np.all([np.allclose(eas4_y_arr[i],eas4_y_arr[0],rtol=1e-8) for i in range(len(fn_eas4_list))]):
                raise AssertionError('EAS4 files do not have the same y coordinates')
            else:
                if verbose: even_print('check: y coordinate vectors equal','passed')
            if not np.all([np.allclose(eas4_x_arr[i],eas4_x_arr[0],rtol=1e-8) for i in range(len(fn_eas4_list))]):
                raise AssertionError('EAS4 files do not have the same x coordinates')
            else:
                if verbose: even_print('check: x coordinate vectors equal','passed')
        
        # === [t] resolution filter (skip every n timesteps)
        tfi = self.tfi = np.arange(t.size, dtype=np.int64)
        if (st!=1):
            if verbose: even_print('st', '%i'%st)
            #print('>>> st : %i'%st)
            tfi = self.tfi = tfi[::st]
        
        # === get doRead vector
        doRead = np.full((t.size,), True, dtype=bool)
        
        ## skip filter
        if hasattr(self, 'tfi'):
            doRead[np.isin(np.arange(t.size),self.tfi,invert=True)] = False
        
        ## min/max index filter
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
            doRead[:ti_min] = False
        if (ti_max is not None):
            if not isinstance(ti_max, int):
                raise TypeError('ti_max must be type int')
            doRead[ti_max:] = False
        
        if (tt_min is not None):
            if (tt_min>=0.):
                doRead[np.where((t-t.min())<tt_min)] = False
            elif (tt_min<0.):
                doRead[np.where((t-t.max())<tt_min)] = False
        
        if (tt_max is not None):
            if (tt_max>=0.):
                doRead[np.where((t-t.min())>tt_max)] = False
            elif (tt_max<0.):
                doRead[np.where((t-t.max())>tt_max)] = False
        
        # === RGD times
        self.t  = np.copy(t[doRead])
        self.nt = self.t.size
        self.ti = np.arange(self.nt, dtype=np.int64)
        
        # === write back time vector 'self.t' to file as 'dims/t'
        
        ## before the write, if appending, check that time vector is the same
        if (self.open_mode=='a') or (self.open_mode=='r+'):
            if ('dims/t' in self):
                t_ = np.copy(self['dims/t'][()])
                np.testing.assert_allclose(t_, self.t, rtol=1e-14, atol=1e-14)
        
        if ('dims/t' in self):
            del self['dims/t']
        self.create_dataset('dims/t', data=self.t)
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === determine RGD scalars (from EAS4 scalars)
        if not hasattr(self, 'scalars') or (len(self.scalars)==0):
            with eas4(fn_eas4_list[0], 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                self.scalars   = hf_eas4.scalars
                self.n_scalars = len(self.scalars)
                
                ## decide dtypes
                for scalar in hf_eas4.scalars:
                    
                    domainName = hf_eas4.domainName
                    ti = 0
                    dset_path = 'Data/%s/ts_%06d/par_%06d'%(domainName,ti,hf_eas4.scalar_n_map[scalar])
                    dset = hf_eas4[dset_path]
                    dtype = dset.dtype
                    
                    if (prec=='same'):
                        self.scalars_dtypes_dict[scalar] = dtype
                    elif (prec=='single'):
                        if (dtype!=np.float32) and (dtype!=np.float64): ## make sure its either a single or double float
                            raise ValueError
                        self.scalars_dtypes_dict[scalar] = np.dtype(np.float32)
                    else:
                        raise ValueError
        
        if self.usingmpi: comm_eas4.Barrier()
        
        # === initialize datasets
        
        for scalar in self.scalars: 
            
            dtype       = self.scalars_dtypes_dict[scalar]
            float_bytes = dtype.itemsize
            data_gb     = float_bytes*self.nt*self.nz*self.ny*self.nx / 1024**3
            shape       = (self.nt,self.nz,self.ny,self.nx)
            chunks      = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
            
            do_dset_initialize = True
            
            dsn = f'data/{scalar}'
            
            ## if exists, check size & chunks & dtype
            if (self.open_mode=='a') or (self.open_mode=='r+'):
                if (dsn in self):
                    
                    dset = self[dsn]
                    if verbose: even_print(f'dset {dsn} already exists', str(True))
                    
                    if ( dset.shape == shape ):
                        if verbose: even_print(f'dset {dsn} shape matches', str(True))
                    else:
                        if verbose: even_print(f'dset {dsn} shape matches', str(False))
                    
                    if ( dset.chunks == chunks ):
                        if verbose: even_print(f'dset {dsn} chunks match', str(True))
                    else:
                        if verbose: even_print(f'dset {dsn} chunks match', str(False))
                    
                    if ( dset.dtype == dtype ):
                        if verbose: even_print(f'dset {dsn} dtype matches', str(True))
                    else:
                        if verbose: even_print(f'dset {dsn} dtype matches', str(False))
                    
                    if ( dset.shape == shape ) and ( dset.chunks == chunks ) and ( dset.dtype == dtype ):
                        do_dset_initialize = False
            
            if do_dset_initialize:
                
                self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if verbose:
                    even_print(f'initializing data/{scalar}', f'{data_gb:0.2f} [GB]')
                
                dset = self.create_dataset(dsn, 
                                           shape=shape, 
                                           dtype=dtype,
                                           chunks=chunks,
                                           )
                
                self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]')
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)', str(dset.chunks))
                even_print('chunk size', f'{int(round(chunk_kb_)):d} [KB]')
        
        if verbose: print(72*'-')
        
        # === report size of RGD after initialization
        if verbose: tqdm.write(even_print(os.path.basename(self.fname), '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3), s=True))
        if verbose: print(72*'-')
        
        # === open EAS4s, read, write to RGD
        
        if not init_dsets_only:
            
            if verbose:
                progress_bar = tqdm(total=(self.nt*self.n_scalars), ncols=100, desc='import', leave=False, file=sys.stdout)
            
            data_gb_read  = 0.
            data_gb_write = 0.
            t_read  = 0.
            t_write = 0.
            
            tii  = -1 ## counter full series
            tiii = -1 ## counter RGD-local
            for fn_eas4 in fn_eas4_list:
                with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                    
                    if verbose: tqdm.write(even_print(os.path.basename(fn_eas4), '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3), s=True))
                    
                    # if verbose: tqdm.write(even_print('gmode_dim1' , '%i'%hf_eas4.gmode_dim1  , s=True))
                    # if verbose: tqdm.write(even_print('gmode_dim2' , '%i'%hf_eas4.gmode_dim2  , s=True))
                    # if verbose: tqdm.write(even_print('gmode_dim3' , '%i'%hf_eas4.gmode_dim3  , s=True))
                    
                    if verbose: tqdm.write(even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1_orig, gmode_dict[hf_eas4.gmode_dim1_orig] ), s=True ))
                    if verbose: tqdm.write(even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2_orig, gmode_dict[hf_eas4.gmode_dim2_orig] ), s=True ))
                    if verbose: tqdm.write(even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3_orig, gmode_dict[hf_eas4.gmode_dim3_orig] ), s=True ))
                    
                    if verbose: tqdm.write(even_print('duration' , '%0.2f'%hf_eas4.duration , s=True))
                    
                    # === write buffer
                    
                    # ## 5D [scalar][x,y,z,t] structured array
                    # buff = np.zeros(shape=(nxr, nyr, nzr, bt), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
                    
                    # ===
                    
                    #domainName = 'DOMAIN_000000' ## only one domain supported
                    domainName = hf_eas4.domainName
                    
                    for ti in range(hf_eas4.nt):
                        tii += 1 ## EAS4 series counter
                        if doRead[tii]:
                            tiii += 1 ## RGD counter
                            for scalar in hf_eas4.scalars:
                                if (scalar in self.scalars):
                                    
                                    # === collective read
                                    
                                    dset_path = 'Data/%s/ts_%06d/par_%06d'%(domainName,ti,hf_eas4.scalar_n_map[scalar])
                                    dset = hf_eas4[dset_path]
                                    
                                    if hf_eas4.usingmpi: comm_eas4.Barrier()
                                    t_start = timeit.default_timer()
                                    if hf_eas4.usingmpi: 
                                        with dset.collective:
                                            data = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                                    else:
                                        data = np.copy( dset[()] )
                                    if hf_eas4.usingmpi: comm_eas4.Barrier()
                                    t_delta = timeit.default_timer() - t_start
                                    
                                    data_gb       = data.nbytes / 1024**3
                                    t_read       += t_delta
                                    data_gb_read += data_gb
                                    
                                    if False:
                                        if verbose:
                                            txt = even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                            tqdm.write(txt)
                                    
                                    ## reduce precision (e.g. for restart which is usually double precision)
                                    # if (data.dtype == np.float64):
                                    #     data = np.copy( data.astype(np.float32) )
                                    
                                    ## reduce precision (if kwarg prec='single' and the incoming data is double)
                                    if (data.dtype != self.scalars_dtypes_dict[scalar]):
                                        data = np.copy( data.astype(self.scalars_dtypes_dict[scalar]) )
                                    
                                    data_gb = data.nbytes / 1024**3
                                    
                                    # === collective write
                                    
                                    dset = self['data/%s'%scalar]
                                    
                                    if self.usingmpi: self.comm.Barrier()
                                    t_start = timeit.default_timer()
                                    if self.usingmpi:
                                        with dset.collective:
                                            dset[tiii,rz1:rz2,ry1:ry2,rx1:rx2] = data.T
                                    else:
                                        
                                        if self.hasGridFilter:
                                            data = data[self.xfi[:,np.newaxis,np.newaxis],
                                                        self.yfi[np.newaxis,:,np.newaxis],
                                                        self.zfi[np.newaxis,np.newaxis,:]]
                                        
                                        dset[tiii,:,:,:] = data.T
                                    
                                    if self.usingmpi: self.comm.Barrier()
                                    t_delta = timeit.default_timer() - t_start
                                    
                                    t_write       += t_delta
                                    data_gb_write += data_gb
                                    
                                    if False:
                                        if verbose:
                                            txt = even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                            tqdm.write(txt)
                                    
                                    if verbose: progress_bar.update()
            
            if verbose: progress_bar.close()
            
            if hf_eas4.usingmpi: comm_eas4.Barrier()
            if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False)
        
        ## get read read/write totals all ranks
        if not init_dsets_only:
            if self.usingmpi:
                G = self.comm.gather([data_gb_read, data_gb_write, self.rank], root=0)
                G = self.comm.bcast(G, root=0)
                data_gb_read  = sum([x[0] for x in G])
                data_gb_write = sum([x[1] for x in G])
        
        if init_dsets_only:
            if verbose: print('>>> init_dsets_only=True, so no EAS4 data was imported')
        
        if verbose: print(72*'-')
        if verbose: even_print('nt',       '%i'%self.nt )
        if verbose: even_print('dt',       '%0.8f'%self.dt )
        if verbose: even_print('duration', '%0.2f'%self.duration )
        
        if not init_dsets_only:
            if verbose: print(72*'-')
            if verbose: even_print('time read',format_time_string(t_read))
            if verbose: even_print('time write',format_time_string(t_write))
            if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
            if verbose: even_print('read total avg', f'{data_gb_read:0.2f} [GB]  {t_read:0.2f} [s]  {(data_gb_read/t_read):0.3f} [GB/s]')
            if verbose: even_print('write total avg', f'{data_gb_write:0.2f} [GB]  {t_write:0.2f} [s]  {(data_gb_write/t_write):0.3f} [GB/s]')
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.import_eas4() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    @staticmethod
    def copy(fn_rgd_src, fn_rgd_tgt, **kwargs):
        '''
        copy header info, selected scalars, and [x,y,z,t] range to new RGD file
        --> this currently does NOT work in serial mode
        '''
        
        #comm    = MPI.COMM_WORLD
        rank    = MPI.COMM_WORLD.Get_rank()
        n_ranks = MPI.COMM_WORLD.Get_size()
        
        if (rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.copy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        rt       = kwargs.get('rt',1)
        force    = kwargs.get('force',False) ## overwrite or raise error if exists
        ti_min   = kwargs.get('ti_min',None)
        ti_max   = kwargs.get('ti_max',None)
        scalars  = kwargs.get('scalars',None)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        xi_min = kwargs.get('xi_min',None) ## 4D coordinate 
        xi_max = kwargs.get('xi_max',None)
        yi_min = kwargs.get('yi_min',None)
        yi_max = kwargs.get('yi_max',None)
        zi_min = kwargs.get('zi_min',None)
        zi_max = kwargs.get('zi_max',None)
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        
        ct = kwargs.get('ct',1) ## 'chunks' in time
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz!=n_ranks):
            raise AssertionError('rx*ry*rz!=n_ranks')
        if not os.path.isfile(fn_rgd_src):
            raise FileNotFoundError('%s not found!'%fn_rgd_src)
        if os.path.isfile(fn_rgd_tgt) and not force:
            raise FileExistsError('%s already exists. delete it or use \'force=True\' kwarg'%fn_rgd_tgt)
        
        ## HDF5 chunk parameters
        if (chunk_constraint is None):
            chunk_constraint = (1,None,None,None) ## single [t] convention
            #chunk_constraint = (None,-1,1,-1) ## single [y] convention
        if (chunk_base is None):
            chunk_base = 2
        
        # ===
        
        with rgd(fn_rgd_src, 'r', comm=MPI.COMM_WORLD, driver='mpio', libver='latest') as hf_src:
            with rgd(fn_rgd_tgt, 'w', comm=MPI.COMM_WORLD, driver='mpio', libver='latest', force=force) as hf_tgt:
                
                ## copy over header info (source --> target)
                hf_tgt.init_from_rgd(fn_rgd_src)
                
                if (scalars is None):
                    scalars = hf_src.scalars
                
                if verbose:
                    even_print('fn_rgd_src' , fn_rgd_src )
                    even_print('nx' , '%i'%hf_src.nx )
                    even_print('ny' , '%i'%hf_src.ny )
                    even_print('nz' , '%i'%hf_src.nz )
                    even_print('nt' , '%i'%hf_src.nt )
                    if verbose: print(72*'-')
                
                if (rx>hf_src.nx):
                    raise AssertionError('rx>nx')
                if (ry>hf_src.ny):
                    raise AssertionError('ry>ny')
                if (rz>hf_src.nz):
                    raise AssertionError('rz>nz')
                if (rt>hf_src.nt):
                    raise AssertionError('rt>nt')
                
                x  = np.copy( hf_src.x )
                y  = np.copy( hf_src.y )
                z  = np.copy( hf_src.z )
                t  = np.copy( hf_src.t )
                
                xi  = np.arange(x.shape[0],dtype=np.int64) ## arange index vector, doesnt get touched!
                yi  = np.arange(y.shape[0],dtype=np.int64)
                zi  = np.arange(z.shape[0],dtype=np.int64)
                ti  = np.arange(t.shape[0],dtype=np.int64)
                
                xfi = np.arange(x.shape[0],dtype=np.int64) ## gets clipped depending on x/y/z/t_min/max opts
                yfi = np.arange(y.shape[0],dtype=np.int64)
                zfi = np.arange(z.shape[0],dtype=np.int64)
                tfi = np.arange(t.shape[0],dtype=np.int64)
                
                # === total bounds clip (coordinate index) --> supports negative indexing!
                
                if True: ## code folding
                    
                    if (xi_min is not None):
                        xfi_ = []
                        if verbose:
                            if (xi_min<0):
                                even_print('xi_min', '%i / %i'%(xi_min,xi[xi_min]))
                            else:
                                even_print('xi_min', '%i'%(xi_min,))
                        for c in xfi:
                            if (xi_min<0) and (c>=(hf_src.nx+xi_min)):
                                xfi_.append(c)
                            elif (xi_min>=0) and (c>=xi_min):
                                xfi_.append(c)
                        xfi=np.array(xfi_, dtype=np.int64)
                    else:
                        xi_min = 0
                    
                    if (xi_max is not None):
                        xfi_ = []
                        if verbose:
                            if (xi_max<0):
                                even_print('xi_max', '%i / %i'%(xi_max,xi[xi_max]))
                            else:
                                even_print('xi_max', '%i'%(xi_max,))
                        for c in xfi:
                            if (xi_max<0) and (c<=(hf_src.nx+xi_max)):
                                xfi_.append(c)
                            elif (xi_max>=0) and (c<=xi_max):
                                xfi_.append(c)
                        xfi=np.array(xfi_, dtype=np.int64)
                    else:
                        xi_max = xi[-1]
                    
                    ## check x
                    if ((xi[xi_max]-xi[xi_min]+1)<1):
                        raise ValueError('invalid xi range requested')
                    if (rx>(xi[xi_max]-xi[xi_min]+1)):
                        raise ValueError('more ranks than grid points in x')
                    
                    if (yi_min is not None):
                        yfi_ = []
                        if verbose:
                            if (yi_min<0):
                                even_print('yi_min', '%i / %i'%(yi_min,yi[yi_min]))
                            else:
                                even_print('yi_min', '%i'%(yi_min,))
                        for c in yfi:
                            if (yi_min<0) and (c>=(hf_src.ny+yi_min)):
                                yfi_.append(c)
                            elif (yi_min>=0) and (c>=yi_min):
                                yfi_.append(c)
                        yfi=np.array(yfi_, dtype=np.int64)
                    else:
                        yi_min = 0
                    
                    if (yi_max is not None):
                        yfi_ = []
                        if verbose:
                            if (yi_max<0):
                                even_print('yi_max', '%i / %i'%(yi_max,yi[yi_max]))
                            else:
                                even_print('yi_max', '%i'%(yi_max,))
                        for c in yfi:
                            if (yi_max<0) and (c<=(hf_src.ny+yi_max)):
                                yfi_.append(c)
                            elif (yi_max>=0) and (c<=yi_max):
                                yfi_.append(c)
                        yfi=np.array(yfi_, dtype=np.int64)
                    else:
                        yi_max = yi[-1]
                    
                    ## check y
                    if ((yi[yi_max]-yi[yi_min]+1)<1):
                        raise ValueError('invalid yi range requested')
                    if (ry>(yi[yi_max]-yi[yi_min]+1)):
                        raise ValueError('more ranks than grid points in y')
                    
                    if (zi_min is not None):
                        zfi_ = []
                        if verbose:
                            if (zi_min<0):
                                even_print('zi_min', '%i / %i'%(zi_min,zi[zi_min]))
                            else:
                                even_print('zi_min', '%i'%(zi_min,))
                        for c in zfi:
                            if (zi_min<0) and (c>=(hf_src.nz+zi_min)):
                                zfi_.append(c)
                            elif (zi_min>=0) and (c>=zi_min):
                                zfi_.append(c)
                        zfi=np.array(zfi_, dtype=np.int64)
                    else:
                        zi_min = 0
                    
                    if (zi_max is not None):
                        zfi_ = []
                        if verbose:
                            if (zi_max<0):
                                even_print('zi_max', '%i / %i'%(zi_max,zi[zi_max]))
                            else:
                                even_print('zi_max', '%i'%(zi_max,))
                        for c in zfi:
                            if (zi_max<0) and (c<=(hf_src.nz+zi_max)):
                                zfi_.append(c)
                            elif (zi_max>=0) and (c<=zi_max):
                                zfi_.append(c)
                        zfi=np.array(zfi_, dtype=np.int64)
                    else:
                        zi_max = zi[-1]
                    
                    ## check z
                    if ((zi[zi_max]-zi[zi_min]+1)<1):
                        raise ValueError('invalid zi range requested')
                    if (rz>(zi[zi_max]-zi[zi_min]+1)):
                        raise ValueError('more ranks than grid points in z')
                    
                    if (ti_min is not None):
                        tfi_ = []
                        if verbose:
                            if (ti_min<0):
                                even_print('ti_min', '%i / %i'%(ti_min,ti[ti_min]))
                            else:
                                even_print('ti_min', '%i'%(ti_min,))
                        for c in tfi:
                            if (ti_min<0) and (c>=(hf_src.nt+ti_min)):
                                tfi_.append(c)
                            elif (ti_min>=0) and (c>=ti_min):
                                tfi_.append(c)
                        tfi=np.array(tfi_, dtype=np.int64)
                    else:
                        ti_min = 0
                    
                    if (ti_max is not None):
                        tfi_ = []
                        if verbose:
                            if (ti_max<0):
                                even_print('ti_max', '%i / %i'%(ti_max,ti[ti_max]))
                            else:
                                even_print('ti_max', '%i'%(ti_max,))
                        for c in tfi:
                            if (ti_max<0) and (c<=(hf_src.nt+ti_max)):
                                tfi_.append(c)
                            elif (ti_max>=0) and (c<=ti_max):
                                tfi_.append(c)
                        tfi=np.array(tfi_, dtype=np.int64)
                    else:
                        ti_max = ti[-1]
                    
                    ## check t
                    if ((ti[ti_max]-ti[ti_min]+1)<1):
                        raise ValueError('invalid ti range requested')
                    if (ct>(ti[ti_max]-ti[ti_min]+1)):
                        raise ValueError('more chunks than timesteps')
                
                # ===
                
                x  = np.copy(x[xfi]) ## target file
                y  = np.copy(y[yfi])
                z  = np.copy(z[zfi])
                t  = np.copy(t[tfi])
                
                nx = x.shape[0] ## target file
                ny = y.shape[0]
                nz = z.shape[0]
                nt = t.shape[0]
                
                if verbose:
                    even_print('fn_rgd_tgt' , fn_rgd_tgt )
                    even_print('nx' , '%i'%nx )
                    even_print('ny' , '%i'%ny )
                    even_print('nz' , '%i'%nz )
                    even_print('nt' , '%i'%nt )
                    print(72*'-')
                
                ## replace coordinate dimension arrays in target file
                if ('dims/x' in hf_tgt):
                    del hf_tgt['dims/x']
                    hf_tgt.create_dataset('dims/x', data=x, dtype=np.float64, chunks=None)
                if ('dims/y' in hf_tgt):
                    del hf_tgt['dims/y']
                    hf_tgt.create_dataset('dims/y', data=y, dtype=np.float64, chunks=None)
                if ('dims/z' in hf_tgt):
                    del hf_tgt['dims/z']
                    hf_tgt.create_dataset('dims/z', data=z, dtype=np.float64, chunks=None)
                if ('dims/t' in hf_tgt):
                    del hf_tgt['dims/t']
                    hf_tgt.create_dataset('dims/t', data=t, dtype=np.float64, chunks=None)
                
                # === 3D/4D communicator
                
                comm4d = hf_src.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                t4d = comm4d.Get_coords(rank)
                
                rxl_ = np.array_split(xfi,rx)
                ryl_ = np.array_split(yfi,ry)
                rzl_ = np.array_split(zfi,rz)
                #rtl_ = np.array_split(tfi,rt)
                
                rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                
                rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
                
                # ===
                
                rx1_ = rx1 - xi[xi_min] ## coords in target file
                ry1_ = ry1 - yi[yi_min]
                rz1_ = rz1 - zi[zi_min]
                #rt1_ = rt1 - ti[ti_min]
                
                rx2_ = rx2 - xi[xi_min] ## coords in target file
                ry2_ = ry2 - yi[yi_min]
                rz2_ = rz2 - zi[zi_min]
                #rt2_ = rt2 - ti[ti_min]
                
                ## time 'chunks' split (number of timesteps to read / write at a time)
                ctl_ = np.array_split(tfi,ct)
                ctl = [[b[0],b[-1]+1] for b in ctl_ ]
                
                shape  = (nt,nz,ny,nx) ## target
                hf_tgt.scalars = []
                #data_gb = 4*nx*ny*nz*nt / 1024**3
                
                ## initialize datasets
                t_start = timeit.default_timer()
                for scalar in hf_src.scalars:
                    
                    dtype = hf_src.scalars_dtypes_dict[scalar]
                    float_bytes = dtype.itemsize
                    
                    chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                    data_gb = float_bytes * nx * ny * nz * nt / 1024**3
                    
                    if (scalar in scalars):
                        if verbose:
                            even_print('initializing data/%s'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        dset = hf_tgt.create_dataset('data/%s'%scalar,
                                                     shape=shape,
                                                     dtype=dtype,
                                                     chunks=chunks)
                        hf_tgt.scalars.append(scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                t_initialize = timeit.default_timer() - t_start
                if verbose:
                    even_print('time initialize',format_time_string(t_initialize))
                    print(72*'-')
                
                # ===
                
                hf_tgt.n_scalars = len(hf_tgt.scalars)
                
                # ===
                
                data_gb_read  = 0.
                data_gb_write = 0.
                t_read  = 0.
                t_write = 0.
                
                if verbose:
                    progress_bar = tqdm(total=len(ctl)*hf_tgt.n_scalars, ncols=100, desc='copy', leave=False, file=sys.stdout)
                
                for scalar in hf_tgt.scalars:
                    dset_src = hf_src[f'data/{scalar}']
                    dset_tgt = hf_tgt[f'data/{scalar}']
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    for ctl_ in ctl:
                        
                        ct1, ct2 = ctl_
                        
                        ct1_ = ct1 - ti[ti_min] ## coords in target file
                        ct2_ = ct2 - ti[ti_min]
                        
                        ## read
                        hf_src.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_src.collective:
                            data = dset_src[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                        hf_src.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                        data_gb = float_bytes*nx*ny*nz*(ct2-ct1) / 1024**3
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        if verbose:
                            tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        ## write
                        hf_tgt.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_tgt.collective:
                            dset_tgt[ct1_:ct2_,rz1_:rz2_,ry1_:ry2_,rx1_:rx2_] = data.T
                        hf_tgt.flush()
                        hf_tgt.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                        data_gb = float_bytes*nx*ny*nz*(ct2-ct1) / 1024**3
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            tqdm.write(even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        if verbose:
                            progress_bar.update()
                
                if verbose:
                    progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: even_print('time initialize',format_time_string(t_initialize))
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_rgd_src, '%0.2f [GB]'%(os.path.getsize(fn_rgd_src)/1024**3))
        if verbose: even_print(fn_rgd_tgt, '%0.2f [GB]'%(os.path.getsize(fn_rgd_tgt)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.copy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def delete_scalars(self,scalar_list,**kwargs):
        '''
        delete scalars from RGD 
        '''
        pass
        return
    
    def read(self,**kwargs):
        '''
        read data from file & return structured array
        '''
        
        verbose_master = kwargs.get('verbose',True)
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose:
            verbose = verbose_master
        
        if verbose: print('\n'+'rgd.read()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print(self.fname,'%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: print(72*'-')
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        rt       = kwargs.get('rt',1)
        scalars_to_read = kwargs.get('scalars',None)
        
        if (rx*ry*rz*rt!=self.n_ranks):
            raise AssertionError('rx*ry*rz*rt!=self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if (scalars_to_read is None):
            scalars_to_read = self.scalars
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d    = comm4d.Get_coords(self.rank)
            ##
            rxl_   = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_   = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_   = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            rtl_   = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            rxl    = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl    = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl    = [[b[0],b[-1]+1] for b in rzl_ ]
            rtl    = [[b[0],b[-1]+1] for b in rtl_ ]
            ##
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            ntr = self.nt
        
        t_read = 0.
        data_gb_read = 0.
        
        data_gb = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        
        # ===
        
        names   = [ s for s in scalars_to_read if s in self.scalars ]
        formats = [ self.scalars_dtypes_dict[n] for n in names ]
        
        ## 5D [scalar][x,y,z,t] structured array
        data = np.zeros(shape=(nxr,nyr,nzr,ntr), dtype={'names':names, 'formats':formats})
        
        for scalar in names:
            
            # === collective read
            dset = self['data/%s'%scalar]
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi: 
                with dset.collective:
                    data[scalar] = dset[rt1:rt2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                data[scalar] = dset[()].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            
            if verbose:
                even_print( 'read: %s'%scalar , '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)) )
            
            t_read       += t_delta
            data_gb_read += data_gb
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('rgd.read()', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: print(72*'-')
        
        return data
    
    # === test data populators
    
    def populate_abc_flow(self, **kwargs):
        '''
        populate (unsteady) ABC flow dummy data
        -----
        https://en.wikipedia.org/wiki/Arnold%E2%80%93Beltrami%E2%80%93Childress_flow
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.populate_abc_flow()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ##
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        
        self.nx = nx = kwargs.get('nx',100)
        self.ny = ny = kwargs.get('ny',100)
        self.nz = nz = kwargs.get('nz',100)
        self.nt = nt = kwargs.get('nt',100)
        
        data_gb = 3 * 4*nx*ny*nz*nt / 1024.**3
        if verbose: even_print(self.fname, '%0.2f [GB]'%(data_gb,))
        
        self.x = x = np.linspace(0., 2*np.pi, nx, dtype=np.float32)
        self.y = y = np.linspace(0., 2*np.pi, ny, dtype=np.float32)
        self.z = z = np.linspace(0., 2*np.pi, nz, dtype=np.float32)
        #self.t = t = np.linspace(0., 10.,     nt, dtype=np.float32)
        self.t = t = 0.1 * np.arange(nt, dtype=np.float32)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        # ===
        
        comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
        t4d = comm4d.Get_coords(self.rank)
        
        rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ## per-rank dim range
        xr = x[rx1:rx2]
        yr = y[ry1:ry2]
        zr = z[rz1:rz2]
        #tr = t[rt1:rt2]
        tr = np.copy(t)
        
        ## write dims
        self.create_dataset('dims/x', data=x)
        self.create_dataset('dims/y', data=y)
        self.create_dataset('dims/z', data=z)
        self.create_dataset('dims/t', data=t)
        
        shape  = (self.nt,self.nz,self.ny,self.nx)
        chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        
        ## initialize
        data_gb = 4*nx*ny*nz*nt / 1024.**3
        for scalar in ['u','v','w']:
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            if verbose:
                even_print('initializing data/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            dset = self.create_dataset('data/%s'%scalar, 
                                        shape=shape,
                                        dtype=np.float32,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === make 4D ABC flow data
        
        t_start = timeit.default_timer()
        A = np.sqrt(3)
        B = np.sqrt(2)
        C = 1.
        na = np.newaxis
        u = (A + 0.5 * tr[na,na,na,:] * np.sin(np.pi*tr[na,na,na,:])) * np.sin(zr[na,na,:,na]) + \
            B * np.cos(yr[na,:,na,na]) + \
            0.*xr[:,na,na,na]
        v = B * np.sin(xr[:,na,na,na]) + \
            C * np.cos(zr[na,na,:,na]) + \
            0.*yr[na,:,na,na] + \
            0.*tr[na,na,na,:]
        w = C * np.sin(yr[na,:,na,na]) + \
            (A + 0.5 * tr[na,na,na,:] * np.sin(np.pi*tr[na,na,na,:])) * np.cos(xr[:,na,na,na]) + \
            0.*zr[na,na,:,na]
        
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('calc flow','%0.3f [s]'%(t_delta,))
        
        # ===
        
        data_gb = 4*nx*ny*nz*nt / 1024.**3
        
        self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['data/u']
        with ds.collective:
            ds[:,rz1:rz2,ry1:ry2,rx1:rx2] = u.T
        self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: u','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['data/v']
        with ds.collective:
            ds[:,rz1:rz2,ry1:ry2,rx1:rx2] = v.T
        self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: v','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['data/w']
        with ds.collective:
            ds[:,rz1:rz2,ry1:ry2,rx1:rx2] = w.T
        self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: w','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        # ===
        
        if verbose: print('\n'+72*'-')
        if verbose: print('total time : rgd.populate_abc_flow() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def populate_white_noise(self, **kwargs):
        '''
        populate white noise dummy data
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.populate_white_noise()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        self.nx = nx = kwargs.get('nx',128)
        self.ny = ny = kwargs.get('ny',128)
        self.nz = nz = kwargs.get('nz',128)
        self.nt = nt = kwargs.get('nt',128)
        
        #data_gb = 3 * 4*nx*ny*nz*nt / 1024.**3
        data_gb = 1 * 4*nx*ny*nz*nt / 1024.**3
        
        if verbose: even_print(self.fname, '%0.2f [GB]'%(data_gb,))
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        
        self.x = x = np.linspace(0., 2*np.pi, nx, dtype=np.float32)
        self.y = y = np.linspace(0., 2*np.pi, ny, dtype=np.float32)
        self.z = z = np.linspace(0., 2*np.pi, nz, dtype=np.float32)
        #self.t = t = np.linspace(0., 10.,     nt, dtype=np.float32)
        self.t = t = 0.1 * np.arange(nt, dtype=np.float32)
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d    = comm4d.Get_coords(self.rank)
            ##
            rxl_   = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_   = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_   = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            rtl_   = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            rxl    = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl    = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl    = [[b[0],b[-1]+1] for b in rzl_ ]
            rtl    = [[b[0],b[-1]+1] for b in rtl_ ]
            ##
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
            ##
            ## ## per-rank dim range
            ## xr = x[rx1:rx2]
            ## yr = y[ry1:ry2]
            ## zr = z[rz1:rz2]
            ## tr = t[rt1:rt2]
        else:
            nxr = nx
            nyr = ny
            nzr = nz
            ntr = nt
        
        ## write dims (independent)
        self.create_dataset('dims/x', data=x, chunks=None)
        self.create_dataset('dims/y', data=y, chunks=None)
        self.create_dataset('dims/z', data=z, chunks=None)
        self.create_dataset('dims/t', data=t, chunks=None)
        
        shape  = (self.nt,self.nz,self.ny,self.nx)
        float_bytes = 4
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        #self.scalars = ['u','v','w']
        self.scalars = ['u']
        self.scalars_dtypes = [np.float32 for s in self.scalars]
        
        ## initialize datasets
        data_gb = 4*nx*ny*nz*nt / 1024.**3
        self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        for scalar in self.scalars:
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            if verbose:
                even_print('initializing data/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            dset = self.create_dataset('data/%s'%scalar, 
                                        shape=shape,
                                        dtype=np.float32,
                                        chunks=chunks )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        self.usingmpi: self.comm.Barrier()
        t_initialize = timeit.default_timer() - t_start
        
        ## 5D [scalar][x,y,z,t] structured array --> data buffer
        data = np.zeros(shape=(nxr,nyr,nzr,ntr), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
        
        ## generate data
        if verbose: print(72*'-')
        
        self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        rng = np.random.default_rng(seed=self.rank)
        for scalar in self.scalars:
            data[scalar] = rng.uniform(-1, +1, size=(nxr,nyr,nzr,ntr)).astype(np.float32)
        self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('gen data','%0.3f [s]'%(t_delta,))
        if verbose: print(72*'-')
        
        ## write data
        data_gb_write = 0.
        t_write = 0.
        for scalar in self.scalars:
            ds = self['data/%s'%scalar]
            self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with ds.collective:
                    ds[rt1:rt2,rz1:rz2,ry1:ry2,rx1:rx2] = data[scalar].T
            else:
                ds[:,:,:,:] = data[scalar].T
            self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4*nx*ny*nz*nt / 1024**3
            
            t_write       += t_delta
            data_gb_write += data_gb
            
            if verbose:
                even_print('write: %s'%(scalar,), '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        if verbose: print(72*'-')
        if verbose: even_print('time initialize',format_time_string(t_initialize))
        if verbose: even_print('write total', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.populate_white_noise() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing
    
    def get_mean_legacy(self, **kwargs):
        '''
        get mean in [t] --> leaves [x,y,z,1]
        --> save to new RGD file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.get_mean()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        #rt = 1
        
        fn_rgd_mean  = kwargs.get('fn_rgd_mean',None)
        #sfm         = kwargs.get('scalars',None) ## scalars to take (for mean)
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        ##
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        ## HDF5 chunk parameters (t,z,y,x)
        if (chunk_constraint is None):
            chunk_constraint = (1,None,None,None) ## single [t] convention
            #chunk_constraint = (None,-1,1,-1) ## single [y] convention
        if (chunk_base is None):
            chunk_base = 2
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === mean file name (for writing)
        if (fn_rgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_rgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_rgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_rgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if verbose: even_print('fn_rgd'       , self.fname   )
        if verbose: even_print('fn_rgd_mean'  , fn_rgd_mean  )
        #if verbose: even_print('fn_rgd_prime' , fn_rgd_prime )
        if verbose: even_print('do Favre avg' , str(favre)   )
        if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        t_avg_start  = self.t[ti_for_avg[0]]
        t_avg_end    = self.t[ti_for_avg[-1]]
        duration_avg = t_avg_end - t_avg_start
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        if verbose: even_print('Δt','%0.2f [-]'%(dt0,))
        #if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        ##data_gb      = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        #data_gb      = 4*self.nx*self.ny*self.nz*nt_avg / 1024**3
        #data_gb_mean = 4*self.nx*self.ny*self.nz*1      / 1024**3
        
        scalars_re = ['u','v','w','p','T','rho']
        scalars_fv = ['u','v','w','T'] ## 'p','rho'
        
        #with rgd(fn_rgd_mean, 'w', force=force, driver='mpio', comm=MPI.COMM_WORLD) as hf_mean:
        with rgd(fn_rgd_mean, 'w', force=force, driver=self.driver, comm=self.comm) as hf_mean:
            
            ## initialize the mean file from the opened unsteady rgd file
            hf_mean.init_from_rgd(self.fname)
            
            ## set some top-level attributes
            hf_mean.attrs['duration_avg'] = duration_avg ## duration of mean
            #hf_mean.attrs['duration_avg'] = self.duration
            hf_mean.attrs['dt'] = dt0
            #hf_mean.attrs['fclass'] = 'rgd'
            hf_mean.attrs['fsubtype'] = 'mean'
            
            if verbose: print(72*'-')
            
            # === initialize mean datasets
            for scalar in self.scalars:
                
                dtype = self.scalars_dtypes_dict[scalar]
                float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                
                data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                
                shape  = (1,self.nz,self.ny,self.nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        if ('data/%s'%scalar in hf_mean):
                            del hf_mean['data/%s'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    
                    if (scalar in scalars_fv):
                        if ('data/%s_fv'%scalar in hf_mean):
                            del hf_mean['data/%s_fv'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}_fv' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}_fv',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s_fv'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            # === read rho
            if favre:
                
                dset = self['data/rho']
                
                dtype = self.scalars_dtypes_dict['rho']
                if (dtype!=dset.dtype):
                    raise ValueError
                float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi: 
                    with dset.collective:
                        #rho = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                        rho = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    #rho = dset[()].T
                    rho = dset[ti_min:,:,:,:].T
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                
                if verbose:
                    txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                    tqdm.write(txt)
                
                t_read       += t_delta
                data_gb_read += data_gb
                
                ## mean ρ in [t] --> leave [x,y,z]
                rho_mean = np.mean(rho, axis=-1, keepdims=True, dtype=np.float64)
                
                if (dtype==np.float32):
                    rho_mean = np.copy( rho_mean.astype(np.float32) )
            
            # === read, do mean, write
            for scalar in self.scalars:
                
                dset = self[f'data/{scalar}']
                dtype = self.scalars_dtypes_dict[scalar]
                if (dtype!=dset.dtype):
                    raise ValueError
                float_bytes = dtype.itemsize
                
                # === collective read
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi:
                    with dset.collective:
                        #data = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                        data = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    #data = dset[()].T
                    data = dset[ti_min:,:,:,:].T
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                
                if (self.rank==0):
                    txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                    tqdm.write(txt)
                
                t_read       += t_delta
                data_gb_read += data_gb
                
                # === do mean in [t]
                if reynolds:
                    
                    data_mean = np.mean(data, axis=-1, keepdims=True, dtype=np.float64)
                    if (dtype==np.float32):
                        data_mean = np.copy( data_mean.astype(np.float32) )
                
                if favre:
                    
                    data_mean_fv = np.mean(data*rho, axis=-1, keepdims=True, dtype=np.float64)
                    if (dtype==np.float32):
                        data_mean_fv = np.copy( data_mean_fv.astype(np.float32) )
                    
                    ## divide off mean mass density
                    data_mean_fv = np.copy( data_mean_fv / rho_mean )
                
                # === write
                if reynolds:
                    if scalar in scalars_re:
                        
                        dset = hf_mean['data/%s'%scalar]
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_mean.T
                        else:
                            dset[:,:,:,:] = data_mean.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                        
                        if verbose:
                            txt = even_print('write: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
                
                if favre:
                    if scalar in scalars_fv:
                        
                        dset = hf_mean['data/%s_fv'%scalar]
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_mean_fv.T
                        else:
                            dset[:,:,:,:] = data_mean_fv.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                        
                        if verbose:
                            txt = even_print('write: %s_fv'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
            
            if self.usingmpi: self.comm.Barrier()
            
            # === replace dims/t array --> take last time of series
            t = np.array([self.t[-1]],dtype=np.float64)
            if ('dims/t' in hf_mean):
                del hf_mean['dims/t']
            hf_mean.create_dataset('dims/t', data=t)
            
            if hasattr(hf_mean, 'duration_avg'):
                if verbose: even_print('duration avg', '%0.2f [-]'%hf_mean.duration_avg)
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_rgd_mean, '%0.2f [GB]'%(os.path.getsize(fn_rgd_mean)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.get_mean() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def get_mean(self, **kwargs):
        '''
        get mean in [t] --> leaves [x,y,z,1]
        --> save to new RGD file
        -----
        - this version uses accumulator buffers and does *(1/n) at end to calculate mean
        - this allows for low RAM usage, as the time dim can be sub-chunked (ct=N)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.get_mean()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_rgd_mean  = kwargs.get('fn_rgd_mean',None)
        #sfm         = kwargs.get('scalars',None) ## scalars to take (for mean)
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        
        ct           = kwargs.get('ct',1)
        
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === mean file name (for writing)
        if (fn_rgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_rgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_rgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_rgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if verbose: even_print('fn_rgd'       , self.fname   )
        if verbose: even_print('fn_rgd_mean'  , fn_rgd_mean  )
        #if verbose: even_print('fn_rgd_prime' , fn_rgd_prime )
        if verbose: even_print('do Favre avg' , str(favre)   )
        if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        t_avg_start  = self.t[ti_for_avg[0]]
        t_avg_end    = self.t[ti_for_avg[-1]]
        duration_avg = t_avg_end - t_avg_start
        
        #if not isinstance(ct, (int,np.int32,np.int64)):
        if not isinstance(ct, int):
            raise ValueError
        if (ct<1):
            raise ValueError
        
        ## [t] sub chunk range
        ctl_ = np.array_split( ti_for_avg, min(ct,nt_avg) )
        ctl = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## check that no sub ranges are <=1
        for a_ in [ ctl_[1]-ctl_[0] for ctl_ in ctl ]:
            if (a_ <= 1):
                raise ValueError
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        if verbose: even_print('Δt','%0.2f [-]'%(dt0,))
        #if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        ##data_gb      = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        #data_gb      = 4*self.nx*self.ny*self.nz*nt_avg / 1024**3
        #data_gb_mean = 4*self.nx*self.ny*self.nz*1      / 1024**3
        
        scalars_re = ['u','v','w','p','T','rho']
        scalars_fv = ['u','v','w','T'] ## 'p','rho'
        
        ## do a loop through to get names
        scalars_mean_names  = []
        scalars_mean_dtypes = []
        for scalar in self.scalars:
            dtype = self.scalars_dtypes_dict[scalar]
            if reynolds:
                if True: ## always
                    sc_name = scalar
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
            if favre:
                if (scalar in scalars_fv):
                    sc_name = f'r_{scalar}'
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
        
        #with rgd(fn_rgd_mean, 'w', force=force, driver='mpio', comm=MPI.COMM_WORLD) as hf_mean:
        with rgd(fn_rgd_mean, 'w', force=force, driver=self.driver, comm=self.comm) as hf_mean:
            
            ## initialize the mean file from the opened unsteady rgd file
            hf_mean.init_from_rgd(self.fname)
            
            ## set some top-level attributes
            hf_mean.attrs['duration_avg'] = duration_avg ## duration of mean
            #hf_mean.attrs['duration_avg'] = self.duration
            hf_mean.attrs['dt'] = dt0
            #hf_mean.attrs['fclass'] = 'rgd'
            hf_mean.attrs['fsubtype'] = 'mean'
            
            if verbose: print(72*'-')
            
            # === initialize mean datasets
            for scalar in self.scalars:
                
                dtype = self.scalars_dtypes_dict[scalar]
                float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                
                data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                
                shape  = (1,self.nz,self.ny,self.nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        if ('data/%s'%scalar in hf_mean):
                            del hf_mean['data/%s'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    
                    if (scalar in scalars_fv):
                        if ('data/%s_fv'%scalar in hf_mean):
                            del hf_mean['data/%s_fv'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}_fv' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}_fv',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s_fv'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            ## accumulator array for local rank --> initialize
            data_sum = np.zeros(shape=(nxr,nyr,nzr,1), dtype={'names':scalars_mean_names, 'formats':[np.float64 for _ in scalars_mean_names]})
            
            # === main loop
            
            if verbose:
                progress_bar = tqdm(total=ct, ncols=100, desc='mean', leave=False, file=sys.stdout)
            
            ct_counter=0
            for ctl_ in ctl:
                ct_counter += 1
                ct1, ct2 = ctl_
                ntc = ct2 - ct1
                
                if (ct>1):
                    if verbose:
                        mesg = f'[t] sub chunk {ct_counter:d}/{ct:d}'
                        tqdm.write( mesg )
                        tqdm.write( '-'*len(mesg) )
                
                # === read rho
                if favre:
                    
                    dset = self['data/rho']
                    
                    dtype = self.scalars_dtypes_dict['rho']
                    if (dtype!=dset.dtype):
                        raise ValueError
                    float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if self.usingmpi: 
                        with dset.collective:
                            ##rho = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            #rho = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            rho = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                    else:
                        #rho = dset[()].T
                        #rho = dset[ti_min:,:,:,:].T
                        rho = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                    data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                    
                    if verbose:
                        txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                
                # === read data, sum
                for scalar in self.scalars:
                    
                    dset = self[f'data/{scalar}']
                    dtype = self.scalars_dtypes_dict[scalar]
                    if (dtype!=dset.dtype):
                        raise ValueError
                    float_bytes = dtype.itemsize
                    
                    # === collective read
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if self.usingmpi:
                        with dset.collective:
                            ##data = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            #data = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            data = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                    else:
                        ##data = dset[()].T
                        #data = dset[ti_min:,:,:,:].T
                        data = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                    data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                    
                    if verbose:
                        txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    # === do sum, add to accumulator
                    if reynolds:
                        sc_name = scalar
                        data_sum[sc_name] += np.sum(data, axis=-1, dtype=np.float64, keepdims=True)
                    if favre:
                        if (scalar in scalars_fv):
                            sc_name = f'r_{scalar}'
                            data_sum[sc_name] += np.sum(data*rho, axis=-1, dtype=np.float64, keepdims=True)
                    
                    if self.usingmpi: self.comm.Barrier()
                
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                
                if verbose: progress_bar.update()
                if verbose: tqdm.write(72*'-')
            if verbose: progress_bar.close()
            
            # ==========================================================
            # multiply accumulators by (1/n)
            # ==========================================================
            
            for scalar in self.scalars:
                if reynolds:
                    sc_name = scalar
                    data_sum[sc_name] *= (1/nt_avg)
                if favre:
                    if (scalar in scalars_fv):
                        sc_name = f'r_{scalar}'
                        data_sum[sc_name] *= (1/nt_avg)
            
            # ==========================================================
            # 'data_sum' now contains averages, not sums!
            # ==========================================================
            
            ## Favre avg : φ_tilde = avg[ρ·φ]/avg[ρ]
            rho_mean = np.copy( data_sum['rho'] )
            
            # === write
            for scalar in self.scalars:
                
                if reynolds:
                    
                    dset = hf_mean[f'data/{scalar}']
                    dtype = dset.dtype
                    float_bytes = dtype.itemsize
                    
                    if (dtype==np.float32):
                        data_out = np.copy( data_sum[scalar].astype(np.float32) )
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                    else:
                        dset[:,:,:,:] = data_out.T
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                    
                    if verbose:
                        txt = even_print(f'write: {scalar}', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_write       += t_delta
                    data_gb_write += data_gb_mean
                
                if favre:
                    if (scalar in scalars_fv):
                        
                        dset = hf_mean[f'data/{scalar}_fv']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        
                        ## φ_tilde = avg[ρ·φ]/avg[ρ]
                        data_out = np.copy( data_sum[f'r_{scalar}'] / rho_mean )
                        
                        if (dtype==np.float32):
                            data_out = np.copy( data_out.astype(np.float32) )
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                        else:
                            dset[:,:,:,:] = data_out.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                        
                        if verbose:
                            txt = even_print(f'write: {scalar}_fv', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
            
            # === replace dims/t array --> take last time of series
            t = np.array([self.t[-1]],dtype=np.float64)
            if ('dims/t' in hf_mean):
                del hf_mean['dims/t']
            hf_mean.create_dataset('dims/t', data=t)
            
            if hasattr(hf_mean, 'duration_avg'):
                if verbose: even_print('duration avg', '%0.2f [-]'%hf_mean.duration_avg)
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_rgd_mean, '%0.2f [GB]'%(os.path.getsize(fn_rgd_mean)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.get_mean() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def add_mean_dimensional_data_xpln(self, **kwargs):
        '''
        get dimensionalized mean data for [x] plane
        --> save to existing RGD file with fsubtype=mean
        - assumes volume which is thin in [x] direction
        - an RGD which is the output of rgd.get_mean() should be opened here
        - not parallel
        '''
        
        verbose = kwargs.get('verbose',True)
        epsilon = kwargs.get('epsilon',5e-4)
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.add_mean_dimensional_data_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## this func is not parallel
        if self.usingmpi:
            raise NotImplementedError('rgd.add_mean_dimensional_data_xpln() is not a parallel function')
        
        ## 'r' and 'w' open modes are not allowed
        if not (self.open_mode=='a') or (self.open_mode=='r+'):
            raise ValueError(f'open mode is {self.open_mode}')
        
        ## assert that this is a mean flow file ( i.e. output from cgd.get_mean() )
        if (self.fsubtype!='mean'):
            print(self.fsubtype)
            raise ValueError
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## read in 1D coordinate arrays, then dimensinoalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        nx = self.nx
        ny = self.ny
        nz = self.nz
        
        if (x.ndim!=1):
            raise ValueError
        if (y.ndim!=1):
            raise ValueError
        if (z.ndim!=1):
            raise ValueError
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## check if mean rgd has attr 'dt'
        if ('dt' in self.attrs.keys()):
            dt = self.attrs['dt']
            if (dt is not None):
                dt *= self.tchar
        else:
            raise ValueError
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration_avg,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration_avg*self.tchar,))
        if verbose: print(72*'-')
        
        ## re-dimensionalize
        u   =  self.U_inf                     * np.copy( self['data/u'][()].T   )
        v   =  self.U_inf                     * np.copy( self['data/v'][()].T   )
        w   =  self.U_inf                     * np.copy( self['data/w'][()].T   )
        rho =  self.rho_inf                   * np.copy( self['data/rho'][()].T )
        p   =  (self.rho_inf * self.U_inf**2) * np.copy( self['data/p'][()].T   )
        T   =  self.T_inf                     * np.copy( self['data/T'][()].T   )
        
        # mu1 = (14.58e-7 * T**1.5) / ( T + 110.4 )
        # mu2 = self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * (self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth)
        # mu3 = self.C_Suth * T**(3/2) / (T + self.S_Suth)
        # np.testing.assert_allclose(mu1, mu2, rtol=1e-14, atol=1e-14)
        # np.testing.assert_allclose(mu2, mu3, rtol=1e-14, atol=1e-14)
        # mu = np.copy(mu3)
        
        mu = self.C_Suth * T**(3/2) / (T + self.S_Suth)
        nu = mu / rho
        
        # === average in [z] --> leave 2D [x,y]
        
        u     = np.squeeze( np.mean( u     , axis=2, dtype=np.float64).astype(np.float32) )
        v     = np.squeeze( np.mean( v     , axis=2, dtype=np.float64).astype(np.float32) )
        w     = np.squeeze( np.mean( w     , axis=2, dtype=np.float64).astype(np.float32) )
        rho   = np.squeeze( np.mean( rho   , axis=2, dtype=np.float64).astype(np.float32) )
        p     = np.squeeze( np.mean( p     , axis=2, dtype=np.float64).astype(np.float32) )
        T     = np.squeeze( np.mean( T     , axis=2, dtype=np.float64).astype(np.float32) )
        #utang = np.squeeze( np.mean( utang , axis=2, dtype=np.float64).astype(np.float32) )
        #unorm = np.squeeze( np.mean( unorm , axis=2, dtype=np.float64).astype(np.float32) )
        mu    = np.squeeze( np.mean( mu    , axis=2, dtype=np.float64).astype(np.float32) )
        nu    = np.squeeze( np.mean( nu    , axis=2, dtype=np.float64).astype(np.float32) )
        
        ## determine finite difference order / size of central stencil 
        if (nx<3):
            raise ValueError('dx[] not possible because nx<3')
        elif (nx>=3) and (nx<5):
            acc = 2
        elif (nx>=5) and (nx<7):
            acc = 4
        elif (nx>=7):
            acc = 6
        else:
            raise ValueError('this should never happen')
        
        edge_stencil = 'half'
        if verbose: even_print('acc','%i'%acc)
        if verbose: even_print('edge_stencil',edge_stencil)
        
        # === get gradients & vort_z
        
        ddx_u = gradient(u, x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddy_u = gradient(u, y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        
        ddx_v = gradient(v, x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddy_v = gradient(v, y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        
        ## dimensional [1/s]
        vort_z = ddx_v - ddy_u
        
        # ===
        
        ## [y] gradients --> yields 1D in [y]
        ddy_u   = gradient(u   , y, axis=1, acc=acc, edge_stencil=edge_stencil)
        ddy_v   = gradient(v   , y, axis=1, acc=acc, edge_stencil=edge_stencil)
        ddy_T   = gradient(T   , y, axis=1, acc=acc, edge_stencil=edge_stencil)
        ddy_p   = gradient(p   , y, axis=1, acc=acc, edge_stencil=edge_stencil)
        ddy_rho = gradient(rho , y, axis=1, acc=acc, edge_stencil=edge_stencil)
        
        ## wall quantities --> mean over [x] --> leave 0D float
        ddy_u_wall  = float( np.mean( ddy_u[:,0] ) )
        ddy_T_wall  = float( np.mean( ddy_T[:,0] ) )
        rho_wall    = float( np.mean( rho[:,0]   ) )
        nu_wall     = float( np.mean( nu[:,0]    ) )
        mu_wall     = float( np.mean( mu[:,0]    ) )
        T_wall      = float( np.mean( T[:,0]     ) )
        tau_wall    = mu_wall * ddy_u_wall
        q_wall      = self.cp * mu_wall / self.Pr * ddy_T_wall ## wall heat flux (?)
        
        ## friction velocity
        u_tau  = np.sqrt(tau_wall/rho_wall)
        #y_plus = np.copy( s2 * u_tau / nu_wall )
        
        # === populate 1D & 2D arrays using calc_bl_edge_1d(), calc_d99_1d()
        
        psvel = np.zeros((nx,ny), dtype=np.float64)
        
        psvel_edge = np.zeros((nx,), dtype=np.float64)
        u_edge     = np.zeros((nx,), dtype=np.float64)
        rho_edge   = np.zeros((nx,), dtype=np.float64)
        mu_edge    = np.zeros((nx,), dtype=np.float64)
        nu_edge    = np.zeros((nx,), dtype=np.float64)
        T_edge     = np.zeros((nx,), dtype=np.float64)
        
        y_edge = np.zeros((nx,), dtype=np.float64)
        d99    = np.zeros((nx,), dtype=np.float64)
        
        u_99   = np.zeros((nx,), dtype=np.float64)
        
        for i in range(nx):
            
            vort_z_ = np.copy( vort_z[i,:] )
            psvel_  = sp.integrate.cumulative_trapezoid(-1*vort_z_, y, initial=0.)
            psvel[i,:] = psvel_
            
            ## get edge
            y_edge_ = calc_bl_edge_1d( y=y, psvel=psvel_, ynorm=self.lchar, acc=acc, edge_stencil=edge_stencil, epsilon=epsilon )
            #aa = ( y_edge_ - y.min() ) / ( y.max() - y.min() )
            #tqdm.write(f'{aa:0.9f}')
            y_edge[i] = y_edge_
            
            ## interpolate at edge
            psvel_edge_ = sp.interpolate.interp1d(y, psvel_ , kind='cubic', bounds_error=True)(y_edge_)
            psvel_edge_ = float(psvel_edge_)
            psvel_edge[i] = psvel_edge_
            
            u_edge[i]   = sp.interpolate.interp1d(y, u[i,:]   , kind='cubic', bounds_error=True)(y_edge_)
            rho_edge[i] = sp.interpolate.interp1d(y, rho[i,:] , kind='cubic', bounds_error=True)(y_edge_)
            mu_edge[i]  = sp.interpolate.interp1d(y, mu[i,:]  , kind='cubic', bounds_error=True)(y_edge_)
            nu_edge[i]  = sp.interpolate.interp1d(y, nu[i,:]  , kind='cubic', bounds_error=True)(y_edge_)
            T_edge[i]   = sp.interpolate.interp1d(y, T[i,:]   , kind='cubic', bounds_error=True)(y_edge_)
            
            ## get d99
            d99_ = calc_d99_1d( y=y, y_edge=y_edge_, psvel=psvel_, psvel_edge=psvel_edge_ )
            d99_ = float(d99_)
            d99[i] = d99_
            
            u_99[i] = sp.interpolate.interp1d(y, u[i,:], kind='cubic', bounds_error=True)(d99_)
        
        # === avg in [x]/[s1] --> leave [y]/[s2]
        
        psvel = np.mean( psvel , axis=0 )
        u     = np.mean( u     , axis=0 )
        rho   = np.mean( rho   , axis=0 )
        
        psvel_edge  = np.mean( psvel_edge )
        u_edge      = np.mean( u_edge     )
        rho_edge    = np.mean( rho_edge   )
        mu_edge     = np.mean( mu_edge    )
        nu_edge     = np.mean( nu_edge    )
        T_edge      = np.mean( T_edge     )
        
        y_edge      = np.mean( y_edge     )
        d99         = np.mean( d99        )
        
        u_99        = np.mean( u_99      )
        
        sc_l_out = d99
        sc_u_out = u_99
        sc_t_out = d99/u_99
        np.testing.assert_allclose(sc_t_out, sc_l_out/sc_u_out, rtol=1e-14, atol=1e-14)
        
        sc_u_in = u_tau
        sc_l_in = nu_wall / u_tau
        sc_t_in = nu_wall / u_tau**2
        np.testing.assert_allclose(sc_t_in, sc_l_in/sc_u_in, rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration_avg * (self.lchar / self.U_inf)
        t_eddy = t_meas / (d99/u_tau)
        
        # === get 0D BL integral quantities
        
        dd = calc_bl_integral_quantities_1d( y=y,
                                             u=u,
                                             rho=rho,
                                             u_tau=u_tau,
                                             d99=d99,
                                             y_edge=y_edge,
                                             rho_edge=rho_edge,
                                             nu_edge=nu_edge,
                                             u_edge=u_edge,
                                             nu_wall=nu_wall,
                                            )
        
        # === add to file
        
        gn = 'data_dim'
        
        ## if group already exists in file, delete it entirely
        if (gn in self):
            del self[gn]
        
        ## 1D
        self.create_dataset(f'{gn}/psvel' , data=psvel , chunks=None)
        self.create_dataset(f'{gn}/u'     , data=u     , chunks=None)
        self.create_dataset(f'{gn}/rho'   , data=rho   , chunks=None)
        
        self.create_dataset(f'{gn}/ddy_u'   , data=ddy_u   , chunks=None)
        self.create_dataset(f'{gn}/ddy_v'   , data=ddy_v   , chunks=None)
        self.create_dataset(f'{gn}/ddy_T'   , data=ddy_T   , chunks=None)
        self.create_dataset(f'{gn}/ddy_p'   , data=ddy_p   , chunks=None)
        self.create_dataset(f'{gn}/ddy_rho' , data=ddy_rho , chunks=None)
        
        self.create_dataset(f'{gn}/z1d' , data=z , chunks=None)
        self.create_dataset(f'{gn}/z'   , data=z , chunks=None)
        
        ## 0D
        self.create_dataset(f'{gn}/dz0'        , data=dz0 , chunks=None)
        self.create_dataset(f'{gn}/dt'         , data=dt  , chunks=None)
        
        self.create_dataset(f'{gn}/y_edge'     , data=y_edge    , chunks=None)
        self.create_dataset(f'{gn}/d99'        , data=d99       , chunks=None)
        self.create_dataset(f'{gn}/u_99'       , data=u_99      , chunks=None)
        
        self.create_dataset(f'{gn}/psvel_edge' , data=psvel_edge , chunks=None)
        self.create_dataset(f'{gn}/u_edge'     , data=u_edge     , chunks=None)
        self.create_dataset(f'{gn}/rho_edge'   , data=rho_edge   , chunks=None)
        self.create_dataset(f'{gn}/mu_edge'    , data=mu_edge    , chunks=None)
        self.create_dataset(f'{gn}/nu_edge'    , data=nu_edge    , chunks=None)
        self.create_dataset(f'{gn}/T_edge'     , data=T_edge     , chunks=None)
        
        self.create_dataset(f'{gn}/u_tau'      , data=u_tau      , chunks=None)
        
        self.create_dataset(f'{gn}/ddy_u_wall' , data=ddy_u_wall , chunks=None )
        self.create_dataset(f'{gn}/ddy_T_wall' , data=ddy_T_wall , chunks=None )
        self.create_dataset(f'{gn}/rho_wall'   , data=rho_wall   , chunks=None )
        self.create_dataset(f'{gn}/nu_wall'    , data=nu_wall    , chunks=None )
        self.create_dataset(f'{gn}/mu_wall'    , data=mu_wall    , chunks=None )
        self.create_dataset(f'{gn}/T_wall'     , data=T_wall     , chunks=None )
        self.create_dataset(f'{gn}/tau_wall'   , data=tau_wall   , chunks=None )
        self.create_dataset(f'{gn}/q_wall'     , data=q_wall     , chunks=None )
        
        self.create_dataset(f'{gn}/sc_u_in'    , data=sc_u_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_l_in'    , data=sc_l_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_t_in'    , data=sc_t_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_u_out'   , data=sc_u_out   , chunks=None)
        self.create_dataset(f'{gn}/sc_l_out'   , data=sc_l_out   , chunks=None)
        self.create_dataset(f'{gn}/sc_t_out'   , data=sc_t_out   , chunks=None)
        
        ## add integrated quantities (all 0D)
        for key,val in dd.items():
            self.create_dataset(f'{gn}/{key}', data=val, chunks=None)
        
        ## report
        if verbose:
            print(72*'-')
            even_print('Re_τ'                      , '%0.1f'%dd['Re_tau']            )
            even_print('Re_θ'                      , '%0.1f'%dd['Re_theta']          )
            even_print('θ'                         , '%0.5e [m]'%dd['theta_cmp']     )
            even_print('δ*'                        , '%0.5e [m]'%dd['dstar_cmp']     )
            even_print('H12'                       , '%0.5f'%dd['H12']               )
            ##
            even_print('δ99'                       , '%0.5e [m]'%d99                 )
            even_print('θ/δ99'                     , '%0.5f'%(dd['theta_cmp']/d99)   )
            even_print('δ*/δ99'                    , '%0.5f'%(dd['dstar_cmp']/d99)   )
            even_print('u_τ'                       , '%0.3f [m/s]'%u_tau             )
            even_print('ν_wall'                    , '%0.5e [m²/s]'%nu_wall          )
            even_print('τ_wall'                    , '%0.5e [Pa]'%tau_wall           )
            even_print('τ_wall/q_inf'              , '%0.5e'%(tau_wall/(self.rho_inf*self.U_inf**2)) )
            even_print('cf = 2·τ_wall/q_edge'      , '%0.5e'%(2*tau_wall/(rho_edge*u_edge**2)) )
            even_print('t_meas'                    , '%0.5e [s]'%t_meas              )
            even_print('t_meas/tchar'              , '%0.1f'%(t_meas/self.tchar)     )
            even_print('t_eddy = t_meas/(δ99/u_τ)' , '%0.2f'%t_eddy                  )
            even_print('t_meas/(δ99/u_99)'         , '%0.2f'%(t_meas/(d99/u_99))     )
            even_print('t_meas/(20·δ99/u_99)'      , '%0.2f'%(t_meas/(20*d99/u_99))  )
            print(72*'-')
            even_print('sc_u_in = u_τ'               , '%0.5e [m/s]'%(sc_u_in,)  )
            even_print('sc_l_in = δ_ν = ν_wall/u_τ'  , '%0.5e [m]'%(sc_l_in,)    )
            even_print('sc_t_in = t_ν = ν_wall/u_τ²' , '%0.5e [s]'%(sc_t_in,)    )
            even_print('sc_u_out = u_99'             , '%0.5e [m/s]'%(sc_u_out,) )
            even_print('sc_l_out = δ99'              , '%0.5e [m]'%(sc_l_out,)   )
            even_print('sc_t_out = δ99/u_99'         , '%0.5e [s]'%(sc_t_out,)   )
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.add_mean_dimensional_data_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_mean_dimensional_data_ypln(self, **kwargs):
        '''
        get dimensionalized mean data for [y] plane
        --> save to existing RGD file with fsubtype=mean
        - assumes volume which is thin in [y] direction
        - an RGD which is the output of rgd.get_mean() should be opened here
        - not parallel
        '''
        
        pass
        raise NotImplementedError
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.add_mean_dimensional_data_ypln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_mean_dimensional_data_zpln(self, **kwargs):
        '''
        get dimensionalized mean data for [z] plane
        --> save to existing RGD file with fsubtype=mean
        - assumes volume which is thin in [z] direction
        - an RGD which is the output of rgd.get_mean() should be opened here
        - not parallel
        '''
        
        pass
        raise NotImplementedError
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.add_mean_dimensional_data_zpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing (unsteady)
    
    def get_prime(self, **kwargs):
        '''
        get mean-removed (prime) variables in [t]
        --> save to new RGD file
        -----
        XI  : Reynolds primes : mean(XI)=0
        XII : Favre primes    : mean(ρ·XII)=0 --> mean(XII)≠0 !!
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        ## assert that the opened RGD has fsubtype 'unsteady'
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        if verbose: print('\n'+'rgd.get_prime()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        fn_rgd_mean  = kwargs.get('fn_rgd_mean',None)
        fn_rgd_prime = kwargs.get('fn_rgd_prime',None)
        sfp          = kwargs.get('scalars',None) ## scalars (for prime)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        ## start timestep index
        ti_min = kwargs.get('ti_min',None)
        
        ## if writing Favre primes, copy over ρ --> mean(ρ·XII)=0 / mean(XII)≠0 !!
        if favre:
            copy_rho = True
        else:
            copy_rho = False
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if (sfp is None):
            sfp = self.scalars
        
        # === ranks
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === get times to take for prime
        if (ti_min is not None):
            ti_for_prime = np.copy( self.ti[ti_min:] )
        else:
            ti_for_prime = np.copy( self.ti )
        
        nt_prime       = ti_for_prime.shape[0]
        t_prime_start  = self.t[ti_for_prime[0]]
        t_prime_end    = self.t[ti_for_prime[-1]]
        duration_prime = t_prime_end - t_prime_start
        
        # === chunks
        #ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl_ = np.array_split(ti_for_prime,min(ct,nt_prime))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # === mean file name (for reading)
        if (fn_rgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_rgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_rgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_rgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if not os.path.isfile(fn_rgd_mean):
            raise FileNotFoundError('%s not found!'%fn_rgd_mean)
        
        # === prime file name (for writing)
        if (fn_rgd_prime is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_prime_h5_base = fname_root+'_prime.h5'
            #fn_rgd_prime = os.path.join(fname_path, fname_prime_h5_base)
            fn_rgd_prime = str(PurePosixPath(fname_path, fname_prime_h5_base))
            #fn_rgd_prime = Path(fname_path, fname_prime_h5_base)
        
        if verbose: even_print('fn_rgd'          , self.fname    )
        if verbose: even_print('fn_rgd_mean'     , fn_rgd_mean   )
        if verbose: even_print('fn_rgd_prime'    , fn_rgd_prime  )
        if verbose: even_print('do Favre avg'    , str(favre)    )
        if verbose: even_print('do Reynolds avg' , str(reynolds) )
        if verbose: even_print('copy rho'        , str(copy_rho) )
        if verbose: even_print('ct'              , '%i'%ct       )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('n timesteps prime','%i/%i'%(nt_prime,self.nt))
        if verbose: even_print('t index prime start','%i'%(ti_for_prime[0],))
        if verbose: even_print('t index prime end','%i'%(ti_for_prime[-1],))
        if verbose: even_print('t prime start','%0.2f [-]'%(t_prime_start,))
        if verbose: even_print('t prime end','%0.2f [-]'%(t_prime_end,))
        if verbose: even_print('duration prime','%0.2f [-]'%(duration_prime,))
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        #data_gb      = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        data_gb      = 4*self.nx*self.ny*self.nz*nt_prime / 1024**3
        data_gb_mean = 4*self.nx*self.ny*self.nz*1       / 1024**3
        
        scalars_re = ['u','v','w','T','p','rho']
        scalars_fv = ['u','v','w','T'] ## p'' and ρ'' are never really needed
        
        scalars_re_ = []
        for scalar in scalars_re:
            if (scalar in self.scalars) and (scalar in sfp):
                scalars_re_.append(scalar)
        scalars_re = scalars_re_
        
        scalars_fv_ = []
        for scalar in scalars_fv:
            if (scalar in self.scalars) and (scalar in sfp):
                scalars_fv_.append(scalar)
        scalars_fv = scalars_fv_
        
        # ===
        
        comm_rgd_prime = MPI.COMM_WORLD
        
        with rgd(fn_rgd_prime, 'w', force=force, driver=self.driver, comm=self.comm) as hf_prime:
            
            ## initialize prime rgd from rgd
            hf_prime.init_from_rgd(self.fname)
            
            ## add top-level attributes
            #hf_prime.attrs['fclass'] = 'rgd'
            hf_prime.attrs['fsubtype'] = 'prime'
            
            #shape  = (self.nt,self.nz,self.ny,self.nx)
            shape  = (nt_prime,self.nz,self.ny,self.nx)
            
            ## determine dtypes for prime file
            for scalar in self.scalars:
                dset = self[f'data/{scalar}']
                dtype = dset.dtype
                if reynolds and (scalar in scalars_re):
                    hf_prime.scalars_dtypes_dict[f'{scalar}I'] = dtype
                if favre and (scalar in scalars_fv):
                    hf_prime.scalars_dtypes_dict[f'{scalar}II'] = dtype
            
            # === initialize prime datasets + rho
            
            if copy_rho:
                
                dtype = self.scalars_dtypes_dict['rho']
                float_bytes = dtype.itemsize
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                
                if verbose:
                    even_print('initializing data/rho','%0.1f [GB]'%(data_gb,))
                
                dset = hf_prime.create_dataset('data/rho',
                                               shape=shape,
                                               dtype=dtype,
                                               chunks=chunks)
                
                chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                if verbose:
                    even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            for scalar in self.scalars:
                
                if reynolds:
                    if (scalar in scalars_re):
                        
                        dtype = hf_prime.scalars_dtypes_dict[f'{scalar}I']
                        float_bytes = dtype.itemsize
                        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                        
                        ## if ('data/%sI'%scalar in hf_prime):
                        ##     del hf_prime['data/%sI'%scalar]
                        if verbose:
                            even_print('initializing data/%sI'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        
                        dset = hf_prime.create_dataset(f'data/{scalar}I',
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_prime.scalars.append(f'{scalar}I')
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    if (scalar in scalars_fv):
                        
                        dtype = hf_prime.scalars_dtypes_dict[f'{scalar}II']
                        float_bytes = dtype.itemsize
                        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                        
                        ## if ('data/%sII'%scalar in hf_prime):
                        ##     del hf_prime['data/%sII'%scalar]
                        if verbose:
                            even_print('initializing data/%sII'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        
                        dset = hf_prime.create_dataset(f'data/{scalar}II',
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_prime.scalars.append(f'{scalar}II')
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if hf_prime.usingmpi: comm_rgd_prime.Barrier()
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            # === read unsteady + mean, do difference, write
            
            n_pbar = 0
            if favre or copy_rho:
                n_pbar += 1
            for scalar in self.scalars:
                if (scalar in scalars_re) and reynolds:
                    n_pbar += 1
                if (scalar in scalars_fv) and favre:
                    n_pbar += 1
            
            comm_rgd_mean = MPI.COMM_WORLD
            
            with rgd(fn_rgd_mean, 'r', driver=self.driver, comm=self.comm) as hf_mean:
                
                ## copy over 'data_dim' from mean file
                if ('data_dim' in hf_mean):
                    grp = hf_mean['data_dim']
                    for dsn in grp.keys():
                        ds = hf_mean[f'data_dim/{dsn}']
                        data = np.copy(ds[()])
                        #if (f'data_dim/{dsn}' in hf_prime):
                        #    del hf_prime[f'data_dim/{dsn}']
                        
                        #if self.usingmpi:
                        #    with dset.collective:
                        #        hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                        #else:
                        #    hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                        
                        hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                    
                    if verbose:
                        even_print('data_dim copied to prime',str(True))
                else:
                    if verbose:
                        even_print('data_dim copied to prime',str(False))
                
                if verbose:
                    progress_bar = tqdm(total=ct*n_pbar, ncols=100, desc='prime', leave=False, file=sys.stdout)
                
                for ctl_ in ctl:
                    ct1, ct2 = ctl_
                    ntc = ct2 - ct1
                    
                    #if verbose: tqdm.write(f'ct1,ct2 = {ct1:d},{ct2:d}')
                    #if verbose: tqdm.write(f'ntc = {ntc:d}')
                    
                    ## chunk range for writing to file (offset from read if using ti_min)
                    if (ti_min is not None):
                        #ct1w,ct2w = ct1-ti_min, ct2-ti_min ## doesnt work for (-) ti_min
                        ct1w,ct2w = ct1-ti_for_prime[0], ct2-ti_for_prime[0]
                    else:
                        ct1w,ct2w = ct1,ct2
                    
                    #if verbose: tqdm.write(f'ct1w,ct2w = {ct1w:d},{ct2w:d}')
                    
                    if favre or copy_rho:
                        
                        ## read rho
                        dset = self['data/rho']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                rho = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                        else:
                            rho = dset[ct1:ct2,:,:,:].T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if verbose:
                            txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        ## write a copy of rho to the prime file
                        dset = hf_prime['data/rho']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if hf_prime.usingmpi: hf_prime.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = rho.T
                                dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = rho.T
                        else:
                            #dset[ct1:ct2,:,:,:] = rho.T
                            dset[ct1w:ct2w,:,:,:] = rho.T
                        if hf_prime.usingmpi: hf_prime.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            txt = even_print('write: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        if verbose: progress_bar.update()
                    
                    for scalar in self.scalars:
                        
                        if (scalar in scalars_re) or (scalar in scalars_fv):
                            
                            ## read RGD data
                            dset = self['data/%s'%scalar]
                            dtype = dset.dtype
                            float_bytes = dtype.itemsize
                            data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                            
                            if self.usingmpi: self.comm.Barrier()
                            t_start = timeit.default_timer()
                            if self.usingmpi:
                                with dset.collective:
                                    data = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                            else:
                                data = dset[ct1:ct2,:,:,:].T
                            if self.usingmpi: self.comm.Barrier()
                            t_delta = timeit.default_timer() - t_start
                            
                            if verbose:
                                txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                tqdm.write(txt)
                            
                            t_read       += t_delta
                            data_gb_read += data_gb
                            
                            # === do prime Reynolds
                            
                            if (scalar in scalars_re) and reynolds:
                                
                                ## read Reynolds avg from mean file
                                dset = hf_mean['data/%s'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_mean.nx * hf_mean.ny * hf_mean.nz * 1 / 1024**3
                                
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_mean.usingmpi:
                                    with dset.collective:
                                        data_mean_re = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                                else:
                                    data_mean_re = dset[()].T
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                ## if verbose:
                                ##     txt = even_print('read: %s (Re avg)'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                ##     tqdm.write(txt)
                                
                                ## calc mean-removed Reynolds
                                data_prime_re = data - data_mean_re
                                
                                ## if False:
                                ##     data_prime_re_mean = np.mean(data_prime_re, axis=-1, dtype=np.float64, keepdims=True).astype(np.float32)
                                ##     
                                ##     ## normalize [mean(prime)] by mean
                                ##     data_prime_re_mean = np.abs(np.divide(data_prime_re_mean,
                                ##                                           data_mean_re, 
                                ##                                           out=np.zeros_like(data_prime_re_mean), 
                                ##                                           where=data_mean_re!=0))
                                ##     
                                ##     # np.testing.assert_allclose( data_prime_re_mean , 
                                ##     #                             np.zeros_like(data_prime_re_mean, dtype=np.float32), atol=1e-4)
                                ##     if verbose:
                                ##         tqdm.write('max(abs(mean(%sI)/mean(%s)))=%0.4e'%(scalar,scalar,data_prime_re_mean.max()))
                                
                                ## write Reynolds prime
                                dset = hf_prime['data/%sI'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_prime.nx * hf_prime.ny * hf_prime.nz * ntc / 1024**3
                                
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_prime.usingmpi:
                                    with dset.collective:
                                        #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_re.T
                                        dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_re.T
                                else:
                                    #dset[ct1:ct2,:,:,:] = data_prime_re.T
                                    dset[ct1w:ct2w,:,:,:] = data_prime_re.T
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if verbose:
                                    txt = even_print('write: %sI'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                    tqdm.write(txt)
                                
                                if verbose: progress_bar.update()
                            
                            # === do prime Favre
                            
                            if (scalar in scalars_fv) and favre:
                                
                                ## read Favre avg from mean file
                                dset = hf_mean['data/%s_fv'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_mean.nx * hf_mean.ny * hf_mean.nz * 1 / 1024**3
                                
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_mean.usingmpi:
                                    with dset.collective:
                                        data_mean_fv = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                                else:
                                    data_mean_fv = dset[()].T
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                ## if verbose:
                                ##     txt = even_print('read: %s (Fv avg)'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                ##     tqdm.write(txt)
                                
                                ## calc mean-removed Favre
                                ## data_prime_fv = ( data - data_mean_fv ) * rho ## pre-multiply with ρ (has zero mean) --> better to not do this here
                                data_prime_fv = data - data_mean_fv
                                
                                ## write Favre prime
                                dset = hf_prime['data/%sII'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_prime.nx * hf_prime.ny * hf_prime.nz * ntc / 1024**3
                                
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_prime.usingmpi:
                                    with dset.collective:
                                        #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_fv.T
                                        dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_fv.T
                                else:
                                    #dset[ct1:ct2,:,:,:] = data_prime_fv.T
                                    dset[ct1w:ct2w,:,:,:] = data_prime_fv.T
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if verbose:
                                    txt = even_print('write: %sII'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                    tqdm.write(txt)
                                
                                pass
                                
                                if verbose: progress_bar.update()
                        
                        if self.usingmpi: self.comm.Barrier()
                        if hf_prime.usingmpi: comm_rgd_prime.Barrier()
                        if hf_mean.usingmpi: comm_rgd_mean.Barrier()
                    
                    ## report mem free
                    mem_avail_gb = psutil.virtual_memory().available/1024**3
                    mem_free_gb  = psutil.virtual_memory().free/1024**3
                    if verbose:
                        tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                
                if verbose:
                    progress_bar.close()
            
            # === replace dims/t array in prime file (if ti_min was given)
            if (ti_min is not None):
                t = np.copy( self.t[ti_min:] )
                if ('dims/t' in hf_prime):
                    del hf_prime['dims/t']
                hf_prime.create_dataset('dims/t', data=t)
            
            if hf_mean.usingmpi: comm_rgd_mean.Barrier()
        if hf_prime.usingmpi: comm_rgd_prime.Barrier()
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_rgd_prime, '%0.2f [GB]'%(os.path.getsize(fn_rgd_prime)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.get_prime() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_lambda2(self, **kwargs):
        '''
        calculate λ-2 & Q
        Jeong & Hussain (1996) : https://doi.org/10.1017/S0022112095000462
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        save_Q       = kwargs.get('save_Q',True)
        save_lambda2 = kwargs.get('save_lambda2',True)
        rx           = kwargs.get('rx',1)
        ry           = kwargs.get('ry',1)
        rz           = kwargs.get('rz',1)
        chunk_kb     = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        acc          = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','half')
        
        d = 1 ## derivative order
        stencil_npts = 2*math.floor((d+1)/2) - 1 + acc
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        
        # ===
        
        if verbose: print('\n'+'turbx.rgd.calc_lambda2()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## checks
        if all([(save_Q is False),(save_lambda2 is False)]):
            raise AssertionError('neither λ-2 nor Q set to be solved')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('save_Q','%s'%save_Q)
        if verbose: even_print('save_lambda2','%s'%save_lambda2)
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        t_q_crit = 0.
        t_l2     = 0.
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # === extend the rank ranges (overlap)
        
        if self.usingmpi:
            
            n_overlap = stencil_npts_one_side + 3
            
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            zA = 0
            zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## overlap in [y]
            if (t4d[1]!=0):
                ry1, ry2 = ry1-n_overlap, ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                ry1, ry2 = ry1, ry2+n_overlap
            
            ## overlap in [z]
            if (t4d[2]!=0):
                rz1, rz2 = rz1-n_overlap, rz2
                zA += n_overlap
                zB += n_overlap
            if (t4d[2]!=rz-1):
                rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        ## determine dtype of Q, lambda2 based on dtype of u
        #dtype = self.scalars_dtypes_dict['u']
        dset = self['data/u']
        dtype = dset.dtype
        float_bytes = dtype.itemsize
        data_gb = float_bytes*self.nt*self.nz*self.ny*self.nx / 1024**3
        
        shape  = (self.nt,self.nz,self.ny,self.nx)
        chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=float_bytes)
        
        # === initialize 4D arrays in HDF5
        
        if save_lambda2:
            if verbose: even_print('initializing data/lambda2','%0.2f [GB]'%(data_gb,))
            if ('data/lambda2' in self):
                del self['data/lambda2']
            dset = self.create_dataset('data/lambda2', 
                                        shape=shape, 
                                        dtype=self['data/u'].dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if save_Q:
            if verbose: even_print('initializing data/Q','%0.2f [GB]'%(data_gb,))
            if ('data/Q' in self):
                del self['data/Q']
            dset = self.create_dataset('data/Q', 
                                        shape=shape, 
                                        dtype=self['data/u'].dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # ===
        
        if verbose:
            progress_bar = tqdm(total=self.nt, ncols=100, desc='calc λ2', leave=False, file=sys.stdout)
        
        for ti in self.ti:
            
            # === read u,v,w
            
            dset = self['data/u']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    u_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                u_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read u', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/v']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    v_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                v_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read v', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/w']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    w_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                w_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # ===
            
            if self.usingmpi:
                x_ = np.copy(self.x[rx1:rx2])
                y_ = np.copy(self.y[ry1:ry2])
                z_ = np.copy(self.z[rz1:rz2])
            else:
                x_ = np.copy(self.x)
                y_ = np.copy(self.y)
                z_ = np.copy(self.z)
            
            # === ∂(u)/∂(x,y,z)
            
            ddx_u = gradient(u_, x_, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_u = gradient(u_, y_, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_u = gradient(u_, z_, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            # === ∂(v)/∂(x,y,z)
            
            ddx_v = gradient(v_, x_, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_v = gradient(v_, y_, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_v = gradient(v_, z_, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            # === ∂(w)/∂(x,y,z)
            
            ddx_w = gradient(w_, x_, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_w = gradient(w_, y_, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_w = gradient(w_, z_, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            ## free memory
            u_ = None; del u_
            v_ = None; del v_
            w_ = None; del w_
            gc.collect()
            
            strain = np.copy( np.stack((np.stack((ddx_u, ddy_u, ddz_u), axis=3),
                                        np.stack((ddx_v, ddy_v, ddz_v), axis=3),
                                        np.stack((ddx_w, ddy_w, ddz_w), axis=3)), axis=4) )
            
            t_delta = timeit.default_timer() - t_start
            if verbose: tqdm.write( even_print('get strain','%0.3f [s]'%(t_delta,), s=True) )
            
            ## free memory
            ddx_u = None; del ddx_u
            ddy_u = None; del ddy_u
            ddz_u = None; del ddz_u
            ddx_v = None; del ddx_v
            ddy_v = None; del ddy_v
            ddz_v = None; del ddz_v
            ddx_w = None; del ddx_w
            ddy_w = None; del ddy_w
            ddz_w = None; del ddz_w
            gc.collect()
            
            # === get the rate-of-strain & vorticity tensors
            
            S = np.copy( 0.5*(strain + np.transpose(strain, axes=(0,1,2,4,3))) ) ## strain rate tensor (symmetric)
            O = np.copy( 0.5*(strain - np.transpose(strain, axes=(0,1,2,4,3))) ) ## rotation rate tensor (anti-symmetric)
            # np.testing.assert_allclose(S+O, strain, atol=1.e-6)
            
            ## free memory
            strain = None; del strain
            gc.collect()
            
            # === Q : second invariant of characteristics equation: λ³ + Pλ² + Qλ + R = 0
            
            if save_Q:
                
                t_start = timeit.default_timer()
                
                O_norm  = np.linalg.norm(O, ord='fro', axis=(3,4))
                S_norm  = np.linalg.norm(S, ord='fro', axis=(3,4))
                Q       = 0.5*(O_norm**2 - S_norm**2)
                
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print('calc Q','%s'%format_time_string(t_delta), s=True))
                
                dset = self['data/Q']
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ti,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = Q[xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ti,:,:,:] = Q.T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = float_bytes * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print('write Q','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                # === second invariant : Q --> an equivalent formulation using eigenvalues (but much slower)
                if False:
                    Q_bak = np.copy(Q)
                    t_start = timeit.default_timer()
                    eigvals = np.linalg.eigvals(strain)
                    P       = -1*np.sum(eigvals, axis=-1) ## first invariant : P
                    SijSji  = np.einsum('xyzij,xyzji->xyz', S, S)
                    OijOji  = np.einsum('xyzij,xyzji->xyz', O, O)
                    Q       = 0.5*(P**2 - SijSji - OijOji)
                    t_delta = timeit.default_timer() - t_start
                    if verbose: tqdm.write(even_print('calc Q','%s'%format_time_string(t_delta), s=True))
                    np.testing.assert_allclose(Q.imag, np.zeros_like(Q.imag, dtype=np.float32), atol=1e-6)
                    Q = np.copy(Q.real)
                    np.testing.assert_allclose(Q, Q_bak, rtol=1e-2, atol=1e-5)
                
                ## free memory
                O_norm = None; del O_norm
                S_norm = None; del S_norm
                Q = None; del Q
                gc.collect()
            
            # === λ-2
            
            if save_lambda2:
                
                t_start = timeit.default_timer()
                
                # === S² and Ω²
                SikSkj = np.einsum('xyzik,xyzkj->xyzij', S, S)
                OikOkj = np.einsum('xyzik,xyzkj->xyzij', O, O)
                #np.testing.assert_allclose(np.matmul(S,S), SikSkj, atol=1e-6)
                #np.testing.assert_allclose(np.matmul(O,O), OikOkj, atol=1e-6)
                
                ## free memory
                S = None; del S
                O = None; del O
                gc.collect()
                
                # === Eigenvalues of (S²+Ω²) --> a real symmetric (Hermitian) matrix
                eigvals            = np.linalg.eigvalsh(SikSkj+OikOkj, UPLO='L')
                #eigvals_sort_order = np.argsort(np.abs(eigvals), axis=3) ## sort order of λ --> magnitude (wrong)
                eigvals_sort_order = np.argsort(eigvals, axis=3) ## sort order of λ
                eigvals_sorted     = np.take_along_axis(eigvals, eigvals_sort_order, axis=3) ## do λ sort
                lambda2            = np.squeeze(eigvals_sorted[:,:,:,1]) ## λ2 is the second eigenvalue (index=1)
                t_delta            = timeit.default_timer() - t_start
                
                if verbose: tqdm.write(even_print('calc λ2','%s'%format_time_string(t_delta), s=True))
                
                dset = self['data/lambda2']
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ti,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = lambda2[xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ti,:,:,:] = lambda2.T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = float_bytes * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print('write λ2','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## free memory
                lambda2 = None; del lambda2
                eigvals = None; del eigvals
                eigvals_sort_order = None; del eigvals_sort_order
                eigvals_sorted = None; del eigvals_sorted
                gc.collect()
            
            if verbose: progress_bar.update()
            if verbose and (ti<self.nt-1): tqdm.write( '---' )
        if verbose: progress_bar.close()
        if verbose: print(72*'-')
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        #if verbose: print('\n'+72*'-')
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.rgd.calc_lambda2() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # ===
    
    def calc_high_order_stats(self,**kwargs):
        '''
        calculate high-order statistical quantities
        - skewness
        - kurtosis / 'flatness'
        - probability distribution function (PDF)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        ## assert that the opened RGD has fsubtype 'unsteady'
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        if verbose: print('\n'+'rgd.calc_high_order_stats()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_dat_hos  = kwargs.get('fn_dat_hos',None) ## hos = 'high-order stats'
        fn_rgd_mean = kwargs.get('fn_rgd_mean',None)
        n_bins      = kwargs.get('n_bins',1024) ## n bins for histogram (PDF) calculation
        
        ## for now only distribute data in [y] --> allows [x,z] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # ===
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))

        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # === mean file name (for reading)
        if (fn_rgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_rgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_rgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_rgd_mean = Path(fname_path, fname_mean_h5_base)
        
        # # === mean (dimensional) file name (for reading) : .dat
        # if (fn_dat_mean_dim is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_dat_mean_base = fname_root+'_mean_dim.dat'
        #     fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
        
        # === high-order stats ('hos') file name (for writing) : dat
        if (fn_dat_hos is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
            fname_hos_dat_base = fname_root+'_hos.dat'
            fn_dat_hos = str(PurePosixPath(fname_path, fname_hos_dat_base))
        
        # ===
        
        if verbose: even_print( 'fn_rgd_mean'  , fn_rgd_mean )
        if verbose: even_print( 'fn_rgd'       , self.fname  )
        if verbose: even_print( 'fn_dat_hos'   , fn_dat_hos  )
        if verbose: print(72*'-')
        
        # if not os.path.isfile(fn_dat_mean_dim):
        #     raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        if False: ## open mean (dimensional) data : .dat
            
            with open(fn_dat_mean_dim,'rb') as f:
                data_mean_dim = pickle.load(f)
            fmd = type('foo', (object,), data_mean_dim)
            
            self.comm.Barrier()
            
            ## the data dictionary to be pickled later
            data = {}
            
            ## 2D dimensional quantities --> [x,z]
            u_tau    = fmd.u_tau    # ; data['u_tau']    = u_tau
            nu_wall  = fmd.nu_wall  # ; data['nu_wall']  = nu_wall
            rho_wall = fmd.rho_wall # ; data['rho_wall'] = rho_wall
            d99      = fmd.d99      # ; data['d99']      = d99
            u99      = fmd.u99      # ; data['u99']      = u99
            Re_tau   = fmd.Re_tau   # ; data['Re_tau']   = Re_tau
            Re_theta = fmd.Re_theta # ; data['Re_theta'] = Re_theta
            
            ## mean [x,z] --> leave 0D scalar
            u_tau_avg    = np.mean(fmd.u_tau    , axis=(0,1)) ; data['u_tau_avg']    = u_tau_avg
            nu_wall_avg  = np.mean(fmd.nu_wall  , axis=(0,1)) ; data['nu_wall_avg']  = nu_wall_avg
            rho_wall_avg = np.mean(fmd.rho_wall , axis=(0,1)) ; data['rho_wall_avg'] = rho_wall_avg
            d99_avg      = np.mean(fmd.d99      , axis=(0,1)) ; data['d99_avg']      = d99_avg
            u99_avg      = np.mean(fmd.u99      , axis=(0,1)) ; data['u99_avg']      = u99_avg
            Re_tau_avg   = np.mean(fmd.Re_tau   , axis=(0,1)) ; data['Re_tau_avg']   = Re_tau_avg
            Re_theta_avg = np.mean(fmd.Re_theta , axis=(0,1)) ; data['Re_theta_avg'] = Re_theta_avg
            
            ## mean [x,z] --> leave 1D [y]
            rho_avg = np.mean(fmd.rho,axis=(0,2))
            data['rho_avg'] = rho_avg
            
            # === 2D inner scales --> [x,z]
            sc_l_in = nu_wall / u_tau
            sc_u_in = u_tau
            sc_t_in = nu_wall / u_tau**2
            
            # === 2D outer scales --> [x,z]
            sc_l_out = d99
            sc_u_out = u99
            sc_t_out = d99/u99
            
            # === check
            np.testing.assert_allclose(fmd.lchar   , self.lchar   , rtol=1e-8)
            np.testing.assert_allclose(fmd.U_inf   , self.U_inf   , rtol=1e-8)
            np.testing.assert_allclose(fmd.rho_inf , self.rho_inf , rtol=1e-8)
            np.testing.assert_allclose(fmd.T_inf   , self.T_inf   , rtol=1e-8)
            np.testing.assert_allclose(fmd.nx      , self.nx      , rtol=1e-8)
            np.testing.assert_allclose(fmd.ny      , self.ny      , rtol=1e-8)
            np.testing.assert_allclose(fmd.nz      , self.nz      , rtol=1e-8)
            np.testing.assert_allclose(fmd.xs      , self.x       , rtol=1e-8)
            np.testing.assert_allclose(fmd.ys      , self.y       , rtol=1e-8)
            np.testing.assert_allclose(fmd.zs      , self.z       , rtol=1e-8)
            
            lchar   = self.lchar   ; data['lchar']   = lchar
            U_inf   = self.U_inf   ; data['U_inf']   = U_inf
            rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
            T_inf   = self.T_inf   ; data['T_inf']   = T_inf
            
            data['Ma'] = self.Ma
            data['Pr'] = self.Pr
            
            nx = self.nx ; data['nx'] = nx
            ny = self.ny ; data['ny'] = ny
            nz = self.nz ; data['nz'] = nz
            nt = self.nt ; data['nt'] = nt
            
            ## dimless (inlet)
            xd = self.x
            yd = self.y
            zd = self.z
            td = self.t
            
            ## dimensional [m] / [s]
            x      = self.x * lchar
            y      = self.y * lchar
            z      = self.z * lchar
            t      = self.t * (lchar/U_inf)
            t_meas = t[-1]-t[0]
            dt     = self.dt * (lchar/U_inf)
            
            data['x'] = x
            data['y'] = y
            data['z'] = z
            data['t'] = t
            data['t_meas'] = t_meas
            data['dt'] = dt
            
            np.testing.assert_equal(nx,x.size)
            np.testing.assert_equal(ny,y.size)
            np.testing.assert_equal(nz,z.size)
            np.testing.assert_equal(nt,t.size)
            np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-8)
            
            # === report
            if verbose:
                even_print('nx'     , '%i'        %nx     )
                even_print('ny'     , '%i'        %ny     )
                even_print('nz'     , '%i'        %nz     )
                even_print('nt'     , '%i'        %nt     )
                even_print('dt'     , '%0.5e [s]' %dt     )
                even_print('t_meas' , '%0.5e [s]' %t_meas )
                print(72*'-')
            
            if verbose:
                even_print('Re_τ'   , '%0.1f'         % Re_tau_avg   )
                even_print('Re_θ'   , '%0.1f'         % Re_theta_avg )
                even_print('δ99'    , '%0.5e [m]'     % d99_avg      )
                even_print('U_inf'  , '%0.3f [m/s]'   % U_inf        )
                even_print('u_τ'    , '%0.3f [m/s]'   % u_tau_avg    )
                even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall_avg  )
                even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall_avg  )
                print(72*'-')
            
            t_eddy = t_meas / ( d99_avg / u_tau_avg )
            
            if verbose:
                even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
                even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99_avg/u99_avg)))
                even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99_avg/u99_avg)))
                print(72*'-')
        
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        if True: ## open mean (dimensional) data : from mean .h5
            
            with rgd(fn_rgd_mean, 'r', driver=self.driver, comm=self.comm) as hf_mean:
                
                ## assert that the opened RGD has fsubtype 'mean'
                if (hf_mean.fsubtype!='mean'):
                    raise ValueError
                
                ## assert that 'data_dim' is present in mean file
                if ('data_dim' not in hf_mean):
                    raise ValueError(f'group data_dim not present in {fn_rgd_mean}')
                
                ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
                for dsn in hf_mean['data_dim'].keys():
                    d_ = np.copy( hf_mean[f'data_dim/{dsn}'][()] )
                    if (d_.ndim == 0):
                        d_ = float(d_)
                    data[dsn] = d_
                
                ## 1D
                rho_avg = np.copy( hf_mean['data_dim/rho'][()] )
                u_avg   = np.copy( hf_mean['data_dim/u'][()]   )
                
                ## 0D
                u_tau    = float( hf_mean['data_dim/u_tau'][()]    )
                nu_wall  = float( hf_mean['data_dim/nu_wall'][()]  )
                rho_wall = float( hf_mean['data_dim/rho_wall'][()] )
                d99      = float( hf_mean['data_dim/d99'][()]      )
                u_99     = float( hf_mean['data_dim/u_99'][()]     )
                Re_tau   = float( hf_mean['data_dim/Re_tau'][()]   )
                Re_theta = float( hf_mean['data_dim/Re_theta'][()] )
                sc_u_in  = float( hf_mean['data_dim/sc_u_in'][()]  )
                sc_l_in  = float( hf_mean['data_dim/sc_l_in'][()]  )
                sc_t_in  = float( hf_mean['data_dim/sc_t_in'][()]  )
                sc_u_out = float( hf_mean['data_dim/sc_u_out'][()] )
                sc_l_out = float( hf_mean['data_dim/sc_l_out'][()] )
                sc_t_out = float( hf_mean['data_dim/sc_t_out'][()] )
                
                ## these are recalculated and checked in next step
                z1d_ = np.copy( hf_mean['data_dim/z1d'][()] )
                dz0_ = np.copy( hf_mean['data_dim/dz0'][()] )
                dt_  = np.copy( hf_mean['data_dim/dt'][()]  )
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays, then re-dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        # ## dimless (inlet/characteristic)
        # xd = self.x
        # yd = self.y
        # zd = self.z
        # td = self.t
        
        ## redimensionalize coordinates --> [m],[s],[m/s],etc.
        x      = self.x * lchar
        y      = self.y * lchar
        z      = self.z * lchar
        t      = self.t * (lchar/U_inf)
        t_meas = t[-1]-t[0]
        dt     = self.dt * (lchar/U_inf)
        
        z1d = np.copy(z)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z1d.max() - z1d.min()
        #zrange = z[-1]-z[0]
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        #data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('dt'     , '%0.5e [s]' % dt      )
            even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print('Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print('zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print('Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        t_eddy = t_meas / ( d99 / u_tau )
        
        if verbose:
            even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
            even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
            even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
        
        # ===
        
        scalars = [ 'u', 'v', 'w', 'T' ]
        scalars_dtypes = [ self.scalars_dtypes_dict[s] for s in scalars ]
        n_scalars = len(scalars)
        
        ## dtype of unsteady data (currently must be all same dtype)
        if np.all( [ (dtp==np.float32) for dtp in scalars_dtypes ] ):
            dtype_unsteady = np.float32
            float_bytes = 4
        elif np.all( [ (dtp==np.float64) for dtp in scalars_dtypes ] ):
            dtype_unsteady = np.float64
            float_bytes = 8
        else:
            raise NotImplementedError('primes dont all have same dtype --> needs update')
        
        ## initialize buffers
        hos_scalars = [ f'{s}_mean'    for s in scalars ] + \
                      [ f'{s}I_median' for s in scalars ] + \
                      [ f'{s}_std'     for s in scalars ] + \
                      [ f'{s}_skew'    for s in scalars ] + \
                      [ f'{s}_kurt'    for s in scalars ]
        
        hos_scalars_dtypes = [ np.float64 for s in hos_scalars ]
        #hos                = np.zeros(shape=(self.nx, nyr, self.nz) , dtype={'names':hos_scalars, 'formats':hos_scalars_dtypes})
        hos                = np.zeros(shape=(nyr,) , dtype={'names':hos_scalars, 'formats':hos_scalars_dtypes})
        
        hist_scalars = [ '%s'%(s,) for s in scalars ]
        hist_scalars_dtypes = [ np.float64 for s in hist_scalars ]
        hist = np.zeros(shape=(nyr, n_bins) , dtype={'names':hist_scalars, 'formats':hist_scalars_dtypes})
        
        bins_scalars = [ '%s'%(s,) for s in scalars ]
        bins_scalars_dtypes = [ np.float64 for s in bins_scalars ]
        bins = np.zeros(shape=(nyr, n_bins+1) , dtype={'names':bins_scalars, 'formats':bins_scalars_dtypes})
        
        ## check memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose:
            even_print('mem total',     '%0.1f [GB]'%(mem_total_gb,))
            even_print('mem available', '%0.1f [GB] / %0.1f[%%]'%(mem_avail_gb,(100*mem_avail_gb/mem_total_gb)))
            even_print('mem free',      '%0.1f [GB] / %0.1f[%%]'%(mem_free_gb,(100*mem_free_gb/mem_total_gb)))
            print(72*'-')
        
        ## main loop
        self.comm.Barrier()
        if verbose: progress_bar = tqdm(total=n_scalars*nyr, ncols=100, desc='high order stats', leave=False, file=sys.stdout)
        
        for s in scalars:
            
            #data_unsteady = np.zeros(shape=(nx, nyr, nz, nt), dtype={'names':scalars, 'formats':scalars_dtypes})
            data_unsteady = np.zeros(shape=(nx, nyr, nz, nt), dtype=dtype_unsteady)
            
            ## read prime data
            dset = self[f'data/{s}']
            self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    data_unsteady[:,:,:,:] = np.copy( dset[:,:,ry1:ry2,:].T )
            else:
                data_unsteady[:,:,:,:] = np.copy( dset[()].T )
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * self.nt / 1024**3
            if verbose:
                tqdm.write(even_print(f'read: {s}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            ## redimensionalize prime data
            if s in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII']:
                data_unsteady *= U_inf
            elif s in ['r_uII','r_vII','r_wII']:
                data_unsteady *= (U_inf*rho_inf)
            elif s in ['T','TI','TII']:
                data_unsteady *= T_inf
            elif s in ['r_TII']:
                data_unsteady *= (T_inf*rho_inf)
            elif s in ['rho','rhoI']:
                data_unsteady *= rho_inf
            elif s in ['p','pI','pII']:
                data_unsteady *= (rho_inf * U_inf**2)
            else:
                raise ValueError('condition needed for redimensionalizing \'%s\''%var)
            
            for yi in range(nyr):
                
                ## all [x,z,t] at this [y]
                d        = np.copy( data_unsteady[:,yi,:,:] ).astype(np.float64).ravel()
                d_mean   = np.mean( d , dtype=np.float64 )
                dI       = np.copy( d - d_mean )
                
                dI_median = np.median( dI )
                
                hist_ , bin_edges_ = np.histogram( dI , bins=n_bins , density=True )
                hist[s][yi,:] = hist_
                bins[s][yi,:] = bin_edges_
                
                #d_var = np.mean( dI**2 ) ## = d.std()**2
                d_std = np.sqrt( np.mean( dI**2 , dtype=np.float64 ) ) ## = d.std()
                
                ## kurtosis can be checked against scipy implementation
                ## sp.stats.kurtosis(d,fisher=False)
                
                if np.isclose(d_std, 0., atol=1e-08):
                    d_skew = 0.
                    d_kurt = 0.
                else:
                    d_skew = np.mean( dI**3 , dtype=np.float64 ) / d_std**3
                    d_kurt = np.mean( dI**4 , dtype=np.float64 ) / d_std**4
                
                hos[f'{s}I_median'][yi] = dI_median
                hos[f'{s}_mean'][yi]    = d_mean
                hos[f'{s}_std'][yi]     = d_std
                hos[f'{s}_skew'][yi]    = d_skew
                hos[f'{s}_kurt'][yi]    = d_kurt
                
                if verbose: progress_bar.update()
        
        self.comm.Barrier()
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        # === average in [x,z], leave [y] --> not needed if stats are calculated over [x,z,t] per [y]
        
        ## hos_ = np.zeros(shape=(nyr,) , dtype={'names':hos_scalars, 'formats':hos_scalars_dtypes})
        ## for tag in hos_scalars:
        ##     hos_[tag] = np.mean( hos[tag] , axis=(0,2) , dtype=np.float64 )
        ## hos = np.copy( hos_ )
        ## self.comm.Barrier()
        
        # === gather
        
        G = self.comm.gather([ self.rank, hos, hist, bins ], root=0)
        G = self.comm.bcast(G, root=0)
        
        hos  = np.zeros( (ny,)         , dtype={'names':hos_scalars  , 'formats':hos_scalars_dtypes}  )
        hist = np.zeros( (ny,n_bins)   , dtype={'names':hist_scalars , 'formats':hist_scalars_dtypes} )
        bins = np.zeros( (ny,n_bins+1) , dtype={'names':bins_scalars , 'formats':bins_scalars_dtypes} )
        
        for ri in range(self.n_ranks):
            j = ri
            for GG in G:
                if (GG[0]==ri):
                    for tag in hos_scalars:
                        hos[tag][ryl[j][0]:ryl[j][1],] = GG[1][tag]
                    for tag in hist_scalars:
                        hist[tag][ryl[j][0]:ryl[j][1],:] = GG[2][tag]
                    for tag in bins_scalars:
                        bins[tag][ryl[j][0]:ryl[j][1],:] = GG[3][tag]
                else:
                    pass
        
        # === save results
        if (self.rank==0):
            
            data['hos']  = hos
            data['hist'] = hist
            data['bins'] = bins
            
            with open(fn_dat_hos,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_hos,os.path.getsize(fn_dat_hos)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_high_order_stats() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_acoustic_spectrum_time_xpln(self,**kwargs):
        '''
        calculate FFT in [t] (frequency) at every [x,y,z], avg in [x,z]
        - calculate power spectrum of p'
        - designed for analyzing unsteady, thin planes in [x]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        ## assert that the opened RGD has fsubtype 'prime'
        if (self.fsubtype!='prime'):
            raise ValueError
        
        if verbose: print('\n'+'rgd.calc_acoustic_spectrum_time_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        overlap_fac_nom = kwargs.get('overlap_fac_nom',0.50)
        n_win           = kwargs.get('n_win',10)
        
        fn_dat_fft      = kwargs.get('fn_dat_fft',None)
        fn_dat_mean_dim = kwargs.get('fn_dat_mean_dim',None)
        
        ## for now only distribute data in [y] --> allows [x,z] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # === distribute 4D data over ranks --> here only in [y]
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # # === mean (dimensional) file name (for reading) : .dat
        # if (fn_dat_mean_dim is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_dat_mean_base = fname_root+'_mean_dim.dat'
        #     fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
        
        # === fft file name (for writing) : dat
        if (fn_dat_fft is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_dat_base = fname_root+'_acoustic_spec_time.dat'
            fn_dat_fft = str(PurePosixPath(fname_path, fname_fft_dat_base))
        
        if verbose: even_print('fn_rgd_prime'    , self.fname       )
        #if verbose: even_print('fn_dat_mean_dim' , fn_dat_mean_dim  )
        if verbose: even_print('fn_dat_fft'      , fn_dat_fft       )
        if verbose: print(72*'-')
        
        #if not os.path.isfile(fn_dat_mean_dim):
        #    raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        # # === read in data (mean dim) --> every rank gets full [x,z]
        # with open(fn_dat_mean_dim,'rb') as f:
        #     data_mean_dim = pickle.load(f)
        # fmd = type('foo', (object,), data_mean_dim)
        
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('group data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho_avg = np.copy( self['data_dim/rho'][()] )
        u_avg   = np.copy( self['data_dim/u'][()]   )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        T_wall   = float( self['data_dim/T_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        u_99     = float( self['data_dim/u_99'][()]     )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## these are recalculated and checked in next step
        z1d_ = np.copy( self['data_dim/z1d'][()] )
        dz0_ = np.copy( self['data_dim/dz0'][()] )
        dt_  = np.copy( self['data_dim/dt'][()]  )
        
        # ===
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays, then re-dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        # ## dimless (inlet/characteristic)
        # xd = self.x
        # yd = self.y
        # zd = self.z
        # td = self.t
        
        ## redimensionalize coordinates --> [m],[s],[m/s],etc.
        x      = self.x * lchar
        y      = self.y * lchar
        z      = self.z * lchar
        t      = self.t * (lchar/U_inf)
        t_meas = t[-1]-t[0]
        dt     = self.dt * (lchar/U_inf)
        
        z1d = np.copy(z)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z[-1]-z[0]
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        #data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('dt'     , '%0.5e [s]' % dt      )
            even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        t_eddy = t_meas / ( d99 / u_tau )
        
        if verbose:
            even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
            even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
            even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
            print(72*'-')
        
        ## establish fft (time) windowing
        win_len, overlap = get_overlapping_window_size(nt, n_win, overlap_fac_nom)
        overlap_fac = overlap / win_len
        tw, n_win, n_pad = get_overlapping_windows(t, win_len, overlap)
        
        data['win_len']     = win_len
        data['overlap_fac'] = overlap_fac
        data['overlap']     = overlap
        data['n_win']       = n_win
        
        t_meas_per_win = (win_len-1)*dt
        t_eddy_per_win = t_meas_per_win / (d99/u_tau)
        
        data['t_eddy_per_win'] = t_eddy_per_win
        
        if verbose:
            even_print('overlap_fac (nominal)' , '%0.5f'%overlap_fac_nom    )
            even_print('n_win'                 , '%i'%n_win                 )
            even_print('win_len'               , '%i'%win_len               )
            even_print('overlap'               , '%i'%overlap               )
            even_print('overlap_fac'           , '%0.5f'%overlap_fac        )
            even_print('n_pad'                 , '%i'%n_pad                 )
            even_print('t_win/(δ99/u_τ)'       , '%0.3f [-]'%t_eddy_per_win )
            print(72*'-')
        
        # === get frequency vector --> here for short time FFT!
        freq_full = sp.fft.fftfreq(n=win_len, d=dt)
        fp        = np.where(freq_full>0)
        freq      = np.copy(freq_full[fp])
        df        = freq[1]-freq[0]
        nf        = freq.size
        
        data['freq'] = freq
        data['df']   = df
        data['nf']   = nf
        
        if verbose:
            even_print('freq min','%0.1f [Hz]'%freq.min())
            even_print('freq max','%0.1f [Hz]'%freq.max())
            even_print('df','%0.1f [Hz]'%df)
            even_print('nf','%i'%nf)
            
            period_eddy = (1/freq) / (d99/u_tau)
            period_plus = (1/freq) / sc_t_in
            even_print('min : period+ = (1/f)/(ν_wall/u_τ²)'   , '%0.5f [-]'%period_plus.min())
            even_print('max : period+ = (1/f)/(ν_wall/u_τ²)'   , '%0.5f [-]'%period_plus.max())
            even_print('min : period_eddy = (1/f)/(d99/u_tau)' , '%0.5e [-]'%period_eddy.min())
            even_print('max : period_eddy = (1/f)/(d99/u_tau)' , '%0.5f [-]'%period_eddy.max())
            
            print(72*'-')
        
        self.comm.Barrier()
        
        # === wavenumber kx, λx
        
        # λx = u/f
        # kx = 2·π·f/u
        # ---
        # λx/δ99
        # λx+ = λx/(ν/u_tau)
        # kx·δ99
        # kx+ = (2·π·f/u)·(ν/u_tau)
        
        na = np.newaxis
        
        ## prevent zeros since later we divide by wall u to get kx
        u_avg[0] = u_avg[1]*1e-6
        
        ## kx = 2·π·f/u --> [y,f]
        kx = (2*np.pi*freq[na,:]/u_avg[:,na])
        
        ## λx = u/f --> [y,f]
        lx = (u_avg[:,na]/freq[na,:])
        
        data['kx']  = kx
        data['lx']  = lx
        
        # === read in data (prime) --> still dimless (char)
        
        scalars = [ 'pI' ]
        #scalars_dtypes = [ self.scalars_dtypes_dict[s] for s in scalars ]
        scalars_dtypes = [ np.float64 for s in scalars ]
        
        dtype_primes = np.float64
        
        psd_scalars        = [ 'pI' ]
        psd_scalars_dtypes = [ dtype_primes for s in psd_scalars ]
        
        ## buffers
        psd_avg    = np.zeros(shape=(nyr, nf) , dtype={'names':psd_scalars, 'formats':psd_scalars_dtypes})
        pI_rms_avg = np.zeros(shape=(nyr,   ) , dtype={'names':psd_scalars, 'formats':psd_scalars_dtypes})
        
        ## check memory
        mem_total_gb = psutil.virtual_memory().total     / 1024**3
        mem_avail_gb = psutil.virtual_memory().available / 1024**3
        mem_free_gb  = psutil.virtual_memory().free      / 1024**3
        if verbose:
            even_print('mem total',     '%0.1f [GB]'%(mem_total_gb,))
            even_print('mem available', '%0.1f [GB] / %0.1f[%%]'%(mem_avail_gb,(100*mem_avail_gb/mem_total_gb)))
            even_print('mem free',      '%0.1f [GB] / %0.1f[%%]'%(mem_free_gb,(100*mem_free_gb/mem_total_gb)))
        
        ## the window function
        window_type = 'hamming'
        if (window_type=='tukey'):
            window = sp.signal.windows.tukey(win_len, alpha=overlap_fac_nom)
        elif (window_type=='hamming'):
            window = sp.signal.windows.hamming(win_len)
        elif (window_type is None):
            window = np.ones(win_len, dtype=np.float64)
        else:
            raise ValueError
        
        if verbose: even_print('window type', '\'%s\''%str(window_type))
        
        ## sum of sqrt of window: needed for power normalization
        sum_sqrt_win = np.sum(np.sqrt(window))
        
        if verbose: even_print('sum(sqrt(window)) / win_len', '%0.5f'%(sum_sqrt_win/win_len))
        
        if verbose:
            print(72*'-')
        
        ## main loop --> acoustic spectrum in [Δt] at every [x,y,z]
        self.comm.Barrier()
        if verbose:
            progress_bar = tqdm(total=1*nx*nyr*nz, ncols=100, desc='acoustic spectrum', leave=False, file=sys.stdout)
        
        #for cci,cc in enumerate(fft_combis):
        for scalar in scalars:
            
            #tag = 'pI'
            tag = scalar
            
            ## prime data buffer
            ## 5D [scalar][x,y,z,t] structured array
            data_prime = np.zeros(shape=(nx, nyr, nz, nt), dtype={'names':scalars, 'formats':scalars_dtypes})
            
            ## unsteady data buffers --> not the [x,z] averaged buffers
            psd    = np.zeros(shape=(self.nx, nyr, self.nz, nf), dtype=np.float64)
            pI_rms = np.zeros(shape=(self.nx, nyr, self.nz),     dtype=np.float64)
            
            ## read prime data
            for scalar in scalars:
                dset = self['data/%s'%scalar]
                self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        data_prime[scalar][:,:,:,:] = np.copy( dset[:,:,ry1:ry2,:].T ).astype(np.float64)
                else:
                    data_prime[scalar][:,:,:,:] = np.copy( dset[()].T ).astype(np.float64)
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
                if verbose:
                    tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            self.comm.Barrier()
            
            ## check memory
            mem_total_gb = psutil.virtual_memory().total     / 1024**3
            mem_avail_gb = psutil.virtual_memory().available / 1024**3
            mem_free_gb  = psutil.virtual_memory().free      / 1024**3
            if verbose:
                #tqdm.write( even_print( 'mem total'     , '%0.1f [GB]'%(mem_total_gb,) , s=True) )
                tqdm.write( even_print( 'mem available' , '%0.1f [GB] / %0.1f[%%]'%(mem_avail_gb,(100*mem_avail_gb/mem_total_gb)) , s=True) )
                tqdm.write( even_print( 'mem free'      , '%0.1f [GB] / %0.1f[%%]'%(mem_free_gb,(100*mem_free_gb/mem_total_gb)) , s=True) )
            
            self.comm.Barrier()
            
            ## redimensionalize prime data
            for var in data_prime.dtype.names:
                if var in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII']:
                    data_prime[var] *= U_inf
                elif var in ['r_uII','r_vII','r_wII']:
                    data_prime[var] *= (U_inf*rho_inf)
                elif var in ['T','TI','TII']:
                    data_prime[var] *= T_inf
                elif var in ['r_TII']:
                    data_prime[var] *= (T_inf*rho_inf)
                elif var in ['rho','rhoI']:
                    data_prime[var] *= rho_inf
                elif var in ['p','pI','pII']:
                    data_prime[var] *= (rho_inf * U_inf**2)
                else:
                    raise ValueError('condition needed for redimensionalizing \'%s\''%var)
            
            ## print names of components being cross-correlated
            if verbose:
                fancy_tag = '<%s>'%(scalar,)
                tqdm.write(even_print('running Δt fft calc', fancy_tag, s=True))
            
            for xi in range(nx):
                for yi in range(nyr):
                    for zi in range(nz):
                        
                        pI = np.copy( data_prime[scalar][xi,yi,zi,:] )
                        
                        pI_var_ijk = np.mean( pI*pI , dtype=np.float64 )
                        pI_rms_ijk = np.sqrt( pI_var_ijk )
                        
                        ## write to buffer
                        #pI_var[xi,yi,zi] = pI_var_ijk
                        pI_rms[xi,yi,zi] = pI_rms_ijk
                        
                        ## window time series into several overlapping windows
                        pI, nw, n_pad = get_overlapping_windows(pI, win_len, overlap)
                        
                        ## STFT buffer
                        psd_ijk = np.zeros((nw,nf), dtype=dtype_primes)
                        
                        ## do fft for each segment
                        for wi in range(nw):
                            
                            pI_wi   = np.copy( pI[wi,:] )
                            n       = pI_wi.size
                            pI_wi  *= window
                            
                            A_ui = sp.fft.fft(pI_wi)[fp] / sum_sqrt_win
                            A_uj = sp.fft.fft(pI_wi)[fp] / sum_sqrt_win
                            
                            psd_ijk_wi = 2 * np.real(A_ui*np.conj(A_uj)) / df
                            
                            # energy_norm_fac = np.sum(df*psd_ijk_wi) / np.mean( pI_wi*pI_wi , dtype=np.float64 )
                            # if verbose:
                            #     tqdm.write(f'{energy_norm_fac:0.14f}')
                            
                            psd_ijk[wi,:] = psd_ijk_wi
                        
                        ## mean across short time fft (STFT) segments
                        psd_ijk = np.mean(psd_ijk, axis=0, dtype=np.float64)
                        
                        ## write to buffer
                        psd[xi,yi,zi,:] = psd_ijk
                        
                        if verbose: progress_bar.update()
            
            ## average in [x,z]
            psd_avg[tag]    = np.mean( psd    , axis=(0,2) , dtype=np.float64)
            pI_rms_avg[tag] = np.mean( pI_rms , axis=(0,2) , dtype=np.float64)
            
            ## barrier per FFT scalar pair
            self.comm.Barrier()
        
        if verbose: progress_bar.close()
        
        # === gather all results
        # --> arrays are very small at this point, just do 'lazy' gather/bcast, dont worry about buffers etc
        
        G = self.comm.gather([ self.rank, np.copy(psd_avg), np.copy(pI_rms_avg) ], root=0)
        G = self.comm.bcast(G, root=0)
        
        psd_avg    = np.zeros( (ny,nf) , dtype={'names':psd_scalars, 'formats':psd_scalars_dtypes})
        pI_rms_avg = np.zeros( (ny,)   , dtype={'names':psd_scalars, 'formats':psd_scalars_dtypes})
        
        for ri in range(self.n_ranks):
            j = ri
            for GG in G:
                if (GG[0]==ri):
                    for tag in psd_scalars:
                        psd_avg[tag][ryl[j][0]:ryl[j][1],:] = GG[1][tag]
                        pI_rms_avg[tag][ryl[j][0]:ryl[j][1]]  = GG[2][tag]
                else:
                    pass
        if verbose: print(72*'-')
        
        # === save results
        if (self.rank==0):
            
            data['psd']    = psd_avg
            data['pI_rms'] = pI_rms_avg
            
            with open(fn_dat_fft,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_fft,os.path.getsize(fn_dat_fft)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_acoustic_spectrum_time_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_turb_spectrum_time(self, **kwargs):
        '''
        calculate FFT in [t] (frequency) at every [x,y,z], avg in [x,z]
        - designed for analyzing unsteady, thin planes in [x]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_turb_spectrum_time()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        overlap_fac_nom = kwargs.get('overlap_fac_nom',0.50)
        n_win           = kwargs.get('n_win',10)
        
        fn_dat_fft      = kwargs.get('fn_dat_fft',None)
        fn_dat_mean_dim = kwargs.get('fn_dat_mean_dim',None)
        
        ## for now only distribute data in [y] --> allows [x,z] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # === distribute 4D data over ranks --> here only in [y]
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # # === mean (dimensional) file name (for reading) : .dat
        # if (fn_dat_mean_dim is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_dat_mean_base = fname_root+'_mean_dim.dat'
        #     fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
        
        # === fft file name (for writing) : dat
        if (fn_dat_fft is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_dat_base = fname_root+'_turb_spec_time.dat'
            fn_dat_fft = str(PurePosixPath(fname_path, fname_fft_dat_base))
        
        if verbose: even_print('fn_rgd_prime'    , self.fname       )
        #if verbose: even_print('fn_dat_mean_dim' , fn_dat_mean_dim  )
        if verbose: even_print('fn_dat_fft'      , fn_dat_fft       )
        if verbose: print(72*'-')
        
        #if not os.path.isfile(fn_dat_mean_dim):
        #    raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        # # === read in data (mean dim) --> every rank gets full [x,z]
        # with open(fn_dat_mean_dim,'rb') as f:
        #     data_mean_dim = pickle.load(f)
        # fmd = type('foo', (object,), data_mean_dim)
        
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('group data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho_avg = np.copy( self['data_dim/rho'][()] )
        u_avg   = np.copy( self['data_dim/u'][()]   )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        u_99     = float( self['data_dim/u_99'][()]     )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## these are recalculated and checked in next step
        z1d_ = np.copy( self['data_dim/z1d'][()] )
        dz0_ = np.copy( self['data_dim/dz0'][()] )
        dt_  = np.copy( self['data_dim/dt'][()]  )
        
        # ===
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays, then re-dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        # ## dimless (inlet/characteristic)
        # xd = self.x
        # yd = self.y
        # zd = self.z
        # td = self.t
        
        ## redimensionalize coordinates --> [m],[s],[m/s],etc.
        x      = self.x * lchar
        y      = self.y * lchar
        z      = self.z * lchar
        t      = self.t * (lchar/U_inf)
        t_meas = t[-1]-t[0]
        dt     = self.dt * (lchar/U_inf)
        
        z1d = np.copy(z)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z[-1]-z[0]
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        #data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('dt'     , '%0.5e [s]' % dt      )
            even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        t_eddy = t_meas / ( d99 / u_tau )
        
        if verbose:
            even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
            even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
            even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
            print(72*'-')
        
        ## establish fft (time) windowing
        win_len, overlap = get_overlapping_window_size(nt, n_win, overlap_fac_nom)
        overlap_fac = overlap / win_len
        tw, n_win, n_pad = get_overlapping_windows(t, win_len, overlap)
        
        data['win_len']     = win_len
        data['overlap_fac'] = overlap_fac
        data['overlap']     = overlap
        data['n_win']       = n_win
        
        t_meas_per_win = (win_len-1)*dt
        t_eddy_per_win = t_meas_per_win / (d99/u_tau)
        
        data['t_eddy_per_win'] = t_eddy_per_win
        
        if verbose:
            even_print('overlap_fac (nominal)' , '%0.5f'%overlap_fac_nom    )
            even_print('n_win'                 , '%i'%n_win                 )
            even_print('win_len'               , '%i'%win_len               )
            even_print('overlap'               , '%i'%overlap               )
            even_print('overlap_fac'           , '%0.5f'%overlap_fac        )
            even_print('n_pad'                 , '%i'%n_pad                 )
            even_print('t_win/(δ99/u_τ)'       , '%0.3f [-]'%t_eddy_per_win )
            print(72*'-')
        
        # === get frequency vector --> here for short time FFT!
        freq_full = sp.fft.fftfreq(n=win_len, d=dt)
        fp        = np.where(freq_full>0)
        freq      = np.copy(freq_full[fp])
        df        = freq[1]-freq[0]
        nf        = freq.size
        
        data['freq'] = freq
        data['df']   = df
        data['nf']   = nf
        
        if verbose:
            even_print('freq min','%0.1f [Hz]'%freq.min())
            even_print('freq max','%0.1f [Hz]'%freq.max())
            even_print('df','%0.1f [Hz]'%df)
            even_print('nf','%i'%nf)
            
            period_eddy = (1/freq) / (d99/u_tau)
            period_plus = (1/freq) / sc_t_in
            even_print('min : period+ = (1/f)/(ν_wall/u_τ²)'   , '%0.5f [-]'%period_plus.min())
            even_print('max : period+ = (1/f)/(ν_wall/u_τ²)'   , '%0.5f [-]'%period_plus.max())
            even_print('min : period_eddy = (1/f)/(d99/u_tau)' , '%0.5e [-]'%period_eddy.min())
            even_print('max : period_eddy = (1/f)/(d99/u_tau)' , '%0.5f [-]'%period_eddy.max())
            
            print(72*'-')
        
        self.comm.Barrier()
        
        # === wavenumber kx, λx
        
        # λx = u/f
        # kx = 2·π·f/u
        # ---
        # λx/δ99
        # λx+ = λx/(ν/u_tau)
        # kx·δ99
        # kx+ = (2·π·f/u)·(ν/u_tau)
        
        na = np.newaxis
        
        ## prevent zeros since later we divide by wall u to get kx
        u_avg[0] = u_avg[1]*1e-6
        
        ## kx = 2·π·f/u --> [y,f]
        kx = (2*np.pi*freq[na,:]/u_avg[:,na])
        
        ## λx = u/f --> [y,f]
        lx = (u_avg[:,na]/freq[na,:])
        
        data['kx']  = kx
        data['lx']  = lx
        
        # === read in data (prime) --> still dimless (char)
        
        scalars = [ 'uI','vI','wI' , 'rho', 'uII','vII','wII' ]
        
        ## [var1, var2, density_scaling]
        fft_combis = [
                     [ 'uI'  , 'uI'  , False ],
                     [ 'vI'  , 'vI'  , False ],
                     [ 'wI'  , 'wI'  , False ],
                     [ 'uI'  , 'vI'  , False ],
                     [ 'uI'  , 'wI'  , False ],
                     [ 'vI'  , 'wI'  , False ],
                     [ 'uII' , 'uII' , True  ],
                     [ 'vII' , 'vII' , True  ],
                     [ 'wII' , 'wII' , True  ],
                     [ 'uII' , 'vII' , True  ],
                     [ 'uII' , 'wII' , True  ],
                     [ 'vII' , 'wII' , True  ],
                     ]
        
        if verbose:
            even_print('n turb spectrum (time) scalar combinations' , '%i'%(len(fft_combis),))
            print(72*'-')
        
        ## decide if mass density will be needed
        read_density = False
        for cc in fft_combis:
            scalar_L, scalar_R, density_scaling = cc
            if density_scaling:
                read_density = True
                break
        
        if verbose:
            even_print('read ρ', str(read_density))
        
        ## confirm 'rho' is not in locals
        if read_density and ('rho' in locals()):
            raise ValueError('rho alread in locals... check')
        
        if read_density:
            
            ## buffer for rho ( !! NOT rhoI !! ) --> this is read from 'prime' file
            rho = np.zeros(shape=(nx, nyr, nz, nt), dtype=np.float32)
            
            dset = self['data/rho']
            self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    rho[:,:,:,:] = dset[:,:,ry1:ry2,:].T
            else:
                rho[:,:,:,:] = dset[()].T
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
            if verbose:
                even_print( 'read: rho','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            ## re-dimensionalize rho
            rho *= rho_inf
        
        # ===
        
        scalars_dtypes = [ self.scalars_dtypes_dict[s] for s in scalars ]
        
        ## dtype of prime data (currently must be all same dtype)
        if np.all( [ (dtp==np.float32) for dtp in scalars_dtypes ] ):
            dtype_primes = np.float32
        elif np.all( [ (dtp==np.float64) for dtp in scalars_dtypes ] ):
            dtype_primes = np.float64
        else:
            raise NotImplementedError('primes dont all have same dtype --> needs update')
        
        ## force the ∫PSD == (co)variance
        ## this usually represents about a 1-2% power correction which comes about due to 
        ##   non-stationarity of windowed data
        normalize_psd_by_cov = False
        
        Euu_scalars        = [ '%s%s'%(cc[0],cc[1]) for cc in fft_combis ]
        Euu_scalars_dtypes = [ dtype_primes for s in Euu_scalars ]
        
        ## buffers
        Euu_avg  = np.zeros(shape=(nyr, nf) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg = np.zeros(shape=(nyr,   ) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        if normalize_psd_by_cov:
            energy_norm_fac_arr = np.zeros(shape=(self.nx, nyr, self.nz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes}) ## just for monitoring
        
        ## check memory
        mem_total_gb = psutil.virtual_memory().total     / 1024**3
        mem_avail_gb = psutil.virtual_memory().available / 1024**3
        mem_free_gb  = psutil.virtual_memory().free      / 1024**3
        if verbose:
            even_print('mem total',     '%0.1f [GB]'%(mem_total_gb,))
            even_print('mem available', '%0.1f [GB] / %0.1f[%%]'%(mem_avail_gb,(100*mem_avail_gb/mem_total_gb)))
            even_print('mem free',      '%0.1f [GB] / %0.1f[%%]'%(mem_free_gb,(100*mem_free_gb/mem_total_gb)))
        
        ## the window function
        window_type = 'tukey'
        if (window_type=='tukey'):
            window = sp.signal.windows.tukey(win_len, alpha=overlap_fac_nom)
        elif (window_type is None):
            window = np.ones(win_len, dtype=np.float64)
        if verbose:
            even_print('window type', '\'%s\''%str(window_type))
        
        ## sum of sqrt of window: needed for power normalization
        sum_sqrt_win = np.sum(np.sqrt(window))
        
        if verbose:
            even_print('sum(sqrt(window)) / win_len', '%0.5f'%(sum_sqrt_win/win_len))
        
        if verbose:
            print(72*'-')
        
        ## main loop --> turbulent spectrum in [Δt] at every [x,y,z]
        self.comm.Barrier()
        if verbose:
            progress_bar = tqdm(total=len(fft_combis)*nx*nyr*nz, ncols=100, desc='calc_turb_spectrum_time()', leave=False, file=sys.stdout)
        
        for cci,cc in enumerate(fft_combis):
            
            scalar_L, scalar_R, density_scaling = cc
            tag = '%s%s'%(scalar_L, scalar_R)
            
            ## check if autocorrelation, in which case don't read twice
            if (scalar_L==scalar_R):
                scalars = [ scalar_L ]
            else:
                scalars = [ scalar_L, scalar_R ]
            
            scalars_dtypes = [ self.scalars_dtypes_dict[s] for s in scalars ]
            
            ## prime data buffer
            ## 5D [scalar][x,y,z,t] structured array
            data_prime = np.zeros(shape=(nx, nyr, nz, nt), dtype={'names':scalars, 'formats':scalars_dtypes})
            
            ## unsteady data buffers
            Euu  = np.zeros(shape=(self.nx, nyr, self.nz, nf), dtype=np.float32)
            uIuI = np.zeros(shape=(self.nx, nyr, self.nz),     dtype=np.float32)
            
            ## read prime data
            for scalar in scalars:
                dset = self['data/%s'%scalar]
                self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        data_prime[scalar][:,:,:,:] = np.copy( dset[:,:,ry1:ry2,:].T )
                else:
                    data_prime[scalar][:,:,:,:] = np.copy( dset[()].T )
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
                if verbose:
                    tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            ## redimensionalize prime data
            for var in data_prime.dtype.names:
                if var in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII']:
                    data_prime[var] *= U_inf
                elif var in ['r_uII','r_vII','r_wII']:
                    data_prime[var] *= (U_inf*rho_inf)
                elif var in ['T','TI','TII']:
                    data_prime[var] *= T_inf
                elif var in ['r_TII']:
                    data_prime[var] *= (T_inf*rho_inf)
                elif var in ['rho','rhoI']:
                    data_prime[var] *= rho_inf
                elif var in ['p','pI','pII']:
                    data_prime[var] *= (rho_inf * U_inf**2)
                else:
                    raise ValueError('condition needed for redimensionalizing \'%s\''%var)
            
            ## print names of components being cross-correlated
            if verbose:
                if density_scaling:
                    fancy_tag = '<ρ·%s,ρ·%s>'%(scalar_L,scalar_R)
                else:
                    fancy_tag = '<%s,%s>'%(scalar_L,scalar_R)
                tqdm.write(even_print('running Δt fft calc', fancy_tag, s=True))
            
            for xi in range(nx):
                for yi in range(nyr):
                    for zi in range(nz):
                        
                        uL = np.copy( data_prime[scalar_L][xi,yi,zi,:] )
                        uR = np.copy( data_prime[scalar_R][xi,yi,zi,:] )
                        
                        if density_scaling:
                            rho1d = np.copy( rho[xi,yi,zi,:] )
                            rho_avg = np.mean( rho1d, dtype=np.float64 )
                        else:
                            rho1d   = None
                            rho_avg = None
                        
                        # ===
                        
                        if density_scaling:
                            uIuI_ijk = np.mean(uL*uR*rho1d, dtype=np.float64) / rho_avg
                        else:
                            uIuI_ijk = np.mean(uL*uR, dtype=np.float64)
                        
                        ## write to buffer
                        uIuI[xi,yi,zi] = uIuI_ijk
                        
                        ## window time series into several overlapping windows
                        uL, nw, n_pad = get_overlapping_windows(uL, win_len, overlap)
                        uR, nw, n_pad = get_overlapping_windows(uR, win_len, overlap)
                        if density_scaling:
                            rho1d, _, _ = get_overlapping_windows(rho1d, win_len, overlap)
                        else:
                            rho1d = None
                        
                        ## STFT buffer
                        Euu_ijk = np.zeros((nw,nf), dtype=dtype_primes)
                        
                        ## do fft for each segment
                        for wi in range(nw):
                            if density_scaling:
                                ui = np.copy( uL[wi,:] * rho1d[wi,:] )
                                uj = np.copy( uR[wi,:] * rho1d[wi,:] )
                            else:
                                ui = np.copy( uL[wi,:] )
                                uj = np.copy( uR[wi,:] )
                            n     = ui.size
                            #A_ui = sp.fft.fft(ui)[fp] / n
                            #A_uj = sp.fft.fft(uj)[fp] / n
                            ui   *= window
                            uj   *= window
                            #ui  -= np.mean(ui) ## de-trend
                            #uj  -= np.mean(uj)
                            A_ui          = sp.fft.fft(ui)[fp] / sum_sqrt_win
                            A_uj          = sp.fft.fft(uj)[fp] / sum_sqrt_win
                            Euu_ijk[wi,:] = 2 * np.real(A_ui*np.conj(A_uj)) / df
                        
                        ## mean across short time fft (STFT) segments
                        Euu_ijk = np.mean(Euu_ijk, axis=0, dtype=np.float64)
                        
                        ## divide off mean mass density
                        if density_scaling:
                            Euu_ijk /= rho_avg**2
                        
                        ## normalize such that ∫PSD=(co)variance
                        if normalize_psd_by_cov:
                            if (uIuI_ijk!=0.):
                                energy_norm_fac = np.sum(df*Euu_ijk) / uIuI_ijk
                            else:
                                energy_norm_fac = 1.
                            Euu_ijk /= energy_norm_fac
                            energy_norm_fac_arr[xi,yi,zi] = energy_norm_fac
                        
                        ## write to buffer
                        Euu[xi,yi,zi,:] = Euu_ijk
                        
                        if verbose: progress_bar.update()
            
            ## average in [x,z]
            Euu_avg[tag]  = np.mean( Euu  , axis=(0,2) , dtype=np.float64)
            uIuI_avg[tag] = np.mean( uIuI , axis=(0,2) , dtype=np.float64)
            
            ## barrier per FFT scalar pair
            self.comm.Barrier()
        
        if verbose: progress_bar.close()
        
        # === gather all results
        # --> arrays are very small at this point, just do 'lazy' gather/bcast, dont worry about buffers etc
        
        G = self.comm.gather([ self.rank, np.copy(Euu_avg), np.copy(uIuI_avg) ], root=0)
        G = self.comm.bcast(G, root=0)
        
        Euu_avg  = np.zeros( (ny,nf) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg = np.zeros( (ny,)   , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        
        for ri in range(self.n_ranks):
            j = ri
            for GG in G:
                if (GG[0]==ri):
                    for tag in Euu_scalars:
                        Euu_avg[tag][ryl[j][0]:ryl[j][1],:] = GG[1][tag]
                        uIuI_avg[tag][ryl[j][0]:ryl[j][1]]  = GG[2][tag]
                else:
                    pass
        if verbose: print(72*'-')
        
        # === save results
        if (self.rank==0):
            
            data['Euu']  = Euu_avg
            data['uIuI'] = uIuI_avg
            
            with open(fn_dat_fft,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_fft,os.path.getsize(fn_dat_fft)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_turb_spectrum_time() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_turb_spectrum_span(self, **kwargs):
        '''
        calculate FFT in [z] (wavenumber) at every [x,y,t], avg in [x,t]
        - designed for analyzing unsteady, thin planes in [x]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_turb_spectrum_span()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_dat_fft      = kwargs.get('fn_dat_fft',None)
        fn_dat_mean_dim = kwargs.get('fn_dat_mean_dim',None)
        
        ## for now only distribute data in [y] --> allows [x,t] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # === distribute 4D data over ranks --> here only in [y]
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # # === mean (dimensional) file name (for reading) : .dat
        # if (fn_dat_mean_dim is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_dat_mean_base = fname_root+'_mean_dim.dat'
        #     fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
        
        # === fft file name (for writing) : dat
        if (fn_dat_fft is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_dat_base = fname_root+'_turb_spec_span.dat'
            fn_dat_fft = str(PurePosixPath(fname_path, fname_fft_dat_base))
        
        if verbose: even_print('fn_rgd_prime'    , self.fname       )
        #if verbose: even_print('fn_dat_mean_dim' , fn_dat_mean_dim  )
        if verbose: even_print('fn_dat_fft'      , fn_dat_fft       )
        if verbose: print(72*'-')
        
        #if not os.path.isfile(fn_dat_mean_dim):
        #    raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        # # === read in data (mean dim) --> every rank gets full [x,z]
        # with open(fn_dat_mean_dim,'rb') as f:
        #     data_mean_dim = pickle.load(f)
        # fmd = type('foo', (object,), data_mean_dim)
        
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('group data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho_avg = np.copy( self['data_dim/rho'][()] )
        u_avg   = np.copy( self['data_dim/u'][()]   )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        u_99     = float( self['data_dim/u_99'][()]     )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## these are recalculated and checked in next step
        z1d_ = np.copy( self['data_dim/z1d'][()] )
        dz0_ = np.copy( self['data_dim/dz0'][()] )
        dt_  = np.copy( self['data_dim/dt'][()]  )
        
        # ===
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays, then re-dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        # ## dimless (inlet/characteristic)
        # xd = self.x
        # yd = self.y
        # zd = self.z
        # td = self.t
        
        ## redimensionalize coordinates --> [m],[s],[m/s],etc.
        x      = self.x * lchar
        y      = self.y * lchar
        z      = self.z * lchar
        t      = self.t * (lchar/U_inf)
        t_meas = t[-1]-t[0]
        dt     = self.dt * (lchar/U_inf)
        
        z1d = np.copy(z)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z1d.max() - z1d.min()
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        #data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('dt'     , '%0.5e [s]' % dt      )
            even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        # t_eddy = t_meas / ( d99_avg / u_tau_avg )
        # 
        # if verbose:
        #     even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
        #     even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99_avg/u99_avg)))
        #     even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99_avg/u99_avg)))
        #     print(72*'-')
        
        ## get wavenumber vector
        kz_full = sp.fft.fftfreq(n=nz, d=dz0) * ( 2 * np.pi )
        kzp     = np.where(kz_full>0)
        kz      = np.copy(kz_full[kzp])
        dkz     = kz[1]-kz[0]
        nkz     = kz.size
        
        data['kz']  = kz
        data['dkz'] = dkz
        data['nkz'] = nkz
        
        if verbose:
            even_print('kz min','%0.1f [1/m]'%kz.min())
            even_print('kz max','%0.1f [1/m]'%kz.max())
            even_print('dkz','%0.1f [1/m]'%dkz)
            even_print('nkz','%i'%nkz)
            
            kz_inner = np.copy( kz * sc_l_in  )
            kz_outer = np.copy( kz * sc_l_out )
            
            even_print('(kz·δ_ν) min' , '%0.5e [-]'%kz_inner.min())
            even_print('(kz·δ_ν) max' , '%0.5f [-]'%kz_inner.max())
            even_print('(kz·δ99) min' , '%0.5f [-]'%kz_outer.min())
            even_print('(kz·δ99) max' , '%0.5e [-]'%kz_outer.max())
            
            print(72*'-')
        
        # === read in data (prime) --> still dimless (char)
        
        scalars = [ 'uI','vI','wI' , 'rho', 'uII','vII','wII' ]
        scalars_dtypes = [ self.scalars_dtypes_dict[s] for s in scalars ]
        
        ## [var1, var2, density_scaling]
        fft_combis = [
                     [ 'uI'  , 'uI'  , False ],
                     [ 'vI'  , 'vI'  , False ],
                     [ 'wI'  , 'wI'  , False ],
                     [ 'uI'  , 'vI'  , False ],
                     [ 'uI'  , 'wI'  , False ],
                     [ 'vI'  , 'wI'  , False ],
                     [ 'uII' , 'uII' , True  ],
                     [ 'vII' , 'vII' , True  ],
                     [ 'wII' , 'wII' , True  ],
                     [ 'uII' , 'vII' , True  ],
                     [ 'uII' , 'wII' , True  ],
                     [ 'vII' , 'wII' , True  ],
                     ]
        
        if verbose:
            even_print('n turb spectrum (span) scalar combinations' , '%i'%(len(fft_combis),))
            print(72*'-')
        
        ## decide if mass density will be needed
        read_density = False
        for cc in fft_combis:
            scalar_L, scalar_R, density_scaling = cc
            if density_scaling:
                read_density = True
                break
        
        if verbose:
            even_print('read ρ', str(read_density))
        
        ## confirm 'rho' is not in locals
        if read_density and ('rho' in locals()):
            raise ValueError('rho alread in locals... check')
        
        if read_density:
            
            ## buffer for rho ( !! NOT rhoI !! ) --> this is read from 'prime' file
            rho = np.zeros(shape=(nx, nyr, nz, nt), dtype=np.float32)
            
            dset = self['data/rho']
            self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    rho[:,:,:,:] = np.copy( dset[:,:,ry1:ry2,:].T )
            else:
                rho[:,:,:,:] = np.copy( dset[()].T )
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
            if verbose:
                even_print( 'read: rho','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            ## re-dimensionalize rho
            rho *= rho_inf
        
        # ===
        
        ## dtype of prime data (currently must be all same dtype)
        if np.all( [ (dtp==np.float32) for dtp in scalars_dtypes ] ):
            dtype_primes = np.float32
        elif np.all( [ (dtp==np.float64) for dtp in scalars_dtypes ] ):
            dtype_primes = np.float64
        else:
            raise NotImplementedError
        
        ## force the ∫PSD == (co)variance
        ## this usually represents about a 1-2% power correction which comes about due to 
        ##   non-stationarity of windowed data
        normalize_psd_by_cov = False
        
        Euu_scalars        = [ '%s%s'%(cc[0],cc[1]) for cc in fft_combis ]
        Euu_scalars_dtypes = [ dtype_primes for s in Euu_scalars ]
        
        ## buffers (for averaged data)
        Euu_avg  = np.zeros(shape=(nyr, nkz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg = np.zeros(shape=(nyr,    ) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        if normalize_psd_by_cov:
            energy_norm_fac_arr = np.zeros(shape=(self.nx, nyr, self.nz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes}) ## just for monitoring
        
        ## check memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose:
            even_print('mem total',     '%0.1f [GB]'%(mem_total_gb,))
            even_print('mem available', '%0.1f [GB] / %0.1f[%%]'%(mem_avail_gb,(100*mem_avail_gb/mem_total_gb)))
            even_print('mem free',      '%0.1f [GB] / %0.1f[%%]'%(mem_free_gb,(100*mem_free_gb/mem_total_gb)))
        
        ## for spanwise mean, no overlapping windows are applied, so win_len = [z] vec length
        win_len = nz
        
        ## the window function
        #window_type = 'tukey'
        window_type = None
        if (window_type=='tukey'):
            window = sp.signal.windows.tukey(win_len, alpha=0.1)
        elif (window_type is None):
            window = np.ones(win_len, dtype=np.float64)
        if verbose:
            even_print('window type', '\'%s\''%str(window_type))
        
        ## sum of sqrt of window: needed for power normalization
        sum_sqrt_win = np.sum(np.sqrt(window))
        
        if verbose:
            even_print('sum(sqrt(window))'          , '%0.5f'%(sum_sqrt_win,))
            even_print('sum(sqrt(window)) / win_len', '%0.5f'%(sum_sqrt_win/win_len,))
        
        if verbose:
            print(72*'-')
        
        # === main loop
        
        ## main loop --> turbulent spectrum in [Δz] at every [x,y,t]
        self.comm.Barrier()
        if verbose:
            progress_bar = tqdm(total=len(fft_combis)*nx*nyr*nt, ncols=100, desc='calc_turb_spectrum_span()', leave=False, file=sys.stdout)
        
        for cci,cc in enumerate(fft_combis):
            
            scalar_L, scalar_R, density_scaling = cc
            tag = '%s%s'%(scalar_L, scalar_R)
            
            ## check if autocorrelation, in which case don't read twice
            if (scalar_L==scalar_R):
                scalars = [ scalar_L ]
            else:
                scalars = [ scalar_L, scalar_R ]
            
            scalars_dtypes = [ self.scalars_dtypes_dict[s] for s in scalars ]
            
            ## prime data buffer
            ## 5D [scalar][x,y,z,t] structured array
            data_prime = np.zeros(shape=(nx, nyr, nz, nt), dtype={'names':scalars, 'formats':scalars_dtypes})
            
            ## unsteady data buffers
            Euu  = np.zeros(shape=(self.nx, nyr, self.nt, nkz), dtype=np.float32)
            uIuI = np.zeros(shape=(self.nx, nyr, self.nt),      dtype=np.float32)
            
            ## read prime data
            for scalar in scalars:
                dset = self['data/%s'%scalar]
                self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        data_prime[scalar][:,:,:,:] = np.copy( dset[:,:,ry1:ry2,:].T )
                else:
                    data_prime[scalar][:,:,:,:] = np.copy( dset[()].T )
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
                if verbose:
                    tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            ## redimensionalize prime data
            for var in data_prime.dtype.names:
                if var in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII']:
                    data_prime[var] *= U_inf
                elif var in ['r_uII','r_vII','r_wII']:
                    data_prime[var] *= (U_inf*rho_inf)
                elif var in ['T','TI','TII']:
                    data_prime[var] *= T_inf
                elif var in ['r_TII']:
                    data_prime[var] *= (T_inf*rho_inf)
                elif var in ['rho','rhoI']:
                    data_prime[var] *= rho_inf
                elif var in ['p','pI','pII']:
                    data_prime[var] *= (rho_inf * U_inf**2)
                else:
                    raise ValueError('condition needed for redimensionalizing \'%s\''%var)
            
            ## print names of components being cross-correlated
            if verbose:
                if density_scaling:
                    fancy_tag = '<ρ·%s,ρ·%s>'%(scalar_L,scalar_R)
                else:
                    fancy_tag = '<%s,%s>'%(scalar_L,scalar_R)
                tqdm.write(even_print('running Δz fft calc', fancy_tag, s=True))
            
            for xi in range(self.nx):
                for yi in range(nyr):
                    for ti in range(self.nt):
                        
                        uL = np.copy( data_prime[scalar_L][xi,yi,:,ti] )
                        uR = np.copy( data_prime[scalar_R][xi,yi,:,ti] )
                        
                        if density_scaling:
                            #rho1d = np.copy( rho[xi,yi,zi,:] )
                            rho1d = np.copy( rho[xi,yi,:,ti] )
                            rho_avg = np.mean( rho1d, dtype=np.float64 )
                        else:
                            rho1d   = None
                            rho_avg = None
                        
                        # ===
                        
                        if density_scaling:
                            uIuI_ijk = np.mean(uL*uR*rho1d, dtype=np.float64) / rho_avg
                        else:
                            uIuI_ijk = np.mean(uL*uR, dtype=np.float64)
                        
                        ## write to buffer
                        uIuI[xi,yi,ti] = uIuI_ijk
                        
                        if density_scaling:
                            ui = np.copy( uL * rho1d )
                            uj = np.copy( uR * rho1d )
                        else:
                            ui = np.copy( uL )
                            uj = np.copy( uR )
                        
                        n     = ui.size
                        #A_ui = sp.fft.fft(ui)[fp] / n
                        #A_uj = sp.fft.fft(uj)[fp] / n
                        ui   *= window
                        uj   *= window
                        #ui  -= np.mean(ui) ## de-trend
                        #uj  -= np.mean(uj)
                        A_ui    = sp.fft.fft(ui)[kzp] / sum_sqrt_win
                        A_uj    = sp.fft.fft(uj)[kzp] / sum_sqrt_win
                        Euu_ijk = 2. * np.real(A_ui*np.conj(A_uj)) / dkz
                        
                        ## divide off mean mass density
                        if density_scaling:
                            Euu_ijk /= rho_avg**2
                        
                        ## normalize such that ∫PSD=(co)variance
                        if normalize_psd_by_cov:
                            if (uIuI_ijk!=0.):
                                energy_norm_fac = np.sum(dkz*Euu_ijk) / uIuI_ijk
                            else:
                                energy_norm_fac = 1.
                            Euu_ijk /= energy_norm_fac
                            energy_norm_fac_arr[xi,yi,ti] = energy_norm_fac
                        
                        ## write to buffer
                        Euu[xi,yi,ti,:] = Euu_ijk
                        
                        if verbose: progress_bar.update()
            
            ## average in [x,t] --> leave [y,kz]
            Euu_avg[tag]  = np.mean( Euu  , axis=(0,2) , dtype=np.float64)
            uIuI_avg[tag] = np.mean( uIuI , axis=(0,2) , dtype=np.float64)
            
            ## barrier per FFT scalar pair
            self.comm.Barrier()
        
        if verbose: progress_bar.close()
        
        # === gather all results
        # --> arrays are very small at this point, just do 'lazy' gather/bcast, dont worry about buffers etc
        
        G = self.comm.gather([ self.rank, np.copy(Euu_avg), np.copy(uIuI_avg) ], root=0)
        G = self.comm.bcast(G, root=0)
        
        Euu_avg  = np.zeros( (ny,nkz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg = np.zeros( (ny,)    , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        
        for ri in range(self.n_ranks):
            j = ri
            for GG in G:
                if (GG[0]==ri):
                    for tag in Euu_scalars:
                        Euu_avg[tag][ryl[j][0]:ryl[j][1],:] = GG[1][tag]
                        uIuI_avg[tag][ryl[j][0]:ryl[j][1]]  = GG[2][tag]
                else:
                    pass
        if verbose: print(72*'-')
        
        # === save results
        if (self.rank==0):
            
            data['Euu']      = Euu_avg
            data['uIuI_avg'] = uIuI_avg
            
            with open(fn_dat_fft,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_fft,os.path.getsize(fn_dat_fft)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_turb_spectrum_span() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_turb_spectrum_2d_span_time(self, **kwargs):
        '''
        calculate 2D FFT in [z,t] at every [x,y], avg in [x]
        - designed for analyzing unsteady, thin planes in [x]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_turb_spectrum_2d_span_time()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if (self.fsubtype!='prime'):
            raise ValueError
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        overlap_fac_nom = kwargs.get('overlap_fac_nom',0.50)
        n_win           = kwargs.get('n_win',10)
        fn_dat_fft      = kwargs.get('fn_dat_fft',None)
        
        ## only distribute data in [y]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # === distribute 4D data over ranks --> here only in [y]
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # === fft file name (for writing) : dat
        if (fn_dat_fft is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_dat_base = fname_root+'_turb_spec_2d_zt.dat'
            fn_dat_fft = str(PurePosixPath(fname_path, fname_fft_dat_base))
        
        if verbose: even_print('fn_rgd_prime' , self.fname )
        if verbose: even_print('fn_dat_fft'   , fn_dat_fft )
        if verbose: print(72*'-')
        
        # ===
        
        if verbose: even_print('nx' , '%i'%self.nx)
        if verbose: even_print('ny' , '%i'%self.ny)
        if verbose: even_print('nz' , '%i'%self.nz)
        if verbose: even_print('nt' , '%i'%self.nt)
        if verbose: print(72*'-')
        
        if self.usingmpi: self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('group data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho_avg = np.copy( self['data_dim/rho'][()] )
        u_avg   = np.copy( self['data_dim/u'][()]   )
        #v_avg   = np.copy( self['data_dim/v'][()]   )
        #w_avg   = np.copy( self['data_dim/w'][()]   )
        #T_avg   = np.copy( self['data_dim/T'][()]   )
        #p_avg   = np.copy( self['data_dim/p'][()]   )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        T_wall   = float( self['data_dim/T_wall'][()]   )
        d99      = float( self['data_dim/d99'][()]      )
        u_99     = float( self['data_dim/u_99'][()]     )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays, then re-dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        t_eddy = t_meas / ( d99 / u_tau )
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-7)):
            raise NotImplementedError
        np.testing.assert_allclose(dz0, z[1]-z[0], rtol=1e-14, atol=1e-14)
        
        zrange = z.max() - z.min()
        np.testing.assert_allclose(zrange, z[-1]-z[0], rtol=1e-14, atol=1e-14)
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        ## add data to dict that gets output
        data['x'] = x
        data['y'] = y
        data['z'] = z
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('dt'     , '%0.5e [s]' % dt      )
            even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
            even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
            even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
            print(72*'-')
        
        # ===
        
        ## establish fft (time) windowing
        win_len, overlap = get_overlapping_window_size(nt, n_win, overlap_fac_nom)
        overlap_fac = overlap / win_len
        tw, n_win, n_pad = get_overlapping_windows(t, win_len, overlap)
        
        data['win_len']     = win_len
        data['overlap_fac'] = overlap_fac
        data['overlap']     = overlap
        data['n_win']       = n_win
        
        t_meas_per_win = (win_len-1)*dt
        t_eddy_per_win = t_meas_per_win / (d99/u_tau)
        
        data['t_eddy_per_win'] = t_eddy_per_win
        
        if verbose:
            even_print('overlap_fac (nominal)' , '%0.5f'%overlap_fac_nom    )
            even_print('n_win'                 , '%i'%n_win                 )
            even_print('win_len'               , '%i'%win_len               )
            even_print('overlap'               , '%i'%overlap               )
            even_print('overlap_fac'           , '%0.5f'%overlap_fac        )
            even_print('n_pad'                 , '%i'%n_pad                 )
            even_print('t_win/(δ99/u_τ)'       , '%0.3f [-]'%t_eddy_per_win )
            print(72*'-')
        
        ## get frequency vector --> here for short time FFT, i.e. for [t] signal broken into overlapping segments
        freq_full = sp.fft.fftfreq(n=win_len, d=dt)
        fp        = np.where(freq_full>0)
        freq      = np.copy(freq_full[fp])
        df        = freq[1]-freq[0]
        nf        = freq.size
        
        data['freq'] = freq
        data['df']   = df
        data['nf']   = nf
        data['fp']   = fp ## tmp
        data['freq_full'] = freq_full ## tmp
        
        # λx = u/f
        # kx = 2·π·f/u
        # ---
        # λx/δ99
        # λx+ = λx/(ν/u_tau)
        # kx·δ99
        # kx+ = (2·π·f/u)·(ν/u_tau)
        
        ## get wavenumber vector/array [x]
        
        na = np.newaxis
        
        ## prevent zeros since later we divide by wall u to get kx
        u_avg[0] = u_avg[1]*1e-6
        
        ## kx = 2·π·f/u --> shape=(ny,nf)
        kx = np.copy( 2*np.pi*freq[na,:]/u_avg[:,na] )
        kx_full = np.copy( 2*np.pi*freq_full[na,:]/u_avg[:,na] ) ## tmp
        
        ## λx = u/f --> shape=(ny,nf)
        lx = np.copy( u_avg[:,na]/freq[na,:] )
        lx_full = np.copy( u_avg[:,na]/freq_full[na,:] ) ## tmp
        
        data['kx']  = kx
        data['kx_full'] = kx_full ## tmp
        data['lx'] = lx
        data['lx_full'] = lx_full ## tmp
        
        ## get wavenumber vector [z]
        kz_full = sp.fft.fftfreq(n=nz, d=dz0) * ( 2 * np.pi )
        kzp     = np.where(kz_full>0)
        kz      = np.copy(kz_full[kzp])
        dkz     = kz[1]-kz[0]
        nkz     = kz.size
        
        data['kz']  = kz
        data['dkz'] = dkz
        data['nkz'] = nkz
        data['kzp'] = kzp ## tmp
        data['kz_full'] = kz_full ## tmp
        
        ## λz = (2·π)/kz
        lz = np.copy( 2 * np.pi / kz )
        lz_full = np.copy( 2 * np.pi / kz_full ) ## tmp
        data['lz'] = lz
        data['lz_full'] = lz_full ## tmp
        
        if verbose:
            even_print('freq min','%0.1f [Hz]'%freq.min())
            even_print('freq max','%0.1f [Hz]'%freq.max())
            even_print('df','%0.1f [Hz]'%df)
            even_print('nf','%i'%nf)
            
            period_eddy = (1/freq) / (d99/u_tau)
            period_plus = (1/freq) / sc_t_in
            even_print('min : period+ = (1/f)/(ν_wall/u_τ²)'   , '%0.5f [-]'%period_plus.min())
            even_print('max : period+ = (1/f)/(ν_wall/u_τ²)'   , '%0.5f [-]'%period_plus.max())
            even_print('min : period_eddy = (1/f)/(d99/u_tau)' , '%0.5e [-]'%period_eddy.min())
            even_print('max : period_eddy = (1/f)/(d99/u_tau)' , '%0.5f [-]'%period_eddy.max())
            
            print(72*'-')
        
        if verbose:
            even_print('kz min','%0.1f [1/m]'%kz.min())
            even_print('kz max','%0.1f [1/m]'%kz.max())
            even_print('dkz','%0.1f [1/m]'%dkz)
            even_print('nkz','%i'%nkz)
            
            kz_inner = np.copy( kz * sc_l_in  )
            kz_outer = np.copy( kz * sc_l_out )
            
            even_print('min : (kz·δ_ν) ' , '%0.5e [-]'%kz_inner.min())
            even_print('max : (kz·δ_ν) ' , '%0.5f [-]'%kz_inner.max())
            even_print('min : (kz·δ99) ' , '%0.5f [-]'%kz_outer.min())
            even_print('max : (kz·δ99) ' , '%0.5e [-]'%kz_outer.max())
            
            print(72*'-')
        
        kzp2d,fp2d = np.meshgrid(kzp,fp, indexing='ij')
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        if True: ## just a test currently --> not finished!!!
            
            yp_tgt = 100.
            yp = np.copy( y / sc_l_in )
            yi = np.abs(yp-yp_tgt).argmin()
            
            dd = np.copy( self.U_inf * self['data/uI'][:win_len,:,yi,0].T )
            #print(dd.shape)
            
            A_ui = sp.fft.fftn(dd) # [kzp2d, fp2d]
            A_uj = sp.fft.fftn(dd) # [kzp2d, fp2d]
            data['A_ui'] = A_ui
            data['A_uj'] = A_uj
            
            #Euu_ijk = np.copy(  2 * np.real(A_ui*np.conj(A_uj)) / ( df * dkz ) )
            #data['Euu'] = Euu_ijk
        
        # === save results
        if (self.rank==0):
            
            #data['Euu']      = Euu_avg
            #data['uIuI_avg'] = uIuI_avg
            
            with open(fn_dat_fft,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_fft,os.path.getsize(fn_dat_fft)/1024**2))
        
        # ===
        
        if self.usingmpi: self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_turb_spectrum_2d_span_time() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_ccor_time(self, **kwargs):
        '''
        calculate cross-correlation in [t] and avg in [x,z] --> leave [y,Δt]
        - designed for analyzing unsteady, thin planes in [x]
        '''
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_ccor_time()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_dat_ccor_time = kwargs.get('fn_dat_ccor_time',None)
        fn_dat_mean_dim  = kwargs.get('fn_dat_mean_dim',None)
        
        ## for now only distribute data in [y]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # ===
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # # === mean dimensional file name (for reading) : .dat
        # if (fn_dat_mean_dim is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_dat_mean_base = fname_root+'_mean_dim.dat'
        #     fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
        
        # === cross-correlation file name (for writing) : dat
        if (fn_dat_ccor_time is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
            fname_ccor_time_dat_base = fname_root+'_ccor_time.dat'
            fn_dat_ccor_time = str(PurePosixPath(fname_path, fname_ccor_time_dat_base))
        
        if verbose: even_print('fn_rgd_prime'     , self.fname       )
        #if verbose: even_print('fn_dat_mean_dim'  , fn_dat_mean_dim  )
        if verbose: even_print('fn_dat_ccor_time' , fn_dat_ccor_time )
        if verbose: print(72*'-')
        
        #if not os.path.isfile(fn_dat_mean_dim):
        #    raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        # # === read in data (mean dim) --> every rank gets full [x,z]
        # with open(fn_dat_mean_dim,'rb') as f:
        #     data_mean_dim = pickle.load(f)
        # fmd = type('foo', (object,), data_mean_dim)
        
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('group data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho_avg = np.copy( self['data_dim/rho'][()] )
        u_avg   = np.copy( self['data_dim/u'][()]   )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        u_99     = float( self['data_dim/u_99'][()]     )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## these are recalculated and checked in next step
        z1d_ = np.copy( self['data_dim/z1d'][()] )
        dz0_ = np.copy( self['data_dim/dz0'][()] )
        dt_  = np.copy( self['data_dim/dt'][()]  )
        
        # ===
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays, then re-dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        # ## dimless (inlet/characteristic)
        # xd = self.x
        # yd = self.y
        # zd = self.z
        # td = self.t
        
        ## dimensional [m] / [s]
        x      = self.x * lchar 
        y      = self.y * lchar
        z      = self.z * lchar
        t      = self.t * (lchar/U_inf)
        t_meas = t[-1]-t[0]
        dt     = self.dt * (lchar/U_inf)
        
        z1d = np.copy(z)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z[-1]-z[0]
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        #data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('dt'     , '%0.5e [s]' % dt      )
            even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        t_eddy = t_meas / ( d99 / u_tau )
        
        if verbose:
            even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
            even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
            even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
        
        ## get lags
        lags,_  = ccor( np.ones(nt,dtype=np.float32) , np.ones(nt,dtype=np.float32), get_lags=True )
        n_lags_ = nt*2-1
        n_lags  = lags.shape[0]
        if (n_lags!=n_lags_):
            raise AssertionError('possible problem with lags calc --> check!')
        
        data['lags'] = lags
        data['n_lags'] = n_lags
        
        if verbose:
            even_print('n lags (Δt)' , '%i'%(n_lags,))
        
        ## [var1, var2, density_scaling]
        R_combis = [
                   [ 'uI'  , 'uI'  , False ],
                   [ 'vI'  , 'vI'  , False ],
                   [ 'wI'  , 'wI'  , False ],
                   [ 'uI'  , 'vI'  , False ],
                   [ 'uI'  , 'TI'  , False ],
                   [ 'vI'  , 'TI'  , False ],
                   [ 'TI'  , 'TI'  , False ],
                   [ 'uII' , 'uII' , True  ],
                   [ 'vII' , 'vII' , True  ],
                   [ 'wII' , 'wII' , True  ],
                   [ 'uII' , 'vII' , True  ],
                   [ 'uII' , 'TII' , True  ],
                   [ 'vII' , 'TII' , True  ],
                   [ 'TII' , 'TII' , True  ],
                   ]
        
        if verbose:
            even_print('n ccor (Δt) scalar combinations' , '%i'%(len(R_combis),))
            print(72*'-')
        
        ## decide if density will be needed
        read_density = False
        for cc in R_combis:
            scalar_L, scalar_R, density_scaling = cc
            if density_scaling:
                read_density = True
                break
        
        if verbose:
            even_print('read ρ', str(read_density))
        
        ## confirm 'rho' is not in locals
        if read_density and ('rho' in locals()):
            raise ValueError('rho alread in locals... check')
        
        if read_density:
            
            ## buffer for rho (NOT rhoI) --> this is read from 'prime' file
            rho = np.zeros(shape=(nx, nyr, nz, nt), dtype=np.float32)
            
            dset = self['data/rho']
            self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    rho[:,:,:,:] = dset[:,:,ry1:ry2,:].T
            else:
                rho[:,:,:,:] = dset[()].T
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
            if verbose:
                even_print( 'read: rho','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            ## re-dimensionalize rho
            rho *= rho_inf
        
        scalars_R        = [ '%s%s'%(cc[0],cc[1]) for cc in R_combis ]
        scalars_dtypes_R = [ np.float32 for s in scalars_R ]
        
        ## averaged cross-correlation data buffer
        ## 3D [scalar][y,Δt] structured array
        data_R_avg = np.zeros(shape=(nyr, n_lags), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
        
        ## main loop --> cross-correlation in [Δt] at every [x,y,z]
        if verbose:
            progress_bar = tqdm(total=len(R_combis)*nx*nyr*nz, ncols=100, desc='ccor_time()', leave=False, file=sys.stdout)
        
        for cci,cc in enumerate(R_combis):
            
            scalar_L, scalar_R, density_scaling = cc
            tag = '%s%s'%(scalar_L, scalar_R)
            
            ## check if autocorrelation
            if (scalar_L==scalar_R):
                scalars = [ scalar_L ]
            else:
                scalars = [ scalar_L, scalar_R ]
            
            scalars_dtypes = [self.scalars_dtypes_dict[s] for s in scalars]
            
            ## prime data buffer
            ## 5D [scalar][x,y,z,t] structured array
            data_prime = np.zeros(shape=(nx, nyr, nz, nt), dtype={'names':scalars, 'formats':scalars_dtypes})
            
            ## cross-correlation data buffer
            ## 5D [scalar][x,y,z,t] structured array
            #scalars_R        = [ tag ]
            #scalars_dtypes_R = [ np.float32 for s in scalars_R ]
            #data_R           = np.zeros(shape=(nx, nyr, n_lags, nt), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
            
            ## cross-correlation data buffer
            ## 4D numpy array
            data_R = np.zeros(shape=(nx, nyr, nz, n_lags), dtype=np.float32)
            
            ## read prime data
            for scalar in scalars:
                dset = self['data/%s'%scalar]
                self.comm.Barrier()
                t_start = timeit.default_timer()
                with dset.collective:
                    data_prime[scalar][:,:,:,:] = np.copy( dset[:,:,ry1:ry2,:].T )
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
                if verbose:
                    tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            ## redimensionalize prime data
            for var in data_prime.dtype.names:
                if var in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII']:
                    data_prime[var] *= U_inf
                elif var in ['r_uII','r_vII','r_wII']:
                    data_prime[var] *= (U_inf*rho_inf)
                elif var in ['T','TI','TII']:
                    data_prime[var] *= T_inf
                elif var in ['r_TII']:
                    data_prime[var] *= (T_inf*rho_inf)
                elif var in ['rho','rhoI']:
                    data_prime[var] *= rho_inf
                elif var in ['p','pI','pII']:
                    data_prime[var] *= (rho_inf * U_inf**2)
                else:
                    raise ValueError('condition needed for redimensionalizing \'%s\''%var)
            
            # ===
            
            ## print names of components being cross-correlated
            if verbose:
                if density_scaling:
                    fancy_tag = '<ρ·%s,ρ·%s>'%(scalar_L,scalar_R)
                else:
                    fancy_tag = '<%s,%s>'%(scalar_L,scalar_R)
                tqdm.write(even_print('running Δt cross-correlation calc', fancy_tag, s=True))
            
            for xi in range(nx):
                for yi in range(nyr):
                    for zi in range(nz):
                        
                        uL = np.copy( data_prime[scalar_L][xi,yi,zi,:]   )
                        uR = np.copy( data_prime[scalar_R][xi,yi,zi,:]   )
                        
                        if density_scaling:
                            rho1d = np.copy( rho[xi,yi,zi,:] )
                        else:
                            rho1d = None
                        
                        if density_scaling:
                            data_R[xi,yi,zi,:] = ccor( rho1d*uL , rho1d*uR )
                        else:
                            data_R[xi,yi,zi,:] = ccor( uL , uR )
                        
                        if verbose: progress_bar.update()
            
            # ===
            
            ## average in [x,z] --> leave [y,lag] (where lag is Δt)
            data_R_avg[tag] = np.mean(data_R, axis=(0,2), dtype=np.float64).astype(np.float32)
            data_R = None; del data_R
            
            ## manually delete the prime data from memory
            data_prime = None; del data_prime
            self.comm.Barrier()
        
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        # === gather all (pre-averaged) results
        
        self.comm.Barrier()
        data_R_all = None
        if (self.rank==0):
            
            j=0
            data_R_all = np.zeros(shape=(ny,n_lags), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
            
            ## data this rank
            for scalar_R in scalars_R:
                data_R_all[scalar_R][ryl[j][0]:ryl[j][1],:] = data_R_avg[scalar_R]
        
        for scalar_R in scalars_R:
            for ri in range(1,self.n_ranks):
                j = ri
                self.comm.Barrier()
                if (self.rank==ri):
                    sendbuf = np.copy(data_R_avg[scalar_R])
                    self.comm.Send(sendbuf, dest=0, tag=ri)
                    #print('rank %i sending %s'%(ri,scalar_R))
                elif (self.rank==0):
                    #print('rank %i receiving %s'%(self.rank,scalar_R))
                    nyri = ryl[j][1] - ryl[j][0]
                    ##
                    recvbuf = np.zeros((nyri,n_lags), dtype=data_R_avg[scalar_R].dtype)
                    ##
                    #print('rank %i : recvbuf.shape=%s'%(rank,str(recvbuf.shape)))
                    self.comm.Recv(recvbuf, source=ri, tag=ri)
                    data_R_all[scalar_R][ryl[j][0]:ryl[j][1],:] = recvbuf
                else:
                    pass
        
        ## overwrite
        if (self.rank==0):
            R = np.copy(data_R_all)
        
        # === save results
        
        if (self.rank==0):
            
            data['R']    = R ## the main cross-correlation data array
            data['lags'] = lags
            
            with open(fn_dat_ccor_time,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_ccor_time,os.path.getsize(fn_dat_ccor_time)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        
        #if verbose: print('\n'+72*'-')
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_ccor_time() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_ccor_span(self, **kwargs):
        '''
        calculate cross-correlation in [z] and avg in [x,t] --> leave [y,Δz]
        - designed for analyzing unsteady, thin planes in [x]
        '''
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_ccor_span()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_dat_ccor_span = kwargs.get('fn_dat_ccor_span',None)
        #fn_dat_mean_dim  = kwargs.get('fn_dat_mean_dim',None)
        
        ## for now only distribute data in [y]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # ===
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # # === mean dimensional file name (for reading) : .dat
        # if (fn_dat_mean_dim is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_dat_mean_base = fname_root+'_mean_dim.dat'
        #     fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
        
        # === cross-correlation file name (for writing) : dat
        if (fn_dat_ccor_span is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
            fname_ccor_span_dat_base = fname_root+'_ccor_span.dat'
            fn_dat_ccor_span = str(PurePosixPath(fname_path, fname_ccor_span_dat_base))
        
        if verbose: even_print('fn_rgd_prime'     , self.fname       )
        #if verbose: even_print('fn_dat_mean_dim'  , fn_dat_mean_dim  )
        if verbose: even_print('fn_dat_ccor_span' , fn_dat_ccor_span )
        if verbose: print(72*'-')
        
        # if not os.path.isfile(fn_dat_mean_dim):
        #     raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        # # === read in data (mean dim) --> every rank gets full [x,z]
        # with open(fn_dat_mean_dim,'rb') as f:
        #     data_mean_dim = pickle.load(f)
        # fmd = type('foo', (object,), data_mean_dim)
        
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('group data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho_avg = np.copy( self['data_dim/rho'][()] )
        u_avg   = np.copy( self['data_dim/u'][()]   )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        u_99     = float( self['data_dim/u_99'][()]     )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## these are recalculated and checked in next step
        z1d_ = np.copy( self['data_dim/z1d'][()] )
        dz0_ = np.copy( self['data_dim/dz0'][()] )
        dt_  = np.copy( self['data_dim/dt'][()]  )
        
        # ===
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays, then re-dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        # ## dimless (inlet/characteristic)
        # xd = self.x
        # yd = self.y
        # zd = self.z
        # td = self.t
        
        ## redimensionalize coordinates --> [m],[s],[m/s],etc.
        x      = self.x * lchar
        y      = self.y * lchar
        z      = self.z * lchar
        t      = self.t * (lchar/U_inf)
        t_meas = t[-1]-t[0]
        dt     = self.dt * (lchar/U_inf)
        
        z1d = np.copy(z)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z1d.max() - z1d.min()
        #zrange = z[-1]-z[0]
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        #data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('dt'     , '%0.5e [s]' % dt      )
            even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        t_eddy = t_meas / ( d99 / u_tau )
        
        if verbose:
            even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
            even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
            even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
        
        ## get lags
        lags,_  = ccor( np.ones(nz,dtype=np.float32) , np.ones(nz,dtype=np.float32), get_lags=True )
        n_lags_ = nz*2-1
        n_lags  = lags.shape[0]
        if (n_lags!=n_lags_):
            raise AssertionError('possible problem with lags calc --> check!')
        
        if verbose:
            even_print('n lags (Δz)' , '%i'%(n_lags,))
        
        ## [var1, var2, density_scaling]
        R_combis = [
                   [ 'uI'  , 'uI'  , False ],
                   [ 'vI'  , 'vI'  , False ],
                   [ 'wI'  , 'wI'  , False ],
                   [ 'uI'  , 'vI'  , False ],
                   [ 'uI'  , 'TI'  , False ],
                   [ 'vI'  , 'TI'  , False ],
                   [ 'TI'  , 'TI'  , False ],
                   [ 'uII' , 'uII' , True  ],
                   [ 'vII' , 'vII' , True  ],
                   [ 'wII' , 'wII' , True  ],
                   [ 'uII' , 'vII' , True  ],
                   [ 'uII' , 'TII' , True  ],
                   [ 'vII' , 'TII' , True  ],
                   [ 'TII' , 'TII' , True  ],
                   ]
        
        if verbose:
            even_print('n ccor (Δz) scalar combinations' , '%i'%(len(R_combis),))
            print(72*'-')
        
        ## decide if density will be needed
        read_density = False
        for cc in R_combis:
            scalar_L, scalar_R, density_scaling = cc
            if density_scaling:
                read_density = True
                break
        
        if verbose:
            even_print('read ρ', str(read_density))
        
        ## confirm 'rho' is not in locals
        if read_density and ('rho' in locals()):
            raise ValueError('rho alread in locals... check')
        
        if read_density:
            
            ## buffer for rho (NOT rhoI) --> this is read from 'prime' file
            rho = np.zeros(shape=(nx, nyr, nz, nt), dtype=np.float32)
            
            dset = self['data/rho']
            self.comm.Barrier()
            t_start = timeit.default_timer()
            with dset.collective:
                rho[:,:,:,:] = dset[:,:,ry1:ry2,:].T
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
            if verbose:
                even_print( 'read: rho','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            ## re-dimensionalize rho
            rho *= rho_inf
        
        scalars_R        = [ '%s%s'%(cc[0],cc[1]) for cc in R_combis ]
        scalars_dtypes_R = [ np.float32 for s in scalars_R ]
        
        ## averaged cross-correlation data buffer
        ## 3D [scalar][y,Δz] structured array
        data_R_avg = np.zeros(shape=(nyr, n_lags), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
        
        ## main loop --> cross-correlation in [Δz] at every [x,y,t]
        if verbose:
            progress_bar = tqdm(total=len(R_combis)*nx*nyr*nt, ncols=100, desc='ccor_span()', leave=False, file=sys.stdout)
        
        for cci,cc in enumerate(R_combis):
            
            scalar_L, scalar_R, density_scaling = cc
            tag = '%s%s'%(scalar_L, scalar_R)
            
            ## check if autocorrelation
            if (scalar_L==scalar_R):
                scalars = [ scalar_L ]
            else:
                scalars = [ scalar_L, scalar_R ]
            
            scalars_dtypes = [self.scalars_dtypes_dict[s] for s in scalars]
            
            ## prime data buffer
            ## 5D [scalar][x,y,z,t] structured array
            data_prime = np.zeros(shape=(nx, nyr, nz, nt), dtype={'names':scalars, 'formats':scalars_dtypes})
            
            ## cross-correlation data buffer
            ## 5D [scalar][x,y,z,t] structured array
            #scalars_R        = [ tag ]
            #scalars_dtypes_R = [ np.float32 for s in scalars_R ]
            #data_R           = np.zeros(shape=(nx, nyr, n_lags, nt), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
            
            ## cross-correlation data buffer
            ## 4D numpy array
            data_R = np.zeros(shape=(nx, nyr, n_lags, nt), dtype=np.float32)
            
            ## read prime data
            for scalar in scalars:
                dset = self['data/%s'%scalar]
                self.comm.Barrier()
                t_start = timeit.default_timer()
                with dset.collective:
                    data_prime[scalar][:,:,:,:] = np.copy( dset[:,:,ry1:ry2,:].T )
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
                if verbose:
                    tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            ## redimensionalize prime data
            for var in data_prime.dtype.names:
                if var in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII']:
                    data_prime[var] *= U_inf
                elif var in ['r_uII','r_vII','r_wII']:
                    data_prime[var] *= (U_inf*rho_inf)
                elif var in ['T','TI','TII']:
                    data_prime[var] *= T_inf
                elif var in ['r_TII']:
                    data_prime[var] *= (T_inf*rho_inf)
                elif var in ['rho','rhoI']:
                    data_prime[var] *= rho_inf
                elif var in ['p','pI','pII']:
                    data_prime[var] *= (rho_inf * U_inf**2)
                else:
                    raise ValueError('condition needed for redimensionalizing \'%s\''%var)
            
            # ===
            
            ## print names of components being cross-correlated
            if verbose:
                if density_scaling:
                    fancy_tag = '<ρ·%s,ρ·%s>'%(scalar_L,scalar_R)
                else:
                    fancy_tag = '<%s,%s>'%(scalar_L,scalar_R)
                tqdm.write(even_print('running Δz cross-correlation calc', fancy_tag, s=True))
            
            for xi in range(nx):
                for yi in range(nyr):
                    for ti in range(nt):
                        
                        uL = np.copy( data_prime[scalar_L][xi,yi,:,ti]   )
                        uR = np.copy( data_prime[scalar_R][xi,yi,:,ti]   )
                        
                        if density_scaling:
                            rho1d = np.copy( rho[xi,yi,:,ti] )
                        else:
                            rho1d = None
                        
                        if density_scaling:
                            data_R[xi,yi,:,ti] = ccor( rho1d*uL , rho1d*uR )
                        else:
                            data_R[xi,yi,:,ti] = ccor( uL , uR )
                        
                        if verbose: progress_bar.update()
            
            # ===
            
            ## average in [x,t] --> leave [y,lag] (where lag is Δz)
            data_R_avg[tag] = np.mean(data_R, axis=(0,3), dtype=np.float64).astype(np.float32)
            data_R = None; del data_R
            
            ## manually delete the prime data from memory
            data_prime = None; del data_prime
            self.comm.Barrier()
        
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        # === gather all (pre-averaged) results
        
        self.comm.Barrier()
        data_R_all = None
        if (self.rank==0):
            
            j=0
            data_R_all = np.zeros(shape=(ny,n_lags), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
            
            ## data this rank
            for scalar_R in scalars_R:
                data_R_all[scalar_R][ryl[j][0]:ryl[j][1],:] = data_R_avg[scalar_R]
        
        for scalar_R in scalars_R:
            for ri in range(1,self.n_ranks):
                j = ri
                self.comm.Barrier()
                if (self.rank==ri):
                    sendbuf = np.copy(data_R_avg[scalar_R])
                    self.comm.Send(sendbuf, dest=0, tag=ri)
                    #print('rank %i sending %s'%(ri,scalar_R))
                elif (self.rank==0):
                    #print('rank %i receiving %s'%(self.rank,scalar_R))
                    nyri = ryl[j][1] - ryl[j][0]
                    ##
                    recvbuf = np.zeros((nyri,n_lags), dtype=data_R_avg[scalar_R].dtype)
                    ##
                    #print('rank %i : recvbuf.shape=%s'%(rank,str(recvbuf.shape)))
                    self.comm.Recv(recvbuf, source=ri, tag=ri)
                    data_R_all[scalar_R][ryl[j][0]:ryl[j][1],:] = recvbuf
                else:
                    pass
        
        ## overwrite
        if (self.rank==0):
            R = np.copy(data_R_all)
        
        # === save results
        
        if (self.rank==0):
            
            data['R']    = R ## the main cross-correlation data array
            data['lags'] = lags
            
            with open(fn_dat_ccor_span,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_ccor_span,os.path.getsize(fn_dat_ccor_span)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        
        #if verbose: print('\n'+72*'-')
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_ccor_span() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_turb_budget(self, **kwargs):
        '''
        calculate budget of turbulent kinetic energy (k)
        -----
        - parallelized over time [t] dimension
        - dimensional [SI]
        - designed for analyzing unsteady, thin planes in [x]
        - requires that get_prime() and get_mean() were run on unsteady RGD
        - requires that get_prime() was run with favre=True
        - currently does NOT work in serial mode (parallel only)
        -----
        SI units of terms are [kg/(m·s³)] or [kg m^-1 s^-3]
        normalize from SI with
        * nu_wall / u_tau**4 / rho_wall --> *[ kg^-1 m s^3]
        or
        / ( u_tau**4 * rho_wall / nu_wall ) --> /[kg m^-1 s^-3]
        -----
        Gaurini, S., Moser, R., Shariff, K., & Wray, A. (2000).
        Direct numerical simulation of a supersonic turbulent boundary layer at Mach 2.5.
        Journal of Fluid Mechanics, 414, 1-33.
        doi:10.1017/S0022112000008466
        -----
        Pirozzoli, S., Grasso, F., & Gatski, T. B. (2004).
        Direct numerical simulation and analysis of a spatially evolving supersonic turbulent boundary layer at M= 2.25.
        Physics of fluids, 16(3), 530-545.
        doi:10.1063/1.1637604
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_turb_budget()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        if ( not self.usingmpi ):
            raise ValueError('rgd.calc_turb_budget() currently only works in MPI mode')
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        ct = kwargs.get('ct',1)
        
        force = kwargs.get('force',False)
        
        fn_rgd_mean        = kwargs.get('fn_rgd_mean',None)
        fn_rgd_prime       = kwargs.get('fn_rgd_prime',None)
        fn_rgd_turb_budget = kwargs.get('fn_rgd_turb_budget',None)
        fn_rgd_turb_budget = kwargs.get('fn_rgd_turb_budget',None)
        fn_dat_turb_budget = kwargs.get('fn_dat_turb_budget',None)
        
        #save_unsteady = kwargs.get('save_unsteady',False)
        
        acc          = kwargs.get('acc', 6)
        edge_stencil = kwargs.get('edge_stencil', 'half')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        ## for now only distribute data in [t]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (ry!=1):
            raise AssertionError('ry!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # ===
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        #ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        #ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        rt1,rt2 = rtl[self.rank]; ntr = rt2 - rt1
        
        ## [t] sub chunk range --> ctl = list of ranges in rt1:rt2
        ctl_ = np.array_split( np.arange(rt1,rt2) , min(ct,ntr) )
        ctl = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## check that no sub ranges are <=1
        for a_ in [ ctl_[1]-ctl_[0] for ctl_ in ctl ]:
            if (a_ <= 1):
                raise ValueError
        
        ## the average sub [t] chunk size on this rank
        avg_chunk_nt = np.mean( [ ctl_[1]-ctl_[0] for ctl_ in ctl ] )
        
        if True: ## check that [t] sub-chunk ranges are correct
            
            mytimeindices = []
            for ctl_ in ctl:
                ct1, ct2 = ctl_
                mytimeindices += [ ti_ for ti_ in self.ti[ct1:ct2] ]
            
            G = self.comm.gather([ self.rank , mytimeindices ], root=0)
            G = self.comm.bcast(G, root=0)
            
            alltimeindices = []
            for G_ in G:
                alltimeindices += G_[1]
            alltimeindices = np.array( sorted(alltimeindices), dtype=np.int64 )
            
            if not np.array_equal( alltimeindices , self.ti ):
                raise AssertionError
            if not np.array_equal( alltimeindices , np.arange(self.nt, dtype=np.int64) ):
                raise AssertionError
        
        # === mean file name (for reading)
        if (fn_rgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_rgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_rgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_rgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if not os.path.isfile(fn_rgd_mean):
            raise FileNotFoundError('%s not found!'%fn_rgd_mean)
        
        # === prime file name (for reading)
        if (fn_rgd_prime is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_prime_h5_base = fname_root+'_prime.h5'
            #fn_rgd_prime = os.path.join(fname_path, fname_prime_h5_base)
            fn_rgd_prime = str(PurePosixPath(fname_path, fname_prime_h5_base))
            #fn_rgd_prime = Path(fname_path, fname_prime_h5_base)
        
        if not os.path.isfile(fn_rgd_prime):
            raise FileNotFoundError('%s not found!'%fn_rgd_prime)
        
        # === turb_budget .h5 file name (for writing)
        if (fn_rgd_turb_budget is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fn_rgd_turb_budget_base = fname_root+'_turb_budget.h5'
            fn_rgd_turb_budget = str(PurePosixPath(fname_path, fn_rgd_turb_budget_base))
        
        if os.path.isfile(fn_rgd_turb_budget) and (force is False):
            raise ValueError(f'{fn_rgd_turb_budget} already present & force=False')
        
        # === turb_budget .dat file name (for writing)
        if (fn_dat_turb_budget is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fn_dat_turb_budget_base = fname_root+'_turb_budget.dat'
            fn_dat_turb_budget = str(PurePosixPath(fname_path, fn_dat_turb_budget_base))
        
        if verbose: even_print('fn_rgd'             , self.fname         )
        if verbose: even_print('fn_rgd_mean'        , fn_rgd_mean        )
        if verbose: even_print('fn_rgd_prime'       , fn_rgd_prime       )
        if verbose: even_print('fn_rgd_turb_budget' , fn_rgd_turb_budget )
        if verbose: even_print('fn_dat_turb_budget' , fn_dat_turb_budget )
        if verbose: print(72*'-')
        
        # ===
        
        if verbose: even_print('nx' , '%i'%self.nx)
        if verbose: even_print('ny' , '%i'%self.ny)
        if verbose: even_print('nz' , '%i'%self.nz)
        if verbose: even_print('nt' , '%i'%self.nt)
        #if verbose: even_print('save_unsteady', str(save_unsteady))
        if verbose: print(72*'-')
        if verbose: even_print('rt' , '%i'%rt)
        if verbose: even_print('ct' , '%i'%ct)
        if verbose: even_print('ntr' , '%i'%ntr)
        if verbose: even_print('avg [t] chunk nt' , '%0.2f'%avg_chunk_nt)
        if verbose: print(72*'-')
        
        # === init outfiles
        
        ## initialize file: turbulent kinetic energy budget, averaged
        with rgd(fn_rgd_turb_budget, 'w', force=force, driver='mpio', comm=self.comm) as f1:
            
            f1.init_from_rgd(self.fname, t_info=False)
            
            ## set some top-level attributes
            #f1.attrs['duration_avg'] = duration_avg ## duration of mean
            f1.attrs['duration_avg'] = self.t[-1] - self.t[0]
            #f1_mean.attrs['duration_avg'] = self.duration
            f1.attrs['dt'] = self.t[1] - self.t[0]
            #f1.attrs['fclass'] = 'rgd'
            f1.attrs['fsubtype'] = 'mean'
            
            shape = (1,f1.nz,f1.ny,f1.nx)
            chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=4)
            
            data_gb = 4*1*f1.nz*f1.ny*f1.nx / 1024**3
            
            for dss in ['production','dissipation','transport','diffusion','p_dilatation','p_diffusion']:
                
                if verbose:
                    even_print('initializing data/%s'%(dss,),'%0.1f [GB]'%(data_gb,))
                
                dset = f1.create_dataset('data/%s'%dss, 
                                         shape=shape, 
                                         dtype=np.float32,
                                         chunks=chunks)
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            # === replace dims/t array --> take last time of series
            if ('dims/t' in f1):
                del f1['dims/t']
            f1.create_dataset('dims/t', data=np.array([self.t[-1]],dtype=np.float64))
            
            if hasattr(f1, 'duration_avg'):
                if verbose: even_print('duration_avg', '%0.2f'%f1.duration_avg)
        
        if verbose: print(72*'-')
        
        self.comm.Barrier()
        
        # ===
        
        ## with rgd(fn_rgd, 'r', driver='mpio', comm=comm, verbose=False) as hf_rgd: #self
        with rgd(fn_rgd_prime, 'r', driver='mpio', comm=self.comm) as hf_prime:
            with rgd(fn_rgd_mean, 'r', driver='mpio', comm=self.comm) as hf_mean:
                
                np.testing.assert_allclose(hf_prime.t, self.t, atol=1e-14, rtol=1e-14)
                
                np.testing.assert_allclose(hf_prime.x, self.x, atol=1e-14, rtol=1e-14)
                np.testing.assert_allclose(hf_prime.y, self.y, atol=1e-14, rtol=1e-14)
                np.testing.assert_allclose(hf_prime.z, self.z, atol=1e-14, rtol=1e-14)
                
                np.testing.assert_allclose(hf_prime.x, hf_mean.x, atol=1e-14, rtol=1e-14)
                np.testing.assert_allclose(hf_prime.y, hf_mean.y, atol=1e-14, rtol=1e-14)
                np.testing.assert_allclose(hf_prime.z, hf_mean.z, atol=1e-14, rtol=1e-14)
                
                np.testing.assert_allclose( hf_prime.lchar   , hf_mean.lchar   , atol=1e-14, rtol=1e-14 )
                np.testing.assert_allclose( hf_prime.lchar   , self.lchar      , atol=1e-14, rtol=1e-14 )
                np.testing.assert_allclose( hf_prime.U_inf   , hf_mean.U_inf   , atol=1e-14, rtol=1e-14 )
                np.testing.assert_allclose( hf_prime.U_inf   , self.U_inf      , atol=1e-14, rtol=1e-14 )
                np.testing.assert_allclose( hf_prime.T_inf   , hf_mean.T_inf   , atol=1e-14, rtol=1e-14 )
                np.testing.assert_allclose( hf_prime.T_inf   , self.T_inf      , atol=1e-14, rtol=1e-14 )
                np.testing.assert_allclose( hf_prime.rho_inf , hf_mean.rho_inf , atol=1e-14, rtol=1e-14 )
                np.testing.assert_allclose( hf_prime.rho_inf , self.rho_inf    , atol=1e-14, rtol=1e-14 )
                
                # ===
                
                if (hf_prime.fsubtype!='prime'):
                    raise ValueError
                if (hf_mean.fsubtype!='mean'):
                    raise ValueError
                
                data = {} ## dict to be output to .dat file
                
                if ('data_dim' not in hf_mean):
                    raise ValueError('group data_dim not present')
                
                ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
                for dsn in hf_mean['data_dim'].keys():
                    d_ = np.copy( hf_mean[f'data_dim/{dsn}'][()] )
                    if (d_.ndim == 0):
                        d_ = float(d_)
                    data[dsn] = d_
                
                ## 1D
                #rho_avg = np.copy( hf_mean['data_dim/rho'][()] )
                #u_avg   = np.copy( hf_mean['data_dim/u'][()]   )
                #v_avg   = np.copy( hf_mean['data_dim/v'][()]   )
                #w_avg   = np.copy( hf_mean['data_dim/w'][()]   )
                #T_avg   = np.copy( hf_mean['data_dim/T'][()]   )
                #p_avg   = np.copy( hf_mean['data_dim/p'][()]   )
                
                ## 0D
                u_tau    = float( hf_mean['data_dim/u_tau'][()]    )
                nu_wall  = float( hf_mean['data_dim/nu_wall'][()]  )
                rho_wall = float( hf_mean['data_dim/rho_wall'][()] )
                T_wall   = float( hf_mean['data_dim/T_wall'][()]   )
                d99      = float( hf_mean['data_dim/d99'][()]      )
                u_99     = float( hf_mean['data_dim/u_99'][()]     )
                Re_tau   = float( hf_mean['data_dim/Re_tau'][()]   )
                Re_theta = float( hf_mean['data_dim/Re_theta'][()] )
                sc_u_in  = float( hf_mean['data_dim/sc_u_in'][()]  )
                sc_l_in  = float( hf_mean['data_dim/sc_l_in'][()]  )
                sc_t_in  = float( hf_mean['data_dim/sc_t_in'][()]  )
                sc_u_out = float( hf_mean['data_dim/sc_u_out'][()] )
                sc_l_out = float( hf_mean['data_dim/sc_l_out'][()] )
                sc_t_out = float( hf_mean['data_dim/sc_t_out'][()] )
                
                ## 0D scalars
                lchar   = self.lchar   ; data['lchar']   = lchar
                U_inf   = self.U_inf   ; data['U_inf']   = U_inf
                rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
                T_inf   = self.T_inf   ; data['T_inf']   = T_inf
                
                #data['M_inf'] = self.M_inf
                data['Ma'] = self.Ma
                data['Pr'] = self.Pr
                
                ## read in 1D coordinate arrays, then re-dimensionalize [m]
                x = np.copy( self['dims/x'][()] * self.lchar )
                y = np.copy( self['dims/y'][()] * self.lchar )
                z = np.copy( self['dims/z'][()] * self.lchar )
                t = np.copy( self['dims/t'][()] * self.tchar )
                
                ## dimensional [s]
                dt = self.dt * self.tchar
                np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
                
                t_meas = self.duration * self.tchar
                np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
                
                t_eddy = t_meas / ( d99 / u_tau )
                
                ## check if constant Δz (calculate Δz+ later)
                dz0 = np.diff(z)[0]
                if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-7)):
                    raise NotImplementedError
                np.testing.assert_allclose(dz0, z[1]-z[0], rtol=1e-14, atol=1e-14)
                
                zrange = z.max() - z.min()
                np.testing.assert_allclose(zrange, z[-1]-z[0], rtol=1e-14, atol=1e-14)
                
                nx = self.nx ; data['nx'] = nx
                ny = self.ny ; data['ny'] = ny
                nz = self.nz ; data['nz'] = nz
                nt = self.nt ; data['nt'] = nt
                
                ## add data to dict that gets output
                data['x'] = x
                data['y'] = y
                data['z'] = z
                data['t'] = t
                data['t_meas'] = t_meas
                data['dt'] = dt
                data['dz0'] = dz0
                data['zrange'] = zrange
                
                if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
                if verbose: even_print('Δt','%0.3e [s]'%(dt,))
                if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
                if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
                if verbose: print(72*'-')
                
                ## report
                if verbose:
                    even_print('dt'     , '%0.5e [s]' % dt      )
                    even_print('t_meas' , '%0.5e [s]' % t_meas  )
                    even_print('dz0'    , '%0.5e [m]' % dz0     )
                    even_print('zrange' , '%0.5e [m]' % zrange  )
                    print(72*'-')
                
                ## report
                if verbose:
                    even_print('Re_τ'    , '%0.1f'        % Re_tau    )
                    even_print('Re_θ'    , '%0.1f'        % Re_theta  )
                    even_print('δ99'     , '%0.5e [m]'    % d99       )
                    even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
                    even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
                    even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
                    even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
                    even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
                    ##
                    even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
                    even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
                    even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
                    print(72*'-')
                
                ## report
                if verbose:
                    even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
                    even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
                    even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
                    print(72*'-')
                
                # ===
                
                ## copy AVG [u,v,w] into memory
                u_re = np.copy( U_inf * hf_mean['data/u'][0,:,:,:].T    )
                v_re = np.copy( U_inf * hf_mean['data/v'][0,:,:,:].T    )
                w_re = np.copy( U_inf * hf_mean['data/w'][0,:,:,:].T    )
                u_fv = np.copy( U_inf * hf_mean['data/u_fv'][0,:,:,:].T )
                v_fv = np.copy( U_inf * hf_mean['data/v_fv'][0,:,:,:].T )
                w_fv = np.copy( U_inf * hf_mean['data/w_fv'][0,:,:,:].T )
                
                ## get Reynolds avg strain tensor elements
                dudx_re = gradient( u=u_re , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dudy_re = gradient( u=u_re , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dudz_re = gradient( u=u_re , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dvdx_re = gradient( u=v_re , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dvdy_re = gradient( u=v_re , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dvdz_re = gradient( u=v_re , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dwdx_re = gradient( u=w_re , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dwdy_re = gradient( u=w_re , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dwdz_re = gradient( u=w_re , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                
                ## get Favre avg strain tensor elements
                dudx_fv = gradient( u=u_fv , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dudy_fv = gradient( u=u_fv , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dudz_fv = gradient( u=u_fv , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dvdx_fv = gradient( u=v_fv , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dvdy_fv = gradient( u=v_fv , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dvdz_fv = gradient( u=v_fv , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dwdx_fv = gradient( u=w_fv , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dwdy_fv = gradient( u=w_fv , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                dwdz_fv = gradient( u=w_fv , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
                
                ## accumulators for per-timestep sum, at end gets multiplied by (1/nt) to get average
                unsteady_production_sum   = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                unsteady_dissipation_sum  = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                unsteady_transport_sum    = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                unsteady_diffusion_sum    = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                unsteady_p_diffusion_sum  = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                unsteady_p_dilatation_sum = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                
                # === main loop (iterate through sub-ranges within rank-distributed [t] range)
                
                if verbose:
                    progress_bar = tqdm(total=ct, ncols=100, desc='turb budget', leave=False, file=sys.stdout)
                
                ct_counter=0
                for ctl_ in ctl:
                    ct_counter += 1
                    ct1, ct2 = ctl_
                    ntc = ct2 - ct1
                    
                    if (ct>1):
                        if verbose:
                            mesg = f'[t] sub chunk {ct_counter:d}/{ct:d}'
                            tqdm.write( mesg )
                            tqdm.write( '-'*len(mesg) )
                    
                    # === read unsteady data
                    
                    ss1 = ['u','v','w','T','rho'] ## 'p'
                    ss2 = ['uI','vI','wI', 'uII','vII','wII', 'pI'] ## 'TI','rhoI'
                    
                    formats = [ np.float32 for s in ss1+ss2 ]
                    #shape = (nx,ny,nz,ntr)
                    shape = (nx,ny,nz,ntc)
                    dd = np.zeros(shape=shape, dtype={'names':ss1+ss2, 'formats':formats}, order='C')
                    
                    for ss in ss1:
                        dset = self['data/%s'%ss]
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            #dd[ss] = dset[rt1:rt2,:,:,:].T
                            dd[ss] = dset[ct1:ct2,:,:,:].T
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        #data_gb = 4*nx*ny*nz*nt/1024**3
                        data_gb = self.n_ranks * 4*nx*ny*nz*ntc/1024**3 ## approximate!
                        if verbose:
                            tqdm.write( even_print('read: %s'%ss, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True) )
                    
                    for ss in ss2:
                        dset = hf_prime['data/%s'%ss]
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            #dd[ss] = dset[rt1:rt2,:,:,:].T
                            dd[ss] = dset[ct1:ct2,:,:,:].T
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        #data_gb = 4*nx*ny*nz*nt/1024**3
                        data_gb = self.n_ranks * 4*nx*ny*nz*ntc/1024**3 ## approximate!
                        if verbose:
                            tqdm.write( even_print('read: %s'%ss, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True) )
                    
                    if verbose: tqdm.write(72*'-')
                    
                    # ===
                    
                    data_gb = dd.nbytes / 1024**3
                    if verbose:
                        tqdm.write(even_print('data_gb (rank)', '%0.1f [GB]'%data_gb, s=True))
                    
                    mem_avail_gb = psutil.virtual_memory().available/1024**3
                    mem_free_gb  = psutil.virtual_memory().free/1024**3
                    if verbose:
                        tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                    
                    # === redimensionalize the unsteady data
                    
                    u    = np.copy( dd['u']   * U_inf )
                    v    = np.copy( dd['v']   * U_inf )
                    w    = np.copy( dd['w']   * U_inf )
                    uI   = np.copy( dd['uI']  * U_inf )
                    vI   = np.copy( dd['vI']  * U_inf )
                    wI   = np.copy( dd['wI']  * U_inf )
                    uII  = np.copy( dd['uII'] * U_inf )
                    vII  = np.copy( dd['vII'] * U_inf )
                    wII  = np.copy( dd['wII'] * U_inf )
                    
                    rho  = np.copy( dd['rho']  * rho_inf              )
                    #rhoI = np.copy( dd['rhoI'] * rho_inf              )
                    #p    = np.copy( dd['p']    * (rho_inf * U_inf**2) )
                    pI   = np.copy( dd['pI']   * (rho_inf * U_inf**2) )
                    T    = np.copy( dd['T']    * T_inf                )
                    #TI   = np.copy( dd['TI']   * T_inf                )
                    
                    #mu = np.copy(14.58e-7*T**1.5/(T+110.4))
                    mu = self.C_Suth * T**(3/2) / (T + self.S_Suth)
                    nu = mu / rho
                    
                    dd = None ; del dd
                    
                    # === get gradients
                    
                    t_start = timeit.default_timer()
                    
                    ## numpy.gradient() --> low order, maximally O2
                    # dudx   = np.gradient(u,   x, edge_order=2, axis=0)
                    # dudy   = np.gradient(u,   y, edge_order=2, axis=1)
                    # dudz   = np.gradient(u,   z, edge_order=2, axis=2)
                    # dvdx   = np.gradient(v,   x, edge_order=2, axis=0)
                    # dvdy   = np.gradient(v,   y, edge_order=2, axis=1)
                    # dvdz   = np.gradient(v,   z, edge_order=2, axis=2)
                    # dwdx   = np.gradient(w,   x, edge_order=2, axis=0)
                    # dwdy   = np.gradient(w,   y, edge_order=2, axis=1)
                    # dwdz   = np.gradient(w,   z, edge_order=2, axis=2)
                    # ##
                    # duIdx  = np.gradient(uI,  x, edge_order=2, axis=0)
                    # duIdy  = np.gradient(uI,  y, edge_order=2, axis=1)
                    # duIdz  = np.gradient(uI,  z, edge_order=2, axis=2)
                    # dvIdx  = np.gradient(vI,  x, edge_order=2, axis=0)
                    # dvIdy  = np.gradient(vI,  y, edge_order=2, axis=1)
                    # dvIdz  = np.gradient(vI,  z, edge_order=2, axis=2)
                    # dwIdx  = np.gradient(wI,  x, edge_order=2, axis=0)
                    # dwIdy  = np.gradient(wI,  y, edge_order=2, axis=1)
                    # dwIdz  = np.gradient(wI,  z, edge_order=2, axis=2)
                    # ##
                    # duIIdx = np.gradient(uII, x, edge_order=2, axis=0)
                    # duIIdy = np.gradient(uII, y, edge_order=2, axis=1)
                    # duIIdz = np.gradient(uII, z, edge_order=2, axis=2)
                    # dvIIdx = np.gradient(vII, x, edge_order=2, axis=0)
                    # dvIIdy = np.gradient(vII, y, edge_order=2, axis=1)
                    # dvIIdz = np.gradient(vII, z, edge_order=2, axis=2)
                    # dwIIdx = np.gradient(wII, x, edge_order=2, axis=0)
                    # dwIIdy = np.gradient(wII, y, edge_order=2, axis=1)
                    # dwIIdz = np.gradient(wII, z, edge_order=2, axis=2)
                    
                    ## turbx.gradient() --> high order
                    ## get unsteady strain tensor elements
                    dudx   = gradient(u, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    dudy   = gradient(u, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    dudz   = gradient(u, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    dvdx   = gradient(v, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    dvdy   = gradient(v, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    dvdz   = gradient(v, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    dwdx   = gradient(w, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    dwdy   = gradient(w, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    dwdz   = gradient(w, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    ##
                    duIdx  = gradient(uI, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    duIdy  = gradient(uI, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    duIdz  = gradient(uI, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    dvIdx  = gradient(vI, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    dvIdy  = gradient(vI, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    dvIdz  = gradient(vI, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    dwIdx  = gradient(wI, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    dwIdy  = gradient(wI, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    dwIdz  = gradient(wI, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    ##
                    duIIdx = gradient(uII, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    duIIdy = gradient(uII, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    duIIdz = gradient(uII, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    dvIIdx = gradient(vII, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    dvIIdy = gradient(vII, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    dvIIdz = gradient(vII, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    dwIIdx = gradient(wII, x, acc=acc, edge_stencil=edge_stencil, axis=0)
                    dwIIdy = gradient(wII, y, acc=acc, edge_stencil=edge_stencil, axis=1)
                    dwIIdz = gradient(wII, z, acc=acc, edge_stencil=edge_stencil, axis=2)
                    
                    t_delta = timeit.default_timer() - t_start
                    
                    if verbose:
                        tqdm.write(even_print('get gradients', format_time_string(t_delta), s=True))
                    
                    self.comm.Barrier()
                    mem_avail_gb = psutil.virtual_memory().available/1024**3
                    mem_free_gb  = psutil.virtual_memory().free/1024**3
                    if verbose:
                        tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                        tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
                    
                    # === stack velocities into vectors, fluctuating strains into (Re stress) tensor
                    
                    t_start = timeit.default_timer()
                    
                    uI_i  = np.copy(np.stack((uI,  vI,  wI ), axis=-1))
                    uII_i = np.copy(np.stack((uII, vII, wII), axis=-1))
                    
                    duIdx_ij = np.copy( np.stack((np.stack((duIdx, duIdy, duIdz), axis=4),
                                                  np.stack((dvIdx, dvIdy, dvIdz), axis=4),
                                                  np.stack((dwIdx, dwIdy, dwIdz), axis=4)), axis=5) )
                    
                    duIIdx_ij = np.copy( np.stack((np.stack((duIIdx, duIIdy, duIIdz), axis=4),
                                                   np.stack((dvIIdx, dvIIdy, dvIIdz), axis=4),
                                                   np.stack((dwIIdx, dwIIdy, dwIIdz), axis=4)), axis=5) )
                    
                    t_delta = timeit.default_timer() - t_start
                    if verbose:
                        tqdm.write(even_print('tensor stacking',format_time_string(t_delta), s=True))
                    
                    self.comm.Barrier()
                    mem_avail_gb = psutil.virtual_memory().available/1024**3
                    mem_free_gb  = psutil.virtual_memory().free/1024**3
                    if verbose:
                        tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                        tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
                    
                    # === production : P
                    if True:
                        
                        if verbose: tqdm.write(72*'-')
                        t_start = timeit.default_timer()
                        
                        r_uII_uII = rho*uII*uII
                        r_uII_vII = rho*uII*vII
                        r_uII_wII = rho*uII*wII
                        
                        r_vII_uII = rho*vII*uII
                        r_vII_vII = rho*vII*vII
                        r_vII_wII = rho*vII*wII
                        
                        r_wII_uII = rho*wII*uII
                        r_wII_vII = rho*wII*vII
                        r_wII_wII = rho*wII*wII
                        
                        ## unsteady_production_ = - ( r_uII_uII * dudx_fv + r_uII_vII * dudy_fv + r_uII_wII * dudz_fv \
                        ##                          + r_uII_vII * dvdx_fv + r_vII_vII * dvdy_fv + r_vII_wII * dvdz_fv \
                        ##                          + r_uII_wII * dwdx_fv + r_vII_wII * dwdy_fv + r_wII_wII * dwdz_fv )
                        
                        r_uIIuII_ij = np.stack((np.stack((r_uII_uII, r_uII_vII, r_uII_wII), axis=4),
                                                np.stack((r_vII_uII, r_vII_vII, r_vII_wII), axis=4),
                                                np.stack((r_wII_uII, r_wII_vII, r_wII_wII), axis=4)), axis=5)
                        
                        dudx_fv_ij = np.stack((np.stack((dudx_fv, dudy_fv, dudz_fv), axis=4),
                                               np.stack((dvdx_fv, dvdy_fv, dvdz_fv), axis=4),
                                               np.stack((dwdx_fv, dwdy_fv, dwdz_fv), axis=4)), axis=5)
                        
                        unsteady_production = -1*np.einsum('xyztij,xyztij->xyzt', r_uIIuII_ij, dudx_fv_ij)
                        
                        ## np.testing.assert_allclose(unsteady_production, unsteady_production_, atol=20000)
                        ## print('check passed : np.einsum()')
                        
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('calc production', format_time_string(t_delta), s=True))
                        
                        self.comm.Barrier()
                        
                        ## combine sums across ranks for this [t] chunk
                        unsteady_production_sum_i   = np.sum(unsteady_production, axis=3, keepdims=True, dtype=np.float64)
                        unsteady_production_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                        self.comm.Reduce([unsteady_production_sum_i, MPI.DOUBLE],
                                         [unsteady_production_sum_buf, MPI.DOUBLE],
                                         op=MPI.SUM,
                                         root=0)
                        
                        ## add aggregated sum (across ranks) to the total accumulator
                        unsteady_production_sum += unsteady_production_sum_buf
                        
                        # === release mem
                        r_uII_uII           = None; del r_uII_uII
                        r_uII_vII           = None; del r_uII_vII
                        r_uII_wII           = None; del r_uII_wII
                        r_vII_uII           = None; del r_vII_uII
                        r_vII_vII           = None; del r_vII_vII
                        r_vII_wII           = None; del r_vII_wII
                        r_wII_uII           = None; del r_wII_uII
                        r_wII_vII           = None; del r_wII_vII
                        r_wII_wII           = None; del r_wII_wII
                        r_uIIuII_ij         = None; del r_uIIuII_ij
                        dudx_fv_ij          = None; del dudx_fv_ij
                        unsteady_production = None; del unsteady_production
                        
                        self.comm.Barrier()
                        mem_avail_gb = psutil.virtual_memory().available/1024**3
                        mem_free_gb  = psutil.virtual_memory().free/1024**3
                        if verbose:
                            tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                            tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
                    
                    # === dissipation : ϕ,ε
                    if True:
                        
                        if verbose: tqdm.write(72*'-')
                        t_start = timeit.default_timer()
                        
                        duIIdx_ij_duIdx_ij = np.einsum('xyztij,xyztij->xyzt', duIIdx_ij, duIdx_ij)
                        duIIdx_ij_duIdx_ji = np.einsum('xyztij,xyztji->xyzt', duIIdx_ij, duIdx_ij)
                        
                        ## # === from pp_turbulent_budget.F90
                        ## unsteady_dissipation_ = mu * ( (duIdx + duIdx) * duIIdx  +  (duIdy + dvIdx) * duIIdy  +  (duIdz + dwIdx) * duIIdz \
                        ##                              + (dvIdx + duIdy) * dvIIdx  +  (dvIdy + dvIdy) * dvIIdy  +  (dvIdz + dwIdy) * dvIIdz \
                        ##                              + (dwIdx + duIdz) * dwIIdx  +  (dwIdy + dvIdz) * dwIIdy  +  (dwIdz + dwIdz) * dwIIdz )
                        
                        unsteady_dissipation = mu*(duIIdx_ij_duIdx_ij + duIIdx_ij_duIdx_ji)
                        
                        ## np.testing.assert_allclose(unsteady_dissipation, unsteady_dissipation_, rtol=1e-4)
                        ## print('check passed : np.einsum() : dissipation')
                        
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('calc dissipation', format_time_string(t_delta),s=True))
                        
                        self.comm.Barrier()
                        
                        ## combine sums across ranks for this [t] chunk
                        unsteady_dissipation_sum_i   = np.sum(unsteady_dissipation, axis=3, keepdims=True, dtype=np.float64)
                        unsteady_dissipation_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                        self.comm.Reduce([unsteady_dissipation_sum_i, MPI.DOUBLE],
                                         [unsteady_dissipation_sum_buf, MPI.DOUBLE],
                                         op=MPI.SUM,
                                         root=0)
                        
                        ## add aggregated sum (across ranks) to the total accumulator
                        unsteady_dissipation_sum += unsteady_dissipation_sum_buf
                        
                        ## release mem
                        duIIdx_ij_duIdx_ij   = None; del duIIdx_ij_duIdx_ij
                        duIIdx_ij_duIdx_ji   = None; del duIIdx_ij_duIdx_ji
                        unsteady_dissipation = None; del unsteady_dissipation
                        
                        self.comm.Barrier()
                        mem_avail_gb = psutil.virtual_memory().available/1024**3
                        mem_free_gb  = psutil.virtual_memory().free/1024**3
                        if verbose:
                            tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                            tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
                    
                    # === transport : T
                    if True:
                        
                        if verbose: tqdm.write(72*'-')
                        t_start = timeit.default_timer()
                        
                        ## triple correlation
                        tc = np.einsum('xyzt,xyzti,xyzti,xyztj->xyztj', rho, uII_i, uII_i, uII_i)
                        
                        #tc_ddx = np.gradient(tc, x, axis=0, edge_order=2)
                        #tc_ddy = np.gradient(tc, y, axis=1, edge_order=2)
                        #tc_ddz = np.gradient(tc, z, axis=2, edge_order=2)
                        
                        tc_ddx = gradient(tc, x, axis=0, acc=acc, edge_stencil=edge_stencil)
                        tc_ddy = gradient(tc, y, axis=1, acc=acc, edge_stencil=edge_stencil)
                        tc_ddz = gradient(tc, z, axis=2, acc=acc, edge_stencil=edge_stencil)
                        
                        unsteady_transport = -0.5*(tc_ddx[:,:,:,:,0] + tc_ddy[:,:,:,:,1] + tc_ddz[:,:,:,:,2])
                        
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('calc transport', format_time_string(t_delta),s=True))
                        
                        ## combine sums across ranks for this [t] chunk
                        unsteady_transport_sum_i   = np.sum(unsteady_transport, axis=3, keepdims=True, dtype=np.float64)
                        unsteady_transport_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                        self.comm.Reduce([unsteady_transport_sum_i, MPI.DOUBLE],
                                         [unsteady_transport_sum_buf, MPI.DOUBLE],
                                         op=MPI.SUM,
                                         root=0)
                        
                        self.comm.Barrier()
                        
                        ## add aggregated sum (across ranks) to the total accumulator
                        unsteady_transport_sum += unsteady_transport_sum_buf
                        
                        ## release mem
                        tc     = None; del tc
                        tc_ddx = None; del tc_ddx
                        tc_ddy = None; del tc_ddy
                        tc_ddz = None; del tc_ddz
                        unsteady_transport = None; del unsteady_transport
                        
                        self.comm.Barrier()
                        mem_avail_gb = psutil.virtual_memory().available/1024**3
                        mem_free_gb  = psutil.virtual_memory().free/1024**3
                        if verbose:
                            tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                            tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
                    
                    # === viscous diffusion : D
                    if True:
                        
                        if verbose: tqdm.write(72*'-')
                        t_start = timeit.default_timer()
                        
                        omega_ij = duIdx_ij + np.transpose(duIdx_ij, axes=(0,1,2,3,5,4))
                        
                        if False:
                            
                            omega_ij_2 = np.stack((np.stack(((duIdx+duIdx), (dvIdx+duIdy), (dwIdx+duIdz)), axis=4),
                                                   np.stack(((duIdy+dvIdx), (dvIdy+dvIdy), (dwIdy+dvIdz)), axis=4),
                                                   np.stack(((duIdz+dwIdx), (dvIdz+dwIdy), (dwIdz+dwIdz)), axis=4)), axis=5)
                            
                            np.testing.assert_allclose(omega_ij, omega_ij_2, rtol=1e-8)
                            
                            if verbose:
                                print('check passed : omega_ij')
                        
                        ## ## this one is likely wrong
                        ## omega_ij = np.stack((np.stack(((duIdx+duIdx), (dvIdx+duIdy), (dwIdx+duIdz)), axis=4),
                        ##                      np.stack(((duIdy+duIdx), (dvIdy+duIdy), (dwIdy+duIdz)), axis=4),
                        ##                      np.stack(((duIdz+duIdx), (dvIdz+duIdy), (dwIdz+duIdz)), axis=4)), axis=5)
                        
                        A = np.einsum('xyzt,xyzti,xyztij->xyztj', mu, uI_i, omega_ij)
                        
                        # A_ddx = np.gradient(A[:,:,:,:,0], x, axis=0, edge_order=2)
                        # A_ddy = np.gradient(A[:,:,:,:,1], y, axis=1, edge_order=2)
                        # A_ddz = np.gradient(A[:,:,:,:,2], z, axis=2, edge_order=2)
                        
                        A_ddx = gradient(A[:,:,:,:,0], x, axis=0, acc=acc, edge_stencil=edge_stencil)
                        A_ddy = gradient(A[:,:,:,:,1], y, axis=1, acc=acc, edge_stencil=edge_stencil)
                        A_ddz = gradient(A[:,:,:,:,2], z, axis=2, acc=acc, edge_stencil=edge_stencil)
                        
                        unsteady_diffusion = A_ddx + A_ddy + A_ddz
                        
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('calc diffusion', format_time_string(t_delta),s=True))
                        
                        self.comm.Barrier()
                        
                        ## combine sums across ranks for this [t] chunk
                        unsteady_diffusion_sum_i   = np.sum(unsteady_diffusion, axis=3, keepdims=True, dtype=np.float64)
                        unsteady_diffusion_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                        self.comm.Reduce([unsteady_diffusion_sum_i, MPI.DOUBLE],
                                         [unsteady_diffusion_sum_buf, MPI.DOUBLE],
                                         op=MPI.SUM,
                                         root=0)
                        
                        ## add aggregated sum (across ranks) to the total accumulator
                        unsteady_diffusion_sum += unsteady_diffusion_sum_buf
                        
                        ## release mem
                        omega_ij = None; del omega_ij
                        A        = None; del A
                        A_ddx    = None; del A_ddx
                        A_ddy    = None; del A_ddy
                        A_ddz    = None; del A_ddz
                        unsteady_diffusion = None; del unsteady_diffusion
                        
                        self.comm.Barrier()
                        mem_avail_gb = psutil.virtual_memory().available/1024**3
                        mem_free_gb  = psutil.virtual_memory().free/1024**3
                        if verbose:
                            tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                            tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
                    
                    # === pressure diffusion
                    if True:
                        
                        if verbose: tqdm.write(72*'-')
                        t_start = timeit.default_timer()
                        
                        A = np.einsum('xyzti,xyzt->xyzti', uII_i, pI)
                        
                        #A_ddx = np.gradient(A[:,:,:,:,0], x, axis=0, edge_order=2)
                        #A_ddy = np.gradient(A[:,:,:,:,1], y, axis=1, edge_order=2)
                        #A_ddz = np.gradient(A[:,:,:,:,2], z, axis=2, edge_order=2)
                        
                        A_ddx = gradient(A[:,:,:,:,0], x, axis=0, acc=acc, edge_stencil=edge_stencil)
                        A_ddy = gradient(A[:,:,:,:,1], y, axis=1, acc=acc, edge_stencil=edge_stencil)
                        A_ddz = gradient(A[:,:,:,:,2], z, axis=2, acc=acc, edge_stencil=edge_stencil)
                        
                        unsteady_p_diffusion = A_ddx + A_ddy + A_ddz
                        
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('calc p_diffusion', format_time_string(t_delta),s=True))
                        
                        self.comm.Barrier()
                        
                        ## combine sums across ranks for this [t] chunk
                        unsteady_p_diffusion_sum_i   = np.sum(unsteady_p_diffusion, axis=3, keepdims=True, dtype=np.float64)
                        unsteady_p_diffusion_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                        self.comm.Reduce([unsteady_p_diffusion_sum_i, MPI.DOUBLE],
                                         [unsteady_p_diffusion_sum_buf, MPI.DOUBLE],
                                         op=MPI.SUM,
                                         root=0)
                        
                        ## add aggregated sum (across ranks) to the total accumulator
                        unsteady_p_diffusion_sum += unsteady_p_diffusion_sum_buf
                        
                        # === release mem
                        A        = None; del A
                        A_ddx    = None; del A_ddx
                        A_ddy    = None; del A_ddy
                        A_ddz    = None; del A_ddz
                        unsteady_p_diffusion = None; del unsteady_p_diffusion
                        
                        self.comm.Barrier()
                        mem_avail_gb = psutil.virtual_memory().available/1024**3
                        mem_free_gb  = psutil.virtual_memory().free/1024**3
                        if verbose:
                            tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                            tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
                    
                    # === pressure dilatation
                    if True:
                        
                        if verbose: tqdm.write(72*'-')
                        t_start = timeit.default_timer()
                        
                        # A = np.einsum('xyzt,xyzti->xyzti', pI, uII_i)
                        # A_ddx = np.gradient(A[:,:,:,:,0], x, axis=0, edge_order=2)
                        # A_ddy = np.gradient(A[:,:,:,:,1], y, axis=1, edge_order=2)
                        # A_ddz = np.gradient(A[:,:,:,:,2], z, axis=2, edge_order=2)
                        # unsteady_p_dilatation = A_ddx + A_ddy + A_ddz
                        
                        unsteady_p_dilatation = pI * ( duIIdx + dvIIdy + dwIIdz )
                        
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('calc p_dilatation', format_time_string(t_delta),s=True))
                        
                        self.comm.Barrier()
                        
                        ## combine sums across ranks for this [t] chunk
                        unsteady_p_dilatation_sum_i   = np.sum(unsteady_p_dilatation, axis=3, keepdims=True, dtype=np.float64)
                        unsteady_p_dilatation_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                        self.comm.Reduce([unsteady_p_dilatation_sum_i, MPI.DOUBLE],
                                         [unsteady_p_dilatation_sum_buf, MPI.DOUBLE],
                                         op=MPI.SUM,
                                         root=0)
                        
                        ## add aggregated sum (across ranks) to the total accumulator
                        unsteady_p_dilatation_sum += unsteady_p_dilatation_sum_buf
                        
                        ## release mem
                        unsteady_p_dilatation = None
                        del unsteady_p_dilatation
                        
                        self.comm.Barrier()
                        mem_avail_gb = psutil.virtual_memory().available/1024**3
                        mem_free_gb  = psutil.virtual_memory().free/1024**3
                        if verbose:
                            tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                            tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
                    
                    ## manual garbage collect
                    #gc.collect()
                    #self.comm.Barrier()
                    
                    if verbose: progress_bar.update()
                    
                    if verbose: tqdm.write(72*'-')
                
                if verbose: progress_bar.close()
        
        # === multiply accumulators by (1/n) to get [t] avg --> leave [x,y,z]
        
        if (self.rank==0):
            production   = np.copy( ((1/nt) * unsteady_production_sum   ) )
            dissipation  = np.copy( ((1/nt) * unsteady_dissipation_sum  ) )
            transport    = np.copy( ((1/nt) * unsteady_transport_sum    ) )
            diffusion    = np.copy( ((1/nt) * unsteady_diffusion_sum    ) )
            p_diffusion  = np.copy( ((1/nt) * unsteady_p_diffusion_sum  ) )
            p_dilatation = np.copy( ((1/nt) * unsteady_p_dilatation_sum ) )
        self.comm.Barrier()
        
        # === write to HDF5 --> [x,y,z,1]
        
        #with rgd(fn_rgd_turb_budget, 'a', driver='mpio', comm=self.comm, libver='latest') as f1:
        if (self.rank==0):
            with rgd(fn_rgd_turb_budget, 'a') as f1:
                f1['data/production'][:,:,:,:]   = production.T.astype(np.float32)
                f1['data/dissipation'][:,:,:,:]  = dissipation.T.astype(np.float32)
                f1['data/transport'][:,:,:,:]    = transport.T.astype(np.float32)
                f1['data/diffusion'][:,:,:,:]    = diffusion.T.astype(np.float32)
                f1['data/p_diffusion'][:,:,:,:]  = p_diffusion.T.astype(np.float32)
                f1['data/p_dilatation'][:,:,:,:] = p_dilatation.T.astype(np.float32)
        self.comm.Barrier()
        
        # === write to .dat --> pre-average in [x,y], leaving [y]
        
        if (self.rank==0):
            
            ## avg in [x,z] --> leave [y]
            data['production']   = np.copy( np.squeeze( np.mean( production   , axis=(0,2) , dtype=np.float64 ) ) )
            data['dissipation']  = np.copy( np.squeeze( np.mean( dissipation  , axis=(0,2) , dtype=np.float64 ) ) )
            data['transport']    = np.copy( np.squeeze( np.mean( transport    , axis=(0,2) , dtype=np.float64 ) ) )
            data['diffusion']    = np.copy( np.squeeze( np.mean( diffusion    , axis=(0,2) , dtype=np.float64 ) ) )
            data['p_diffusion']  = np.copy( np.squeeze( np.mean( p_diffusion  , axis=(0,2) , dtype=np.float64 ) ) )
            data['p_dilatation'] = np.copy( np.squeeze( np.mean( p_dilatation , axis=(0,2) , dtype=np.float64 ) ) )
            
            with open(fn_dat_turb_budget,'wb') as f1:
                pickle.dump(data, f1, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_turb_budget,os.path.getsize(fn_dat_turb_budget)/1024**2))
        self.comm.Barrier()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_turb_budget() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        ## make .xdmf
        with rgd(fn_rgd_turb_budget, 'r', driver='mpio', comm=self.comm, libver='latest') as f1:
            f1.make_xdmf()
        
        return
    
    # === time integration / particle tracking : generate LPD
    
    def time_integrate(self, **kwargs):
        '''
        do Lagrangian-frame time integration of [u,v,w] field
        -----
        --> output LPD (Lagrangian Particle Data) file / lpd() instance
        '''
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.time_integrate()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        force    = kwargs.get('force',False)
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        fn_lpd   = kwargs.get('fn_lpd','pts.h5')
        scheme   = kwargs.get('scheme','RK4')
        npts     = kwargs.get('npts',1e4)
        ntc      = kwargs.get('ntc',None)
        
        if (ntc is not None):
            if not isinstance(ntc,int):
                raise ValueError('ntc should be of type int')
            if (ntc > self.nt):
                raise ValueError('more ts requested than exist')
        
        #rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        #rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        
        #if (rx*ry*rz != self.n_ranks):
        #    raise AssertionError('rx*ry*rz != self.n_ranks')
        if (ry != self.n_ranks):
            raise AssertionError('ry != self.n_ranks')
        #if (rx>self.nx):
        #    raise AssertionError('rx>self.nx')
        #if (ry>self.ny):
        #    raise AssertionError('ry>self.ny')
        #if (rz>self.nz):
        #    raise AssertionError('rz>self.nz')
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # === here we actually want to overlap points i.e. share abutting pts
        
        if (self.rank!=(self.n_ranks-1)):
            ry2 += 1
            nyr += 1
        
        ## these should be abutting
        y_max = self.y[ry1:ry2].max()
        y_min = self.y[ry1:ry2].min()
        
        ## overlap in middle ranks
        if (self.rank==0):
            ry1 -= 0
            ry2 += 6
        elif (self.rank==self.n_ranks-1):
            ry1 -= 6
            ry2 += 0
        else:
            ry1 -= 6
            ry2 += 6
        
        nyr = ry2 - ry1
        
        ## these should be overlapping / intersecting
        y_max_ov = self.y[ry1:ry2].max()
        y_min_ov = self.y[ry1:ry2].min()
        
        ## check rank / grid distribution
        if False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : ry1=%i ry2=%i y_min=%0.8f y_max=%0.8f y_min_ov=%0.8f y_max_ov=%0.8f'%(self.rank,ry1,ry2,y_min,y_max,y_min_ov,y_max_ov))
                    sys.stdout.flush()
            self.comm.Barrier()
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        # === info about the domain
        
        read_dat_mean_dim = False
        
        if read_dat_mean_dim:
            
            fn_dat_mean_dim = None
            
            if (fn_dat_mean_dim is None):
                fname_path = os.path.dirname(self.fname)
                fname_base = os.path.basename(self.fname)
                fname_root, fname_ext = os.path.splitext(fname_base)
                fname_root = re.findall('io\S+_mpi_[0-9]+', fname_root)[0]
                fname_dat_mean_base = fname_root+'_mean_dim.dat'
                fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
            
            if not os.path.isfile(fn_dat_mean_dim):
                raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        if verbose: even_print('fn_rgd'              , self.fname      )
        if read_dat_mean_dim:
            if verbose: even_print('fn_dat_mean_dim' , fn_dat_mean_dim )
        if verbose: even_print('fn_lpd'              , fn_lpd          )
        if verbose: even_print('n ts rgd'            , '%i'%self.nt    )
        
        if read_dat_mean_dim: ## mean dimensional data [x,z]
            
            # === read in data (mean dim) --> every rank gets full [x,z]
            with open(fn_dat_mean_dim,'rb') as f:
                data_mean_dim = pickle.load(f)
            fmd = type('foo', (object,), data_mean_dim)
            
            # === 2D dimensional quantities --> [x,z]
            u_tau    = fmd.u_tau
            nu_wall  = fmd.nu_wall
            d99      = fmd.d99
            u99      = fmd.u99
            Re_tau   = fmd.Re_tau
            Re_theta = fmd.Re_theta
            
            u_tau_avg_end    = np.mean(fmd.u_tau[-1,:]    , axis=(0,))
            nu_wall_avg_end  = np.mean(fmd.nu_wall[-1,:]  , axis=(0,))
            d99_avg_end      = np.mean(fmd.d99[-1,:]      , axis=(0,))
            u99_avg_end      = np.mean(fmd.u99[-1,:]      , axis=(0,))
            Re_tau_avg_end   = np.mean(fmd.Re_tau[-1,:]   , axis=(0,))
            Re_theta_avg_end = np.mean(fmd.Re_theta[-1,:] , axis=(0,))
            
            u_tau_avg_begin    = np.mean(fmd.u_tau[0,:]    , axis=(0,))
            nu_wall_avg_begin  = np.mean(fmd.nu_wall[0,:]  , axis=(0,))
            d99_avg_begin      = np.mean(fmd.d99[0,:]      , axis=(0,))
            u99_avg_begin      = np.mean(fmd.u99[0,:]      , axis=(0,))
            Re_tau_avg_begin   = np.mean(fmd.Re_tau[0,:]   , axis=(0,))
            Re_theta_avg_begin = np.mean(fmd.Re_theta[0,:] , axis=(0,))
            
            # === 2D inner scales --> [x,z]
            sc_l_in = nu_wall / u_tau
            sc_u_in = u_tau
            sc_t_in = nu_wall / u_tau**2
            
            # === 2D outer scales --> [x,z]
            sc_l_out = d99
            sc_u_out = u99
            sc_t_out = d99/u99
            
            # === check
            np.testing.assert_allclose(fmd.lchar   , self.lchar   , rtol=1e-8)
            np.testing.assert_allclose(fmd.U_inf   , self.U_inf   , rtol=1e-8)
            np.testing.assert_allclose(fmd.rho_inf , self.rho_inf , rtol=1e-8)
            np.testing.assert_allclose(fmd.T_inf   , self.T_inf   , rtol=1e-8)
            np.testing.assert_allclose(fmd.nx      , self.nx      , rtol=1e-8)
            np.testing.assert_allclose(fmd.ny      , self.ny      , rtol=1e-8)
            np.testing.assert_allclose(fmd.nz      , self.nz      , rtol=1e-8)
            np.testing.assert_allclose(fmd.xs      , self.x       , rtol=1e-8)
            np.testing.assert_allclose(fmd.ys      , self.y       , rtol=1e-8)
            np.testing.assert_allclose(fmd.zs      , self.z       , rtol=1e-8)
            
            lchar   = self.lchar
            U_inf   = self.U_inf
            rho_inf = self.rho_inf
            T_inf   = self.T_inf
            
            nx = self.nx
            ny = self.ny
            nz = self.nz
            nt = self.nt
            
            ## dimless (inlet)
            xd = self.x
            yd = self.y
            zd = self.z
            td = self.t
            
            ## dimensional [m] / [s]
            x = self.x * lchar 
            y = self.y * lchar
            z = self.z * lchar
            t = self.t * (lchar/U_inf)
            
            t_meas = t[-1]-t[0]
            dt     = self.dt * (lchar/U_inf)
            
            np.testing.assert_equal(nx,x.size)
            np.testing.assert_equal(ny,y.size)
            np.testing.assert_equal(nz,z.size)
            np.testing.assert_equal(nt,t.size)
            np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-8)
            
            # === report
            if verbose:
                even_print('nx'     , '%i'        %nx     )
                even_print('ny'     , '%i'        %ny     )
                even_print('nz'     , '%i'        %nz     )
                even_print('nt'     , '%i'        %nt     )
                even_print('dt'     , '%0.5e [s]' %dt     )
                even_print('t_meas' , '%0.5e [s]' %t_meas )
                even_print('domain x' , '%0.5e [m]' % (x.max()-x.min()) )
                ##
                even_print('U_inf'  , '%0.3f [m/s]'  % U_inf        )
                even_print('U_inf·t_meas/(domain x)' , '%0.3f' % (U_inf*t_meas/(x.max()-x.min())) )
                print(72*'-')
            
            if verbose:
                print('begin x'+'\n'+'-------')
                even_print('Re_τ'   , '%0.1f'        % Re_tau_avg_begin   )
                even_print('Re_θ'   , '%0.1f'        % Re_theta_avg_begin )
                even_print('δ99'    , '%0.5e [m]'    % d99_avg_begin      )
                even_print('u_τ'    , '%0.3f [m/s]'  % u_tau_avg_begin    )
                even_print('ν_wall' , '%0.5e [m²/s]' % nu_wall_avg_begin  )
                even_print('dt+'    , '%0.4f'        % (dt/(nu_wall_avg_begin / u_tau_avg_begin**2)) )
                
                t_eddy_begin = t_meas / (d99_avg_begin/u_tau_avg_begin)
                
                even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy_begin)
                even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99_avg_begin/u99_avg_begin)))
                even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99_avg_begin/u99_avg_begin)))
                ##
                print('end x'+'\n'+'-----')
                even_print('Re_τ'   , '%0.1f'        % Re_tau_avg_end   )
                even_print('Re_θ'   , '%0.1f'        % Re_theta_avg_end )
                even_print('δ99'    , '%0.5e [m]'    % d99_avg_end      )
                even_print('u_τ'    , '%0.3f [m/s]'  % u_tau_avg_end    )
                even_print('ν_wall' , '%0.5e [m²/s]' % nu_wall_avg_end  )
                even_print('dt+'    , '%0.4f'        % (dt/(nu_wall_avg_end / u_tau_avg_end**2)) )
                
                t_eddy_end = t_meas / (d99_avg_end/u_tau_avg_end)
                
                even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy_end)
                even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99_avg_end/u99_avg_end)))
                even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99_avg_end/u99_avg_end)))
                print(72*'-')
        
        # === make initial particle lists
        
        if (ntc is None):
            tc  = np.copy(self.t).astype(np.float64)
            ntc = tc.size
            dtc = tc[1]-tc[0]
        else:
            tc  = np.copy(self.t[:ntc]).astype(np.float64)
            ntc = tc.size
            dtc = tc[1]-tc[0]
        
        if verbose: even_print('n ts for convection' , '%i'%ntc )
        if verbose: print(72*'-')
        
        if False: ## initial points @ grid points
            
            xc = np.copy(self.x)[::5]
            yc = np.copy(self.y[ry1:ry2]) ## [::3]
            zc = np.copy(self.z)[::5]
            
            xxxp, yyyp, zzzp = np.meshgrid(xc, yc, zc, indexing='ij')
            xp = xxxp.ravel(order='F')
            yp = yyyp.ravel(order='F')
            zp = zzzp.ravel(order='F')
            npts = xp.shape[0] ## this rank
            
            ## 4D
            tp = tc[0]*np.ones((npts,), dtype=xp.dtype)
            xyztp = np.stack((xp,yp,zp,tp)).T
            
            ## ## 3D
            ## xyzp = np.stack((xp,yp,zp)).T
            
            ## get total number of particles all ranks
            G = self.comm.gather([npts, self.rank], root=0)
            G = self.comm.bcast(G, root=0)
            npts_total = sum([x[0] for x in G])
            
            ## get the particle ID array for this rank (pnum)
            rp1  = sum([ G[i][0] for i in range(len(G)) if (i<self.rank) ])
            rp2  = rp1 + npts
            pnum = np.arange(rp1,rp2, dtype=np.int64)
            
            ## check particles per rank
            if False:
                for ri in range(self.n_ranks):
                    self.comm.Barrier()
                    if (self.rank == ri):
                        print('rank %04d : %s'%(self.rank,str(pnum)))
                self.comm.Barrier()
            
            if verbose: even_print('n pts','%i'%npts)
            if verbose: even_print('n pts (total)','%i'%npts_total)
            
            npts_total_orig = npts_total
        
        else: ## initial points @ random locations
            
            ## random number generator
            rng = np.random.default_rng(seed=1)
            
            ## pts already in domain at t=0
            if True:
                
                npts_init     = int(round(npts))
                volume_domain = (self.x.max()-self.x.min()) * (self.y.max()-self.y.min()) * (self.z.max()-self.z.min())
                pts_density   = npts_init / volume_domain ## [n pts / m^3]
                area_inlet    = (self.y.max()-self.y.min()) * (self.z.max()-self.z.min())
                volume_flow   = area_inlet * 1 ## still dimless, so U_inf=1 ## [m^2]*[m/s] = [m^3 / s]
                fac           = 0.9500000 ## approximates U(y)dy integral
                volume_flow  *= fac ## since U(y)!=1 across BL
                volume_per_dt = dtc * volume_flow ## [m^3]
                
                pts_per_dt_inlet          = int(round(pts_density*volume_per_dt))
                pts_per_dt_inlet_arr      = pts_per_dt_inlet * np.ones((ntc-1,), dtype=np.int32)
                pts_per_dt_inlet_arr[-5:] = 0 ## dont add any pts in the last N ts
                
                npts_init -= pts_per_dt_inlet ## added back at beginning of loop
                
                if verbose: even_print('volume_domain','%0.5e'%volume_domain)
                if verbose: even_print('pts_density','%0.5e'%pts_density)
                if verbose: even_print('area_inlet','%0.5e'%area_inlet)
                if verbose: even_print('pts_per_dt_inlet','%i'%pts_per_dt_inlet)
                if verbose: print(72*'-')
                
                xp = rng.uniform(self.x.min(), self.x.max(), size=(npts_init,)) ## double precision
                yp = rng.uniform(self.y.min(), self.y.max(), size=(npts_init,))
                zp = rng.uniform(self.z.min(), self.z.max(), size=(npts_init,))
                tp = tc[0]*np.ones((npts_init,), dtype=xp.dtype)
                
                xyztp = np.stack((xp,yp,zp,tp)).T
                
                ## check
                if (xyztp.dtype!=np.float64):
                    raise AssertionError
                
                pnum = np.arange(npts_init, dtype=np.int64)
                
                offset = npts_init
                
                if (self.rank==0):
                    ii = np.where(xyztp[:,1]<=y_max)
                elif (self.rank==self.n_ranks-1):
                    ii = np.where(xyztp[:,1]>y_min)
                else:
                    ii = np.where((xyztp[:,1]>y_min) & (xyztp[:,1]<=y_max))
                
                xyztp = np.copy(xyztp[ii])
                pnum  = np.copy(pnum[ii])
                
                npts_total = npts_init
            
            else: ## no initial particles
                
                xyztp      = None
                pnum       = None
                offset     = 0
                npts_total = 0
                pts_per_dt_inlet = int(10e3)
            
            ## function to replenish pts
            def pts_initializer(rng, npts, tt, offset):
                '''
                the 'new particle' replenishment func
                '''
                zp = rng.uniform(self.z.min(), self.z.max(), size=(npts,))
                yp = rng.uniform(self.y.min(), self.y.max(), size=(npts,))
                xp = 0.5*(self.x[0]+self.x[1]) * np.ones((npts,), dtype=zp.dtype)
                tp = tt * np.ones((npts,), dtype=zp.dtype)
                ##
                xyztp  = np.stack((xp,yp,zp,tp)).T
                
                ## check
                if (xyztp.dtype!=np.float64):
                    raise AssertionError
                
                pnum = int(offset) + np.arange(npts, dtype=np.int64)
                ##
                return xyztp, pnum
        
        ## get the total number of points that will exist for all times
        npts_all_ts = npts_total
        for tci in range(ntc-1):
            xyztp_, pnum_ = pts_initializer(rng, pts_per_dt_inlet_arr[tci], tc[tci], 0)
            npts,_ = xyztp_.shape
            npts_all_ts += npts
        
        # === check if file exists / delete / touch / chmod
        
        ## self.comm.Barrier()
        ## if (self.rank==0):
        ##     if os.path.isfile(fn_lpd):
        ##         os.remove(fn_lpd)
        ##     Path(fn_lpd).touch()
        ##     os.chmod(fn_lpd, int('770', base=8))
        ##     if shutil.which('lfs') is not None:
        ##         return_code = subprocess.call('lfs migrate --stripe-count 16 --stripe-size 8M %s > /dev/null 2>&1'%('particles.h5',), shell=True)
        ## self.comm.Barrier()
        
        # ===
        
        pcomm = MPI.COMM_WORLD
        with lpd(fn_lpd, 'w', force=force, driver='mpio', comm=pcomm, libver='latest') as hf_lpd:
            
            ## 'self' passed here is RGD / h5py.File instance
            ## this copies over all the header info from the RGD: U_inf, lchar, etc
            hf_lpd.init_from_rgd(self, t_info=False)
            
            ## shape & HDF5 chunk scheme for datasets
            shape = (npts_all_ts, ntc-1)
            
            scalars = [ 'x','y','z', 
                        'u','v','w', 
                        't','id'     ]
            
            scalars_dtype = [ np.float64, np.float64, np.float64, 
                              np.float32, np.float32, np.float32, 
                              np.float64, np.int64                 ]
            
            for si in range(len(scalars)):
                
                scalar       = scalars[si]
                scalar_dtype = scalars_dtype[si]
                
                itemsize = np.dtype(scalar_dtype).itemsize
                
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,1), size_kb=chunk_kb, base=2, itemsize=itemsize)
                
                if verbose:
                    even_print('initializing',scalar)
                
                if ('data/%s'%scalar in hf_lpd):
                    del hf_lpd['data/%s'%scalar]
                dset = hf_lpd.create_dataset('data/%s'%scalar, 
                                          shape=shape, 
                                          dtype=scalar_dtype,
                                          fillvalue=np.nan,
                                          #compression='gzip', ## this causes segfaults :( :(
                                          #compression_opts=5,
                                          #shuffle=True,
                                          chunks=chunks,
                                          )
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (pts,nt)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            ## write time vector
            if ('dims/t' in hf_lpd):
                del hf_lpd['dims/t']
            hf_lpd.create_dataset('dims/t', data=tc[:-1], dtype=tc.dtype, chunks=True)
            
            pcomm.Barrier()
            self.comm.Barrier()
            
            if verbose:
                print(72*'-')
                even_print( os.path.basename(fn_lpd) , '%0.2f [GB]'%(os.path.getsize(fn_lpd)/1024**3))
                print(72*'-')
            
            if True: ## convect fwd
                
                if verbose:
                    progress_bar = tqdm(total=ntc-1, ncols=100, desc='convect fwd', leave=False, file=sys.stdout)
                
                t_write = 0.
                
                if (dtc!=self.dt):
                    raise AssertionError('dtc!=self.dt')
                
                ## the global list of all particle IDs that have left volume
                pnum_nan_global = np.array([],dtype=np.int64)
                
                for tci in range(ntc-1):
                    
                    if verbose:
                        if (tci>0):
                            tqdm.write('---')
                    
                    ## get global new pts this ts
                    xyztp_new, pnum_new = pts_initializer(rng, pts_per_dt_inlet_arr[tci], tc[tci], offset)
                    npts_new, _ = xyztp_new.shape
                    offset += npts_new
                    
                    ## take pts in rank bounds
                    if (self.rank==0):
                        ii = np.where(xyztp_new[:,1]<=y_max)
                    elif (self.rank==self.n_ranks-1):
                        ii = np.where(xyztp_new[:,1]>y_min)
                    else:
                        ii = np.where((xyztp_new[:,1]>y_min) & (xyztp_new[:,1]<=y_max))
                    ##
                    xyztp = np.concatenate((xyztp, xyztp_new[ii]), axis=0, casting='no')
                    pnum  = np.concatenate((pnum,  pnum_new[ii]),  axis=0, casting='no')
                    
                    if verbose: tqdm.write(even_print('tci', '%i'%(tci,), s=True))
                    
                    # ===
                    
                    ti1  = tci
                    ti2  = tci+1+1
                    tseg = tc[ti1:ti2]
                    
                    ## RK4 times
                    tbegin = tseg[0]
                    tend   = tseg[-1]
                    tmid   = 0.5*(tbegin+tend)
                    
                    ## update pts list time
                    xyztp[:,3] = tc[tci]
                    
                    ## assert : tc[tci] == segment begin time
                    if not np.allclose(tc[tci], tbegin, rtol=1e-8):
                        raise AssertionError('tc[tci] != tbegin')
                    
                    # === read RGD rectilinear data
                    
                    scalars = ['u','v','w']
                    scalars_dtypes = [np.float32 for s in scalars]
                    data = np.zeros(shape=(self.nx, nyr, self.nz, 2), dtype={'names':scalars, 'formats':scalars_dtypes})
                    
                    for scalar in scalars:
                        
                        data_gb = 4*self.nx*self.ny*self.nz*2 / 1024**3
                        
                        dset = self['data/%s'%scalar]
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            data[scalar] = dset[ti1:ti2,:,ry1:ry2,:].T
                        
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if verbose:
                            tqdm.write(even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)) 
                        pass
                    
                    # ===
                    
                    ## ## make 4D scalar interpolators
                    ## f_u = sp.interpolate.RegularGridInterpolator((self.x,self.y[ry1:ry2],self.z,self.t[ti1:ti2]), data['u'], method='linear', fill_value=np.nan, bounds_error=False)
                    ## f_v = sp.interpolate.RegularGridInterpolator((self.x,self.y[ry1:ry2],self.z,self.t[ti1:ti2]), data['v'], method='linear', fill_value=np.nan, bounds_error=False)
                    ## f_w = sp.interpolate.RegularGridInterpolator((self.x,self.y[ry1:ry2],self.z,self.t[ti1:ti2]), data['w'], method='linear', fill_value=np.nan, bounds_error=False)
                    
                    uvw   = np.stack((data['u'], data['v'], data['w']), axis=4)
                    f_uvw = sp.interpolate.RegularGridInterpolator((self.x,self.y[ry1:ry2],self.z,self.t[ti1:ti2]), uvw, method='linear', fill_value=np.nan, bounds_error=False)
                    
                    # === Trilinear interpolation at beginning of time segment (1/2) --> this may have NaNs
                    
                    xyztp_k1 = np.copy(xyztp)
                    ## u_k1     = f_u(xyztp_k1)
                    ## v_k1     = f_v(xyztp_k1)
                    ## w_k1     = f_w(xyztp_k1)
                    
                    uvw_k1 = f_uvw(xyztp_k1)
                    u_k1 = uvw_k1[:,0]
                    
                    x_ = np.copy( xyztp[:,0] )
                    y_ = np.copy( xyztp[:,1] )
                    z_ = np.copy( xyztp[:,2] )
                    t_ = np.copy( xyztp[:,3] )
                    
                    # === separate out NaN positions / velocities
                    
                    ii_nan    = np.where(  np.isnan(x_) |  np.isnan(u_k1) )
                    ii_notnan = np.where( ~np.isnan(x_) & ~np.isnan(u_k1) )
                    ##
                    pnum_nan  = np.copy(pnum[ii_nan])
                    xyztp_nan = np.copy(xyztp[ii_nan])
                    ##
                    G = self.comm.gather([np.copy(pnum_nan), self.rank], root=0)
                    G = self.comm.bcast(G, root=0)
                    pnum_nan_global_this_ts = np.concatenate( [g[0] for g in G] , casting='no' )
                    pnum_nan_global         = np.concatenate( (pnum_nan_global_this_ts , pnum_nan_global) , casting='no' )
                    ##
                    npts_nan = pnum_nan_global.shape[0]
                    
                    ## take only non-NaN position particles
                    pnum  = np.copy(pnum[ii_notnan])
                    xyztp = np.copy(xyztp[ii_notnan])
                    
                    if True: ## check global pnum (pt id) vector
                        
                        G = self.comm.gather([np.copy(pnum), self.rank], root=0)
                        G = self.comm.bcast(G, root=0)
                        pnum_global = np.sort( np.concatenate( [g[0] for g in G] , casting='no' ) )
                        pnum_global = np.sort( np.concatenate((pnum_global,pnum_nan_global), casting='no') )
                        
                        ## make sure that the union of the current (non-nan) IDs and nan IDs is 
                        ##    equal to the arange of the total number of particles instantiated to
                        ##    this point
                        if not np.array_equal(pnum_global, np.arange(offset, dtype=np.int64)):
                            raise AssertionError('pnum_global!=np.arange(offset, dtype=np.int64)')
                        
                        if verbose: tqdm.write(even_print('pnum check', 'passed', s=True)) 
                    
                    # === Trilinear interpolation at beginning of time segment (2/2) --> again after NaN filter
                    
                    xyztp_k1 = np.copy(xyztp)
                    ## u_k1     = f_u(xyztp_k1)
                    ## v_k1     = f_v(xyztp_k1)
                    ## w_k1     = f_w(xyztp_k1)
                    uvw_k1   = f_uvw(xyztp_k1)
                    u_k1     = uvw_k1[:,0]
                    v_k1     = uvw_k1[:,1]
                    w_k1     = uvw_k1[:,2]
                    
                    x_ = np.copy( xyztp[:,0] )
                    y_ = np.copy( xyztp[:,1] )
                    z_ = np.copy( xyztp[:,2] )
                    t_ = np.copy( xyztp[:,3] )
                    
                    ## this passes
                    if False:
                        if np.isnan(np.min(u_k1)):
                            print('u_k1 has NaNs')
                            pcomm.Abort(1)
                        if np.isnan(np.min(v_k1)):
                            print('v_k1 has NaNs')
                            pcomm.Abort(1)
                        if np.isnan(np.min(w_k1)):
                            print('w_k1 has NaNs')
                            pcomm.Abort(1)
                    
                    # === Gather/Bcast all data
                    
                    pcomm.Barrier()
                    t_start = timeit.default_timer()
                    
                    G = self.comm.gather([self.rank, pnum, x_, y_, z_, t_, u_k1, v_k1, w_k1], root=0)
                    G = self.comm.bcast(G, root=0)
                    
                    pcomm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    if verbose:
                        tqdm.write(even_print('MPI Gather/Bcast', '%0.2f [s]'%(t_delta,), s=True))
                    
                    npts_total = sum([g[1].shape[0] for g in G])
                    pnum_gl    = np.concatenate([g[1] for g in G], axis=0, casting='no',        dtype=np.int64   )
                    x_gl       = np.concatenate([g[2] for g in G], axis=0, casting='no',        dtype=np.float64 )
                    y_gl       = np.concatenate([g[3] for g in G], axis=0, casting='no',        dtype=np.float64 )
                    z_gl       = np.concatenate([g[4] for g in G], axis=0, casting='no',        dtype=np.float64 )
                    t_gl       = np.concatenate([g[5] for g in G], axis=0, casting='no',        dtype=np.float64 )
                    u_gl       = np.concatenate([g[6] for g in G], axis=0, casting='same_kind', dtype=np.float32 )
                    v_gl       = np.concatenate([g[7] for g in G], axis=0, casting='same_kind', dtype=np.float32 )
                    w_gl       = np.concatenate([g[8] for g in G], axis=0, casting='same_kind', dtype=np.float32 )
                    ##
                    if verbose: tqdm.write(even_print('n pts initialized', '%i'%(offset,),      s=True))
                    if verbose: tqdm.write(even_print('n pts in domain',   '%i'%(npts_total,),  s=True))
                    if verbose: tqdm.write(even_print('n pts left domain', '%i'%(npts_nan,),    s=True))
                    if verbose: tqdm.write(even_print('n pts all time',    '%i'%(npts_all_ts,), s=True))
                    
                    # === add NaN IDs, pad scalar vectors, do sort
                    
                    pnum_gl = np.concatenate([pnum_gl,pnum_nan_global], axis=0, casting='no', dtype=np.int64 )
                    npts_total_incl_nan = pnum_gl.shape[0]
                    
                    if (npts_total_incl_nan!=offset):
                        raise AssertionError('npts_total_incl_nan!=offset')
                    
                    nanpad_f32 = np.empty( (npts_nan,), dtype=np.float32 ); nanpad_f32[:] = np.nan
                    nanpad_f64 = np.empty( (npts_nan,), dtype=np.float64 ); nanpad_f64[:] = np.nan
                    #nanpad_i64 = np.empty( (npts_nan,), dtype=np.int64   ); nanpad_i64[:] = np.nan
                    
                    x_gl = np.concatenate( [x_gl,nanpad_f64], axis=0, dtype=np.float64, casting='no' )
                    y_gl = np.concatenate( [y_gl,nanpad_f64], axis=0, dtype=np.float64, casting='no' )
                    z_gl = np.concatenate( [z_gl,nanpad_f64], axis=0, dtype=np.float64, casting='no' )
                    t_gl = np.concatenate( [t_gl,nanpad_f64], axis=0, dtype=np.float64, casting='no' )
                    u_gl = np.concatenate( [u_gl,nanpad_f32], axis=0, dtype=np.float32, casting='no' )
                    v_gl = np.concatenate( [v_gl,nanpad_f32], axis=0, dtype=np.float32, casting='no' )
                    w_gl = np.concatenate( [w_gl,nanpad_f32], axis=0, dtype=np.float32, casting='no' )
                    ##
                    sort_order = np.argsort(pnum_gl, axis=0)
                    pnum_gl    = np.copy(pnum_gl[sort_order])
                    x_gl       = np.copy(x_gl[sort_order])
                    y_gl       = np.copy(y_gl[sort_order])
                    z_gl       = np.copy(z_gl[sort_order])
                    t_gl       = np.copy(t_gl[sort_order])
                    u_gl       = np.copy(u_gl[sort_order])
                    v_gl       = np.copy(v_gl[sort_order])
                    w_gl       = np.copy(w_gl[sort_order])
                    
                    ## check that the global particle number / ID vector is simply arange(total an pts created)
                    if not np.array_equal(pnum_gl, np.arange(offset, dtype=np.int64)):
                        raise AssertionError('pnum_gl!=np.arange(offset, dtype=np.int64)')
                    
                    ## check that all particle times for this ts are equal --> passes
                    if False:
                        ii_notnan = np.where(~np.isnan(t_gl))
                        if not np.all( np.isclose(t_gl[ii_notnan], t_gl[ii_notnan][0], rtol=1e-14) ):
                            raise AssertionError('not all times are the same at this time integration step --> check!!!')
                    
                    # === get collective write bounds
                    
                    rpl_ = np.array_split(np.arange(npts_total_incl_nan,dtype=np.int64) , self.n_ranks)
                    rpl = [[b[0],b[-1]+1] for b in rpl_ ]
                    rp1,rp2 = rpl[self.rank]
                    
                    # === write
                    
                    for key, value in {'id':pnum_gl, 'x':x_gl, 'y':y_gl, 'z':z_gl, 't':t_gl, 'u':u_gl, 'v':v_gl, 'w':w_gl}.items():
                        
                        data_gb = value.itemsize * npts_total_incl_nan / 1024**3
                        
                        dset = hf_lpd['data/%s'%key]
                        pcomm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            dset[rp1:rp2,tci] = value[rp1:rp2]
                        pcomm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        t_write += t_delta
                        if verbose:
                            tqdm.write(even_print('write: %s'%key, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)) 
                    
                    # ===
                    
                    ## ## write the IDs of the particles which have left the domain for sort in a second step
                    ## ## --> this is now done in the step above
                    ## if (npts_nan>0):
                    ##     dset = hf_lpd['data/id']
                    ##     with dset.collective:
                    ##         dset[-npts_nan:,tci] = pnum_nan_global.astype(np.float32)
                    
                    # === Time Integration
                    
                    pcomm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if (scheme=='RK4'):
                        
                        xyztp_    = np.copy(xyztp)
                        xyztp_k2  = np.zeros_like(xyztp)
                        xyztp_k3  = np.zeros_like(xyztp)
                        xyztp_k4  = np.zeros_like(xyztp)
                        
                        xyztp_k2[:,0] = xyztp_[:,0] + u_k1*dtc*0.5
                        xyztp_k2[:,1] = xyztp_[:,1] + v_k1*dtc*0.5
                        xyztp_k2[:,2] = xyztp_[:,2] + w_k1*dtc*0.5
                        xyztp_k2[:,3] = tmid
                        ## u_k2          = f_u(xyztp_k2)
                        ## v_k2          = f_v(xyztp_k2)
                        ## w_k2          = f_w(xyztp_k2)
                        uvw_k2 = f_uvw(xyztp_k2)
                        u_k2   = uvw_k2[:,0]
                        v_k2   = uvw_k2[:,1]
                        w_k2   = uvw_k2[:,2]
                        
                        xyztp_k3[:,0] = xyztp_[:,0] + u_k2*dtc*0.5
                        xyztp_k3[:,1] = xyztp_[:,1] + v_k2*dtc*0.5
                        xyztp_k3[:,2] = xyztp_[:,2] + w_k2*dtc*0.5
                        xyztp_k3[:,3] = tmid
                        ## u_k3          = f_u(xyztp_k3)
                        ## v_k3          = f_v(xyztp_k3)
                        ## w_k3          = f_w(xyztp_k3)
                        uvw_k3 = f_uvw(xyztp_k3)
                        u_k3   = uvw_k3[:,0]
                        v_k3   = uvw_k3[:,1]
                        w_k3   = uvw_k3[:,2]
                        
                        xyztp_k4[:,0] = xyztp_[:,0] + u_k3*dtc*1.0
                        xyztp_k4[:,1] = xyztp_[:,1] + v_k3*dtc*1.0
                        xyztp_k4[:,2] = xyztp_[:,2] + w_k3*dtc*1.0
                        xyztp_k4[:,3] = tend
                        ## u_k4          = f_u(xyztp_k4)
                        ## v_k4          = f_v(xyztp_k4)
                        ## w_k4          = f_w(xyztp_k4)
                        uvw_k4 = f_uvw(xyztp_k4)
                        u_k4   = uvw_k4[:,0]
                        v_k4   = uvw_k4[:,1]
                        w_k4   = uvw_k4[:,2]
                        
                        ## the vel components (RK4 weighted avg) for time integration
                        u = (1./6.) * (1.*u_k1 + 2.*u_k2 + 2.*u_k3 + 1.*u_k4)
                        v = (1./6.) * (1.*v_k1 + 2.*v_k2 + 2.*v_k3 + 1.*v_k4)
                        w = (1./6.) * (1.*w_k1 + 2.*w_k2 + 2.*w_k3 + 1.*w_k4)
                    
                    elif (scheme=='Euler Explicit'):
                        
                        u = u_k1
                        v = v_k1
                        w = w_k1
                    
                    else:
                        raise NotImplementedError('integration scheme \'%s\' not valid. options are: \'Euler Explicit\', \'RK4\''%scheme)
                    
                    ## actual time integration with higher order 'average' [u,v,w] over time segment 
                    xyztp[:,0] = xyztp[:,0] + u*dtc
                    xyztp[:,1] = xyztp[:,1] + v*dtc
                    xyztp[:,2] = xyztp[:,2] + w*dtc
                    
                    pcomm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    t_write += t_delta
                    if verbose:
                        tqdm.write(even_print('%s'%scheme, '%0.2f [s]'%(t_delta,), s=True))
                    
                    # === MPI Gather/Bcast points between domains
                    
                    ## get pts that leave rank bounds in dim [y]
                    i_exit_top = np.where(xyztp[:,1]>y_max)
                    i_exit_bot = np.where(xyztp[:,1]<y_min)
                    
                    ## lists for the pts that leave
                    xyztp_out = np.concatenate((np.copy(xyztp[i_exit_top]) , np.copy(xyztp[i_exit_bot])), axis=0, casting='no' )
                    pnum_out  = np.concatenate((np.copy(pnum[i_exit_top])  , np.copy(pnum[i_exit_bot])),  axis=0, casting='no' )
                    
                    ## delete those from local lists
                    i_del = np.concatenate( (i_exit_top[0], i_exit_bot[0]) , axis=0, casting='no' )
                    xyztp = np.delete(xyztp, (i_del,), axis=0)
                    pnum  = np.delete(pnum,  (i_del,), axis=0)
                    
                    # === MPI : Gather/Bcast all inter-domain pts
                    G = self.comm.gather([np.copy(xyztp_out), np.copy(pnum_out), self.rank], root=0)
                    G = self.comm.bcast(G, root=0)
                    xyztpN = np.concatenate([x[0] for x in G], axis=0, casting='no' )
                    pnumN  = np.concatenate([x[1] for x in G], axis=0, casting='no' )
                    
                    ## get indices to 'take' from inter-domain points
                    if (self.rank==0):
                        i_take = np.where(xyztpN[:,1]<=y_max)
                    elif (self.rank==self.n_ranks-1):
                        i_take = np.where(xyztpN[:,1]>=y_min)
                    else:
                        i_take = np.where((xyztpN[:,1]<y_max) & (xyztpN[:,1]>=y_min))
                    ##
                    xyztp = np.concatenate((xyztp, xyztpN[i_take]), axis=0, casting='no' )
                    pnum  = np.concatenate((pnum,  pnumN[i_take]),  axis=0, casting='no' )
                    
                    if verbose:
                        progress_bar.update()
                    
                    self.comm.Barrier()
                    pcomm.Barrier()
                
                if verbose:
                    progress_bar.close()
        
        if verbose: print(72*'-'+'\n')
        
        pcomm.Barrier()
        self.comm.Barrier()
        
        ## make XDMF/XMF2
        with lpd(fn_lpd, 'r', driver='mpio', comm=pcomm) as hf_lpd:
            hf_lpd.make_xdmf()
        
        # ===
        
        if verbose: print('\n'+72*'-')
        if verbose: print('total time : rgd.time_integrate() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # === testing & assertion
    
    def check_grid_validity(self,):
        '''
        check grid validity for RGD file
        - monotonically increasing
        - no Δ=0 
        '''
        
        verbose = True
        
        if verbose: print('\n'+'rgd.check_grid_validity()'+'\n'+72*'-')
        
        x = np.copy(self.x)
        y = np.copy(self.y)
        z = np.copy(self.z)
        t = np.copy(self.t)
        
        dims = {'x':x,'y':y,'z':z,'t':t}
        
        for dn, d in dims.items():
            
            if (d.shape[0]>1):
                
                ## check no zero distance elements
                if (np.diff(d).size - np.count_nonzero(np.diff(d))) != 0.:
                    #raise AssertionError('%s arr has zero-distance elements'%dn)
                    if verbose: even_print('check: Δ%s!=0'%dn,'failed')
                else:
                    if verbose: even_print('check: Δ%s!=0'%dn,'passed')
                
                ## check monotonically increasing
                if not np.all(np.diff(d) > 0.):
                    #raise AssertionError('%s arr not monotonically increasing'%dn)
                    if verbose: even_print('check: %s mono increasing'%dn,'failed')
                else:
                    if verbose: even_print('check: %s mono increasing'%dn,'passed')
            
            else:
                if verbose: print('dim %s has size 1'%dn)
        
        if verbose: print(72*'-')
        
        return
    
    @staticmethod
    def assert_4d_datasets(fn_h5_A, fn_h5_B, **kwargs):
        '''
        assert that 4D datasets data/<scalar> are the same
        '''
        
        verbose = kwargs.get('verbose',True)
        
        try:
            #comm    = MPI.COMM_WORLD
            rank    = MPI.COMM_WORLD.Get_rank()
            n_ranks = MPI.COMM_WORLD.Get_size()
        except:
            rank = 0
            n_ranks = 1
        
        if (rank==0):
            verbose = True
        else:
            verbose = False
        
        with rgd(fn_h5_A,'r') as fA:
            with rgd(fn_h5_B,'r') as fB:
                
                if verbose: print('\n'+'rgd.assert_4d_datasets()'+'\n'+72*'-')
                t_start_func = timeit.default_timer()
                
                if (fA.fclass != fB.fclass):
                    raise ValueError('fclass not same')
                if verbose: even_print('fclass',str(fA.fclass))
                if (fA.nx != fB.nx):
                    raise ValueError('nx not same')
                if verbose: even_print('nx','%i'%fA.nx)
                if (fA.ny != fB.ny):
                    raise ValueError('ny not same')
                if verbose: even_print('ny','%i'%fA.ny)
                if (fA.nz != fB.nz):
                    raise ValueError('nz not same')
                if verbose: even_print('nz','%i'%fA.nz)
                if (fA.nt != fB.nt):
                    raise ValueError('nt not same')
                if verbose: even_print('nt','%i'%fA.nt)
                if verbose: print(72*'-')
                
                scalars_A = sorted( fA.scalars )
                scalars_B = sorted( fB.scalars )
                
                #if (scalars_A != scalars_B):
                #    raise ValueError('scalars not same')
                
                if (scalars_A == scalars_B):
                    if verbose: even_print('scalars same','True')
                    if verbose: even_print('n scalars','%i'%(len(scalars_A),))
                else:
                    if verbose: even_print('scalars same','False')
                    if verbose: even_print('n scalars A','%i'%(len(fA.scalars),))
                    if verbose: even_print('n scalars B','%i'%(len(fB.scalars),))
                if verbose: print(72*'-')
                
                for scalar in fA.scalars:
                    if scalar in fB.scalars:
                        
                        dsA = fA[f'data/{scalar}']
                        dsB = fB[f'data/{scalar}']
                        
                        for ti_ in range(fA.nt):
                            
                            tiA = fA.ti[ti_]
                            tiB = fB.ti[ti_]
                            dA = np.copy( dsA[tiA,:,:,:].T )
                            dB = np.copy( dsB[tiB,:,:,:].T )
                            
                            if (dA.dtype==np.float64) and (dB.dtype==np.float64):
                                tol = 1e-12
                            else:
                                tol = 1e-6
                            
                            ## this might fail if all 0s
                            np.testing.assert_allclose( np.abs(dA).max() , np.abs(dB).max(), rtol=tol )
                            
                            rdiff_abs_max = np.abs(dA-dB).max()
                            if verbose: even_print(f'max(abs(Δ{scalar}))',f'{rdiff_abs_max:0.3e}')
                            
                            np.testing.assert_allclose( rdiff_abs_max, 0., atol=tol )
        
        # self.comm.Barrier()
        
        #if verbose: print('\n'+72*'-')
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.assert_4d_datasets() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from RGD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        makeVectors = kwargs.get('makeVectors',True) ## write vectors (e.g. velocity, vorticity) to XDMF
        makeTensors = kwargs.get('makeTensors',True) ## write 3x3 tensors (e.g. stress, strain) to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if verbose: print('\n'+'rgd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for scalar in ['x','y','z']:
            if ('dims/'+scalar in self):
                data = self['dims/'+scalar]
                dataset_precision_dict[scalar] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[scalar] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[scalar] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        # scalar names dict
        # --> labels for Paraview could be customized (e.g. units could be added) using a dict
        # --> the block below shows one such example dict, though it is currently inactive
        
        if False:
            units = 'dimless'
            if (units=='SI') or (units=='si'): ## m,s,kg,K
                scalar_names = {'x':'x [m]',
                                'y':'y [m]',
                                'z':'z [m]', 
                                'u':'u [m/s]',
                                'v':'v [m/s]',
                                'w':'w [m/s]', 
                                'T':'T [K]',
                                'rho':'rho [kg/m^3]',
                                'p':'p [Pa]'}
            elif (units=='dimless') or (units=='dimensionless'):
                scalar_names = {'x':'x [dimless]',
                                'y':'y [dimless]',
                                'z':'z [dimless]', 
                                'u':'u [dimless]',
                                'v':'v [dimless]',
                                'w':'w [dimless]',
                                'T':'T [dimless]',
                                'rho':'rho [dimless]',
                                'p':'p [dimless]'}
            else:
                raise ValueError('choice of units not recognized : %s --> options are : %s / %s'%(units,'SI','dimless'))
        else:
            scalar_names = {} ## dummy/empty 
        
        ## refresh header
        self.get_header(verbose=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')
        
        # === write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                ## Dimensions can also be NumberOfElements
                xdmf_str=f'''
                         <Topology TopologyType="3DRectMesh" NumberOfElements="{self.nz:d} {self.ny:d} {self.nx:d}"/>
                         <Geometry GeometryType="VxVyVz">
                           <DataItem Dimensions="{self.nx:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                             {fname_base}:/dims/{'x'}
                           </DataItem>
                           <DataItem Dimensions="{self.ny:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                             {fname_base}:/dims/{'y'}
                           </DataItem>
                           <DataItem Dimensions="{self.nz:d}" NumberType="{dataset_numbertype_dict['z']}" Precision="{dataset_precision_dict['z']:d}" Format="HDF">
                             {fname_base}:/dims/{'z'}
                           </DataItem>
                         </Geometry>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # =====
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===== .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        
                        dset_hf_path = 'data/%s'%scalar
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        if scalar in scalar_names:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        xdmf_str=f'''
                                 <!-- ===== scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                     <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                       {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                       {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                       {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                     </DataItem>
                                     <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                       {fname_base}:/{dset_hf_path}
                                     </DataItem>
                                   </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    if makeVectors:
                        
                        # === .xdmf : <Grid> per vector : velocity vector
                        
                        if ('u' in self.scalars) and ('v' in self.scalars) and ('w' in self.scalars):
                            
                            scalar_name    = 'velocity'
                            dset_hf_path_i = 'data/u'
                            dset_hf_path_j = 'data/v'
                            dset_hf_path_k = 'data/w'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['u']}" Precision="{dataset_precision_dict['u']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['v']}" Precision="{dataset_precision_dict['v']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['w']}" Precision="{dataset_precision_dict['w']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                        
                        # === .xdmf : <Grid> per vector : vorticity vector
                        
                        if ('vort_x' in self.scalars) and ('vort_y' in self.scalars) and ('vort_z' in self.scalars):
                            
                            scalar_name    = 'vorticity'
                            dset_hf_path_i = 'data/vort_x'
                            dset_hf_path_j = 'data/vort_y'
                            dset_hf_path_k = 'data/vort_z'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_x']}" Precision="{dataset_precision_dict['vort_x']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_y']}" Precision="{dataset_precision_dict['vort_y']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_z']}" Precision="{dataset_precision_dict['vort_z']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    if makeTensors:
                        if all([('dudx' in self.scalars),('dvdx' in self.scalars),('dwdx' in self.scalars),
                                ('dudy' in self.scalars),('dvdy' in self.scalars),('dwdy' in self.scalars),
                                ('dudz' in self.scalars),('dvdz' in self.scalars),('dwdz' in self.scalars)]):
                            pass
                            pass ## TODO
                            pass
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if verbose: print('--w-> %s'%fname_xdmf_base)
        return

class eas4(h5py.File):
    '''
    Interface class for EAS4 files
    ------------------------------
    - super()'ed h5py.File class
    - EAS4 is the HDF5-based output format from the flow solver NS3D
    - 3D dataset storage ([x,y,z] per [t])
    '''
    
    def __init__(self, *args, **kwargs):
        
        ## if grid MUST be read as 3D, i.e. the GMODE=(5,5,5), only read if this is TRUE
        ## this can lead to huge RAM usage in MPI mode, so OFF by default
        ##
        ## if a single dimension has GMODE=5 but not ALL, i.e. (5,5,2), this allows for
        ## grid to be read as some combination of 2D&1D
        self.read_3d_grid = kwargs.pop('read_3d_grid', False)
        
        self.fname, openMode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        if (openMode!='r'):
            raise ValueError('turbx.eas4(): opening EAS4 in anything but read mode \'r\' is not allowed!')
        
        ## catch possible user error --> user tries to open non-EAS4 with turbx.eas4()
        if (self.fname_ext!='.eas'):
            raise ValueError('turbx.eas4() should not be used to open non-EAS4 files')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
            if ('comm' in kwargs):
                del kwargs['comm']
        
        ## set library version to latest (if not otherwise set)
        if ('libver' not in kwargs):
            kwargs['libver']='latest'
        
        ## determine MPI info / hints
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(8*1024**2))) ) ## 8 [MB]
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        self.domainName = 'DOMAIN_000000' ## turbx only handles one domain for now
        
        ## eas4() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop('verbose',False)
        self.verbose = verbose
        ## force = kwargs.pop('force',False) ## --> dont need, always read-only!
        
        ## call actual h5py.File.__init__()
        super(eas4, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(eas4, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed EAS4 HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(eas4, self).__exit__()
    
    def get_header(self, **kwargs):
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        # === characteristic values
        
        if self.verbose: print(72*'-')
        Ma    = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['Ma'][0]
        Re    = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['Re'][0]
        Pr    = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['Pr'][0]
        kappa = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['kappa'][0]
        R     = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['R'][0]
        p_inf = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['p_inf'][0]
        T_inf = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['T_inf'][0]
        # !!!!! ----- ACHTUNG ACHTUNG ACHTUNG ----- !!!!! #
        C_Suth_in_file_which_is_mislabelled = self['Kennsatz']['VISCOUS_PROPERTIES'].attrs['C_Suth'][0]
        S_Suth      = C_Suth_in_file_which_is_mislabelled
        # !!!!! ----- ACHTUNG ACHTUNG ACHTUNG ----- !!!!! #
        mu_Suth_ref = self['Kennsatz']['VISCOUS_PROPERTIES'].attrs['mu_Suth_ref'][0]
        T_Suth_ref  = self['Kennsatz']['VISCOUS_PROPERTIES'].attrs['T_Suth_ref'][0]
        
        C_Suth = mu_Suth_ref/(T_Suth_ref**(3/2))*(T_Suth_ref + S_Suth) ## [kg/(m·s·√K)]
        
        if self.verbose: even_print('Ma'          , '%0.2f [-]'           % Ma          )
        if self.verbose: even_print('Re'          , '%0.1f [-]'           % Re          )
        if self.verbose: even_print('Pr'          , '%0.3f [-]'           % Pr          )
        if self.verbose: even_print('T_inf'       , '%0.3f [K]'           % T_inf       )
        if self.verbose: even_print('p_inf'       , '%0.1f [Pa]'          % p_inf       )
        if self.verbose: even_print('kappa'       , '%0.3f [-]'           % kappa       )
        if self.verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % R           )
        if self.verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % mu_Suth_ref )
        if self.verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % T_Suth_ref  )
        if self.verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % C_Suth      )
        if self.verbose: even_print('S_Suth'      , '%0.2f [K]'           % S_Suth      )
        
        # === characteristic values : derived
        
        if self.verbose: print(72*'-')
        rho_inf = p_inf / ( R * T_inf )
        
        mu_inf_1 = 14.58e-7*T_inf**1.5/(T_inf+110.4)
        mu_inf_2 = mu_Suth_ref*(T_inf/T_Suth_ref)**(3/2)*(T_Suth_ref+S_Suth)/(T_inf+S_Suth)
        mu_inf_3 = C_Suth*T_inf**(3/2)/(T_inf+S_Suth)
        
        if not np.isclose(mu_inf_1, mu_inf_2, rtol=1e-08):
            raise AssertionError('inconsistency in Sutherland calc --> check')
        if not np.isclose(mu_inf_2, mu_inf_3, rtol=1e-08):
            raise AssertionError('inconsistency in Sutherland calc --> check')
        
        mu_inf    = mu_inf_3
        nu_inf    = mu_inf/rho_inf
        a_inf     = np.sqrt(kappa*R*T_inf)
        U_inf     = Ma*a_inf
        cp        = R*kappa/(kappa-1.)
        cv        = cp/kappa
        recov_fac = Pr**(1/3)
        Tw        = T_inf
        Taw       = T_inf + recov_fac*U_inf**2/(2*cp)
        lchar     = Re * nu_inf / U_inf
        
        if self.verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % rho_inf   )
        if self.verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % mu_inf    )
        if self.verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % nu_inf    )
        if self.verbose: even_print('a_inf'           , '%0.6f [m/s]'      % a_inf     )
        if self.verbose: even_print('U_inf'           , '%0.6f [m/s]'      % U_inf     )
        if self.verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % cp        )
        if self.verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % cv        )
        if self.verbose: even_print('recovery factor' , '%0.6f [-]'        % recov_fac )
        if self.verbose: even_print('Tw'              , '%0.3f [K]'        % Tw        )
        if self.verbose: even_print('Taw'             , '%0.3f [K]'        % Taw       )
        if self.verbose: even_print('lchar'           , '%0.6E [m]'        % lchar     )
        if self.verbose: print(72*'-'+'\n')
        
        # ===
        
        self.Ma           = Ma
        self.Re           = Re
        self.Pr           = Pr
        self.kappa        = kappa
        self.R            = R
        self.p_inf        = p_inf
        self.T_inf        = T_inf
        self.C_Suth       = C_Suth
        self.S_Suth       = S_Suth
        self.mu_Suth_ref  = mu_Suth_ref
        self.T_Suth_ref   = T_Suth_ref
        
        self.rho_inf      = rho_inf
        self.mu_inf       = mu_inf
        self.nu_inf       = nu_inf
        self.a_inf        = a_inf
        self.U_inf        = U_inf
        self.cp           = cp
        self.cv           = cv
        self.recov_fac    = recov_fac
        self.Tw           = Tw
        self.Taw          = Taw
        self.lchar        = lchar

        # === check if this a 2D average file like 'mean_flow_mpi.eas'
        
        if self.verbose: print(72*'-')
        if ('/Kennsatz/AUXILIARY/AVERAGING' in self):
            self.total_avg_time       = self['/Kennsatz/AUXILIARY/AVERAGING'].attrs['total_avg_time'][0]
            self.total_avg_iter_count = self['/Kennsatz/AUXILIARY/AVERAGING'].attrs['total_avg_iter_count'][0]
            if self.verbose: even_print('total_avg_time', '%0.2f'%self.total_avg_time)
            if self.verbose: even_print('total_avg_iter_count', '%i'%self.total_avg_iter_count)
            self.measType = 'mean'
        else:
            self.measType = 'unsteady'
        if self.verbose: even_print('meas type', '\'%s\''%self.measType)
        if self.verbose: print(72*'-'+'\n')
        
        # === grid info
        
        ndim1 = int( self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_SIZE'][0] )
        ndim2 = int( self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_SIZE'][1] )
        ndim3 = int( self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_SIZE'][2] )
        
        nx = self.nx = ndim1
        ny = self.ny = ndim2
        nz = self.nz = ndim3
        
        if (self.measType=='mean'):
            nz = self.nz = 1
        
        ngp = self.ngp = nx*ny*nz
        
        if self.verbose: print('grid info\n'+72*'-')
        if self.verbose: even_print('nx',  '%i'%nx  )
        if self.verbose: even_print('ny',  '%i'%ny  )
        if self.verbose: even_print('nz',  '%i'%nz  )
        if self.verbose: even_print('ngp', '%i'%ngp )
        
        gmode_dim1 = self.gmode_dim1 = self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_GMODE'][0]
        gmode_dim2 = self.gmode_dim2 = self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_GMODE'][1]
        gmode_dim3 = self.gmode_dim3 = self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_GMODE'][2]
        
        ## the original gmode (pre-conversion)
        gmode_dim1_orig = self.gmode_dim1_orig = gmode_dim1
        gmode_dim2_orig = self.gmode_dim2_orig = gmode_dim2
        gmode_dim3_orig = self.gmode_dim3_orig = gmode_dim3
        
        if self.verbose: even_print( 'gmode dim1' , '%i / %s'%(gmode_dim1,gmode_dict[gmode_dim1]) )
        if self.verbose: even_print( 'gmode dim2' , '%i / %s'%(gmode_dim2,gmode_dict[gmode_dim2]) )
        if self.verbose: even_print( 'gmode dim3' , '%i / %s'%(gmode_dim3,gmode_dict[gmode_dim3]) )
        if self.verbose: print(72*'-')
        
        # === (automatic) grid read
        
        ## can fail if >2[GB] and using driver=='mpio' and using one process
        ## https://github.com/h5py/h5py/issues/1052
        
        if True:
            
            ## dset object handles, no read yet
            dset1 = self[f'Kennsatz/GEOMETRY/{self.domainName}/dim01']
            dset2 = self[f'Kennsatz/GEOMETRY/{self.domainName}/dim02']
            dset3 = self[f'Kennsatz/GEOMETRY/{self.domainName}/dim03']
            
            float_bytes1  = dset1.dtype.itemsize
            float_bytes2  = dset2.dtype.itemsize
            float_bytes3  = dset3.dtype.itemsize
            
            data_gb_dim1 = float_bytes1*np.prod(dset1.shape) / 1024**3
            data_gb_dim2 = float_bytes2*np.prod(dset2.shape) / 1024**3
            data_gb_dim3 = float_bytes3*np.prod(dset3.shape) / 1024**3
            
            ## no 3D datasets, because no dim's GMODE is 5
            ## in this case, the dsets are just 0D/1D, so just read completely to all ranks
            if not any([ (gmode_dim1==5) , (gmode_dim2==5) , (gmode_dim3==5) ]):
                
                self.is_curvilinear = False
                self.is_rectilinear = True
                
                ## check just to be sure that the coord dsets in the HDF5 are indeed not 3D
                if (dset1.ndim>2):
                    raise ValueError
                if (dset2.ndim>2):
                    raise ValueError
                if (dset2.ndim>2):
                    raise ValueError
                
                ## read 1D datasets completely... this should never be a problem for RAM in MPI mode
                dim1_data = np.copy(dset1[()])
                dim2_data = np.copy(dset2[()])
                dim3_data = np.copy(dset3[()])
            
            else:
                
                '''
                at least one dim has GMODE=5, so 3D datasets exist, BUT unless GMODE==(5,5,5) 
                it is still possible to read as combo of 1D/2D, which is safe for MPI
                '''
                
                self.is_curvilinear = True
                self.is_rectilinear = False
                
                if (gmode_dim1==5) and (gmode_dim2==5) and (gmode_dim3==5): ## MUST read as 3D... only do here if read_3d_grid=True explicitly set
                    
                    ## this becomes very RAM intensive in MPI mode (all rannks are reading full 3D coord data)
                    if self.read_3d_grid:
                        dim1_data = np.copy(dset1[()])
                        dim2_data = np.copy(dset2[()])
                        dim3_data = np.copy(dset3[()])
                    else:
                        dim1_data = None
                        dim2_data = None
                        dim3_data = None
                
                elif (gmode_dim1==5) and (gmode_dim2==5) and (gmode_dim3==1): ## 551
                    dim1_data = np.copy(dset1[:,:,0][:,:,np.newaxis]) ## [x] is (nx,ny,1)
                    dim2_data = np.copy(dset2[:,:,0][:,:,np.newaxis]) ## [y] is (nx,ny,1)
                    dim3_data = np.copy(dset3[()])                    ## [z]
                
                elif (gmode_dim1==5) and (gmode_dim2==5) and (gmode_dim3==2): ## 552
                    dim1_data = np.copy(dset1[:,:,0][:,:,np.newaxis]) ## [x] is (nx,ny,1)
                    dim2_data = np.copy(dset2[:,:,0][:,:,np.newaxis]) ## [y] is (nx,ny,1)
                    dim3_data = np.copy(dset3[()])                    ## [z]
                
                elif (gmode_dim1==5) and (gmode_dim2==5) and (gmode_dim3==4): ## 554
                    dim1_data = np.copy(dset1[:,:,0][:,:,np.newaxis]) ## [x] is (nx,ny,1)
                    dim2_data = np.copy(dset2[:,:,0][:,:,np.newaxis]) ## [y] is (nx,ny,1)
                    dim3_data = np.copy(dset3[()])                    ## [z]
                
                else:
                    
                    print(f'gmode combo ({gmode_dim1:d},{gmode_dim2:d},{gmode_dim3:d}) does not yet have instructions in eas4.get_header()')
                    raise NotImplementedError
        
        ## old snippet for getting around HDF5 / h5py bug if >2[GB] and using driver=='mpio' and using one process
        ## keeping here for no3
        ## https://github.com/h5py/h5py/issues/1052
        #except OSError:
        if False:
            
            if (gmode_dim1 == EAS4_FULL_G):
                dim1_data = np.zeros((nx,ny,nz), dtype = self['Kennsatz/GEOMETRY/%s/dim01'%self.domainName].dtype)
                for i in range(nx):
                    dim1_data[i,:,:] = self['Kennsatz/GEOMETRY/%s/dim01'%self.domainName][i,:,:]
            else:
                dim1_data = self['Kennsatz/GEOMETRY/%s/dim01'%self.domainName][:]
            
            if (gmode_dim2 == EAS4_FULL_G):
                dim2_data = np.zeros((nx,ny,nz), dtype = self['Kennsatz/GEOMETRY/%s/dim02'%self.domainName].dtype)
                for i in range(nx):
                    dim2_data[i,:,:] = self['Kennsatz/GEOMETRY/%s/dim02'%self.domainName][i,:,:]
            else:
                dim2_data = self['Kennsatz/GEOMETRY/%s/dim02'%self.domainName][:]
            
            if (gmode_dim3 == EAS4_FULL_G):
                dim3_data = np.zeros((nx,ny,nz), dtype = self['Kennsatz/GEOMETRY/%s/dim03'%self.domainName].dtype)
                for i in range(nx):
                    dim3_data[i,:,:] = self['Kennsatz/GEOMETRY/%s/dim03'%self.domainName][i,:,:]
            else:
                dim3_data = self['Kennsatz/GEOMETRY/%s/dim03'%self.domainName][:]
        
        ## check grid for span avg
        if False:
            if (self.measType == 'mean'):
                if (gmode_dim1 == EAS4_FULL_G):
                    if not np.allclose(dim1_data[:,:,0], dim1_data[:,:,1], rtol=1e-08):
                        raise AssertionError('check')
                if (gmode_dim2 == EAS4_FULL_G):
                    if not np.allclose(dim2_data[:,:,0], dim2_data[:,:,1], rtol=1e-08):
                        raise AssertionError('check')
        
        # === expand gmodes 1,2 --> 4 (always ok, even for MPI mode)
        
        ## convert EAS4_NO_G to EAS4_ALL_G (1 --> 4) --> always do this
        ## dim_n_data are already numpy arrays of shape (1,) --> no conversion necessary, just update 'gmode_dimn' attr
        if (gmode_dim1 == EAS4_NO_G):
            gmode_dim1 = self.gmode_dim1 = EAS4_ALL_G
        if (gmode_dim2 == EAS4_NO_G):
            gmode_dim2 = self.gmode_dim2 = EAS4_ALL_G
        if (gmode_dim3 == EAS4_NO_G):
            gmode_dim3 = self.gmode_dim3 = EAS4_ALL_G
        
        ## convert EAS4_X0DX_G to EAS4_ALL_G (2 --> 4) --> always do this
        if (gmode_dim1 == EAS4_X0DX_G):
            dim1_data  = np.linspace(dim1_data[0],dim1_data[0]+dim1_data[1]*(ndim1-1), ndim1, dtype=np.float64)
            gmode_dim1 = self.gmode_dim1 = EAS4_ALL_G
        if (gmode_dim2 == EAS4_X0DX_G):
            dim2_data  = np.linspace(dim2_data[0],dim2_data[0]+dim2_data[1]*(ndim2-1), ndim2, dtype=np.float64)
            gmode_dim2 = self.gmode_dim2 = EAS4_ALL_G
        if (gmode_dim3 == EAS4_X0DX_G):
            dim3_data  = np.linspace(dim3_data[0],dim3_data[0]+dim3_data[1]*(ndim3-1), ndim3, dtype=np.float64)
            gmode_dim3 = self.gmode_dim3 = EAS4_ALL_G
        
        # ===
        
        x = self.x = np.copy(dim1_data)
        y = self.y = np.copy(dim2_data)
        z = self.z = np.copy(dim3_data)
        
        # ## bug check
        # if (z.size > 1):
        #     if np.all(np.isclose(z,z[0],rtol=1e-12)):
        #         raise AssertionError('z has size > 1 but all grid coords are identical!')
        
        if self.verbose: even_print('x_min', '%0.2f'%x.min())
        if self.verbose: even_print('x_max', '%0.2f'%x.max())
        if self.is_rectilinear:
            if (self.nx>2):
                if self.verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (x[1]-x[0]), (x[-1]-x[-2]) ))
        if self.verbose: even_print('y_min', '%0.2f'%y.min())
        if self.verbose: even_print('y_max', '%0.2f'%y.max())
        if self.is_rectilinear:
            if (self.ny>2):
                if self.verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (y[1]-y[0]), (y[-1]-y[-2]) ))
        if self.verbose: even_print('z_min', '%0.2f'%z.min())
        if self.verbose: even_print('z_max', '%0.2f'%z.max())
        if self.is_rectilinear:
            if (self.nz>2):
                if self.verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (z[1]-z[0]), (z[-1]-z[-2]) ))
        if self.verbose: print(72*'-'+'\n')
        
        # === time & scalar info
        
        if self.verbose: print('time & scalar info\n'+72*'-')
        
        n_scalars = self['Kennsatz/PARAMETER'].attrs['PARAMETER_SIZE'][0]
        
        if ('Kennsatz/PARAMETER/PARAMETERS_ATTRS' in self):
            scalars =  [ s.decode('utf-8').strip() for s in self['Kennsatz/PARAMETER/PARAMETERS_ATTRS'][()] ]
        else:
            ## this is the older gen structure
            scalars = [ self['Kennsatz/PARAMETER'].attrs['PARAMETERS_ATTR_%06d'%i][0].decode('utf-8').strip() for i in range(n_scalars) ]
        
        scalar_n_map = dict(zip(scalars, range(n_scalars)))
        
        self.scalars_dtypes = []
        for scalar in scalars:
            dset_path = 'Data/%s/ts_%06d/par_%06d'%(self.domainName,0,scalar_n_map[scalar])
            if (dset_path in self):
                self.scalars_dtypes.append(self[dset_path].dtype)
            else:
                #self.scalars_dtypes.append(np.float32)
                raise AssertionError('dset not found: %s'%dset_path)
        
        self.scalars_dtypes_dict = dict(zip(scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        
        nt         = self['Kennsatz/TIMESTEP'].attrs['TIMESTEP_SIZE'][0] 
        gmode_time = self['Kennsatz/TIMESTEP'].attrs['TIMESTEP_MODE'][0]
        
        ## a baseflow will not have a TIMEGRID
        if ('Kennsatz/TIMESTEP/TIMEGRID' in self):
            t = self['Kennsatz/TIMESTEP/TIMEGRID'][:]
        else:
            t = np.array( [0.] , dtype=np.float64 )
        
        if (gmode_time==EAS4_X0DX_G): ## =2 --> i.e. more than one timestep
            t = np.linspace(t[0],t[0]+t[1]*(nt - 1), nt  )
            gmode_time = EAS4_ALL_G
        else:
            #print('gmode_time : '+str(gmode_time))
            pass
        
        if (t.size>1):
            dt = t[1] - t[0]
            duration = t[-1] - t[0]
        else:
            dt = 0.
            duration = 0.
        
        if self.verbose: even_print('nt', '%i'%nt )
        if self.verbose: even_print('dt', '%0.6f'%dt)
        if self.verbose: even_print('duration', '%0.2f'%duration )
        
        # === attach to instance
        
        self.n_scalars    = n_scalars
        self.scalars      = scalars
        self.scalar_n_map = scalar_n_map
        self.t            = t
        self.dt           = dt
        self.nt           = nt
        self.duration     = duration
        
        self.ti           = np.arange(self.nt, dtype=np.float64)
        
        if self.verbose: print(72*'-'+'\n')
        
        # === udef dictionary attached to instance
        
        udef_char = [     'Ma',     'Re',     'Pr',      'kappa',    'R',    'p_inf',    'T_inf',    'C_Suth',    'S_Suth',    'mu_Suth_ref',    'T_Suth_ref' ]
        udef_real = [ self.Ma , self.Re , self.Pr ,  self.kappa, self.R, self.p_inf, self.T_inf, self.C_Suth, self.S_Suth, self.mu_Suth_ref, self.T_Suth_ref  ]
        self.udef = dict(zip(udef_char, udef_real))
        return
    
    # ===
    
    def get_mean(self, **kwargs):
        '''
        get spanwise mean of 2D EAS4 file
        '''
        axis = kwargs.get('axis',(2,))
        
        if (self.measType!='mean'):
            raise NotImplementedError('get_mean() not yet valid for measType=\'%s\''%self.measType)
        
        ## numpy structured array
        data_mean = np.zeros(shape=(self.nx,self.ny), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
        
        for si, scalar in enumerate(self.scalars):
            scalar_dtype = self.scalars_dtypes[si]
            dset_path = 'Data/%s/ts_%06d/par_%06d'%(self.domainName,0,self.scalar_n_map[scalar])
            data = np.copy(self[dset_path][()])
            ## perform np.mean() with float64 accumulator!
            scalar_mean = np.mean(data, axis=axis, dtype=np.float64).astype(scalar_dtype)
            data_mean[scalar] = scalar_mean
        
        return data_mean
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        print('make_xdmf() not yet implemented for turbx class EAS4')
        raise NotImplementedError
        return

class ztmd(h5py.File):
    '''
    span (z) & temporal (t) mean data (md)
    -----
    --> mean_flow_mpi.eas
    --> favre_mean_flow_mpi.eas
    --> ext_rms_fluctuation_mpi.eas
    --> ext_favre_fluctuation_mpi.eas
    --> turbulent_budget_mpi.eas
    -----
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.ztmd()')
        
        ## mpio driver for ZTMD currently not supported
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            raise ValueError('ZTMD class is currently not set up to be used with MPI')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## ztmd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 16    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8     )
        perms          = kwargs.pop('stripe_size_mb' , '640' )
        
        if not isinstance(stripe_count, int):
            raise ValueError
        if not isinstance(stripe_size_mb, int):
            raise ValueError
        if not isinstance(perms, str):
            raise ValueError
        if not len(perms)==3:
            raise ValueError
        if not re.fullmatch(r'\d{3}',perms):
            raise ValueError
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using romio --> causes crash
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        
        ## set ROMIO hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(16*1024**2))) ) ## 16 [MB]
                ##
                #mpi_info.Set('romio_no_indep_rw' , 'true' ) ## deferred open + only collective I/O 
                #mpi_info.Set('cb_nodes' , str(int(round(1*self.n_ranks))) )
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        ## | rdcc_nbytes:
        ## | ------------
        ## | Integer setting the total size of the raw data chunk cache for this dataset in bytes.
        ## | In most cases increasing this number will improve performance, as long as you have 
        ## | enough free memory. The default size is 1 MB
        
        ## --> gets passed to H5Pset_chunk_cache
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        ## | rdcc_nslots:
        ## | ------------
        ## | Integer defining the number of chunk slots in the raw data chunk cache for this dataset.
        
        ## if ('rdcc_nslots' not in kwargs):
        ##     kwargs['rdcc_nslots'] = 521
        
        ## ztmd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with ztmd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(0.1)
                Path(self.fname).touch() ## touch a new file
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        self.mod_avail_tqdm = ('tqdm' in sys.modules)
        
        ## call actual h5py.File.__init__()
        super(ztmd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(ztmd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed ZTMD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(ztmd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of ZTMD class instance
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if (self.rank!=0):
            verbose=False
        
        # === udef (header vector dset based)
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## just make udef_real a dict with udef_char as keys
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.C_Suth      = self.udef['C_Suth']
            self.S_Suth      = self.udef['S_Suth']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            
            if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            
            # === characteristic values : derived
            
            rho_inf   = self.rho_inf   = self.p_inf/(self.R * self.T_inf)
            mu_inf    = self.mu_inf    = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            nu_inf    = self.nu_inf    = self.mu_inf/self.rho_inf
            a_inf     = self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            U_inf     = self.U_inf     = self.Ma*self.a_inf
            cp        = self.cp        = self.R*self.kappa/(self.kappa-1.)
            cv        = self.cv        = self.cp/self.kappa
            recov_fac = self.recov_fac = self.Pr**(1/3)
            Tw        = self.Tw        = self.T_inf
            Taw       = self.Taw       = self.T_inf + self.r*self.U_inf**2/(2*self.cp)
            lchar     = self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Tw'              , '%0.3f [K]'        % self.Tw        )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the ZTMD instance
            udef_char_deriv = ['rho_inf', 'mu_inf', 'nu_inf', 'a_inf', 'U_inf', 'cp', 'cv', 'recov_fac', 'Tw', 'Taw', 'lchar']
            udef_real_deriv = [ rho_inf,   mu_inf,   nu_inf,   a_inf,   U_inf,   cp,   cv,   recov_fac,   Tw,   Taw,   lchar ]
            self.udef_deriv = dict(zip(udef_char_deriv, udef_real_deriv))
        
        else:
            #print("dset 'header' not in ZTMD")
            pass
        
        # === udef (attr based)
        
        header_attr_str_list = ['Ma','Re','Pr','kappa','R','p_inf','T_inf','C_Suth','S_Suth','mu_Suth_ref','T_Suth_ref']
        if all([ attr_str in self.attrs.keys() for attr_str in header_attr_str_list ]):
            header_attr_based = True
        else:
            header_attr_based = False
        
        if header_attr_based:
            
            ## set all attributes
            for attr_str in header_attr_str_list:
                setattr( self, attr_str, self.attrs[attr_str] )
            
            if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            
            # === characteristic values : derived
            
            mu_inf_1 = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            mu_inf_2 = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2)*(self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth)
            mu_inf_3 = self.C_Suth*self.T_inf**(3/2)/(self.T_inf+self.S_Suth)
            #print(mu_inf_1)
            #print(mu_inf_2)
            #print(mu_inf_3)
            
            rho_inf   = self.rho_inf   = self.p_inf/(self.R*self.T_inf)
            #mu_inf    = self.mu_inf    = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            mu_inf    = self.mu_inf    = mu_inf_3
            nu_inf    = self.nu_inf    = self.mu_inf/self.rho_inf
            a_inf     = self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            U_inf     = self.U_inf     = self.Ma*self.a_inf
            cp        = self.cp        = self.R*self.kappa/(self.kappa-1.)
            cv        = self.cv        = self.cp/self.kappa
            recov_fac = self.recov_fac = self.Pr**(1/3)
            Tw        = self.Tw        = self.T_inf
            Taw       = self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            lchar     = self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Tw'              , '%0.3f [K]'        % self.Tw        )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            #if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the ZTMD instance
            udef_char_deriv = ['rho_inf', 'mu_inf', 'nu_inf', 'a_inf', 'U_inf', 'cp', 'cv', 'recov_fac', 'Tw', 'Taw', 'lchar']
            udef_real_deriv = [ rho_inf,   mu_inf,   nu_inf,   a_inf,   U_inf,   cp,   cv,   recov_fac,   Tw,   Taw,   lchar ]
            self.udef_deriv = dict(zip(udef_char_deriv, udef_real_deriv))
        
        if ('duration_avg' in self.attrs.keys()):
            self.duration_avg = self.attrs['duration_avg']
        if ('nx' in self.attrs.keys()):
            self.nx = self.attrs['nx']
        if ('ny' in self.attrs.keys()):
            self.ny = self.attrs['ny']
        if ('p_inf' in self.attrs.keys()):
            self.p_inf = self.attrs['p_inf']
        if ('lchar' in self.attrs.keys()):
            self.lchar = self.attrs['lchar']
        if ('U_inf' in self.attrs.keys()):
            self.U_inf = self.attrs['U_inf']
        if ('Re' in self.attrs.keys()):
            self.Re = self.attrs['Re']
        
        if ('T_inf' in self.attrs.keys()):
            self.T_inf = self.attrs['T_inf']
        if ('rho_inf' in self.attrs.keys()):
            self.rho_inf = self.attrs['rho_inf']
        
        if ('dims/x' in self):
            self.x = np.copy( self['dims/x'][()] ) ## dont transpose yet
        if ('dims/y' in self):
            self.y = np.copy( self['dims/y'][()] ) ## dont transpose yet
        if ('dims/t' in self):
            self.t = t = np.copy( self['dims/t'][()] )
        
        if hasattr(self,'x') and hasattr(self,'y'):
            if (self.x.ndim==1) and (self.y.ndim==1):
                self.xx, self.yy = np.meshgrid( self.x, self.y, indexing='ij' )
            elif (self.x.ndim==2) and (self.y.ndim==2):
                self.x  = np.copy( self.x.T )
                self.y  = np.copy( self.y.T )
                self.xx = np.copy( self.x   )
                self.yy = np.copy( self.y   )
            else:
                raise ValueError
        
        if ('dz' in self.attrs.keys()):
            self.dz = self.attrs['dz']
        
        if ('dims/stang' in self):
            self.stang = np.copy( self['dims/stang'][()] )
        if ('dims/snorm' in self):
            self.snorm = np.copy( self['dims/snorm'][()] )
        
        if ('csys/vtang' in self):
            self.vtang = np.copy( self['csys/vtang'][()] )
        if ('csys/vnorm' in self):
            self.vnorm = np.copy( self['csys/vnorm'][()] )
        
        if ('dims/crv_R' in self):
            self.crv_R = np.copy( self['dims/crv_R'][()] )
        if ('dims/R_min' in self):
            self.R_min = self['dims/R_min'][()]
        
        if verbose: print(72*'-')
        if verbose and hasattr(self,'duration_avg'): even_print('duration_avg', '%0.5f'%self.duration_avg)
        if verbose: even_print('nx', '%i'%self.nx)
        if verbose: even_print('ny', '%i'%self.ny)
        #if verbose: print(72*'-')
        
        # ===
        
        if ('rectilinear' in self.attrs.keys()):
            self.rectilinear = self.attrs['rectilinear']
        
        if ('curvilinear' in self.attrs.keys()):
            self.curvilinear = self.attrs['curvilinear']
        
        ## check
        if hasattr(self,'rectilinear') and not hasattr(self,'curvilinear'):
            raise ValueError
        if hasattr(self,'curvilinear') and not hasattr(self,'rectilinear'):
            raise ValueError
        if hasattr(self,'rectilinear') or hasattr(self,'curvilinear'):
            if self.rectilinear and self.curvilinear:
                raise ValueError
            if not self.rectilinear and not self.curvilinear:
                raise ValueError
        
        if ('requires_wall_norm_interp' in self.attrs.keys()):
            self.requires_wall_norm_interp = self.attrs['requires_wall_norm_interp']
        else:
            self.requires_wall_norm_interp = False
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys())
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self['data/%s'%scalar].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    # === I/O funcs
    
    def compile_data(self, path, **kwargs):
        '''
        Copy data from 2D EAS4 containers (output from NS3D) to a ZTMD container
        -----
        
        The 'path' directory should contain one or more of the following files:
        
        --> mean_flow_mpi.eas
        --> favre_mean_flow_mpi.eas
        --> ext_rms_fluctuation_mpi.eas
        --> ext_favre_fluctuation_mpi.eas
        --> turbulent_budget_mpi.eas
        
        /dims : 2D dimension datasets (x,y,..) and possibly 1D dimension datasets (s_wall,..)
        /data : 2D datasets (u,uIuI,..)
        
        Datasets are dimensionalized to SI units upon import!
        
        /dimless : copy the dimless datasets as a reference
        
        Curvilinear cases may have the following additional HDF5 groups
        
        /data_1Dx : 1D datsets in streamwise (x/s1) direction (μ_wall,ρ_wall,u_τ,..)
        /csys     : coordinate system transformation arrays (projection vectors, transform tensors, etc.)
        -----
        /dims_2Dw : alternate grid (e.g. wall-normal projected/interpolation grid)
        /data_2Dw : data interpolated onto alternate grid
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'turbx.ztmd.compile_data()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        even_print('ztmd',str(self.fname))
        
        ## dz,dt should be input as dimless (characteristic/inlet) (output from tgg)
        ## --> dz & dt get re-dimensionalized during this func!
        dz = kwargs.get('dz',None)
        nz = kwargs.get('nz',None)
        dt = kwargs.get('dt',None)
        
        path_ztmean = Path(path)
        if not path_ztmean.is_dir():
            raise FileNotFoundError('%s does not exist.'%str(path_ztmean))
        fn_Re_mean     = Path(path_ztmean, 'mean_flow_mpi.eas')
        fn_Fv_mean     = Path(path_ztmean, 'favre_mean_flow_mpi.eas')
        fn_Re_fluct    = Path(path_ztmean, 'ext_rms_fluctuation_mpi.eas')
        fn_Fv_fluct    = Path(path_ztmean, 'ext_favre_fluctuation_mpi.eas')
        fn_turb_budget = Path(path_ztmean, 'turbulent_budget_mpi.eas')
        
        self.attrs['fn_Re_mean']     = str( fn_Re_mean.relative_to(Path())     )
        self.attrs['fn_Fv_mean']     = str( fn_Fv_mean.relative_to(Path())     )
        self.attrs['fn_Re_fluct']    = str( fn_Re_fluct.relative_to(Path())    )
        self.attrs['fn_Fv_fluct']    = str( fn_Fv_fluct.relative_to(Path())    )
        self.attrs['fn_turb_budget'] = str( fn_turb_budget.relative_to(Path()) )
        
        ## the simulation timestep dt is not known from the averaged files
        if (dt is not None):
            self.attrs['dt'] = dt
        if (nz is not None):
            self.attrs['nz'] = nz
        if (dz is not None):
            self.attrs['dz'] = dz
        
        if verbose:
            if (nz is not None):
                even_print('nz' , '%i'%nz )
            if (dz is not None):
                even_print('dz' , '%0.6e'%dz )
            if (dt is not None):
                even_print('dt' , '%0.6e'%dt )
            print(72*'-')
        
        # ===
        
        if fn_Re_mean.exists():
            even_print('eas4 Re mean',str(fn_Re_mean.relative_to(Path())))
            with eas4(str(fn_Re_mean),'r',verbose=False) as f1:
                
                ## the EAS4 data is still organized by rank in [z], so perform average across ranks
                data_mean = f1.get_mean()
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                nx = f1.nx ; self.attrs['nx'] = nx
                ny = f1.ny ; self.attrs['ny'] = ny
                
                Ma          = f1.Ma          ; self.attrs['Ma']          = Ma
                Re          = f1.Re          ; self.attrs['Re']          = Re
                Pr          = f1.Pr          ; self.attrs['Pr']          = Pr
                T_inf       = f1.T_inf       ; self.attrs['T_inf']       = T_inf
                p_inf       = f1.p_inf       ; self.attrs['p_inf']       = p_inf
                kappa       = f1.kappa       ; self.attrs['kappa']       = kappa
                R           = f1.R           ; self.attrs['R']           = R
                mu_Suth_ref = f1.mu_Suth_ref ; self.attrs['mu_Suth_ref'] = mu_Suth_ref
                T_Suth_ref  = f1.T_Suth_ref  ; self.attrs['T_Suth_ref']  = T_Suth_ref
                C_Suth      = f1.C_Suth      ; self.attrs['C_Suth']      = C_Suth
                S_Suth      = f1.S_Suth      ; self.attrs['S_Suth']      = S_Suth
                
                rho_inf   = f1.rho_inf   # ; self.attrs['rho_inf']   = rho_inf
                mu_inf    = f1.mu_inf    # ; self.attrs['mu_inf']    = mu_inf
                nu_inf    = f1.nu_inf    # ; self.attrs['nu_inf']    = nu_inf
                a_inf     = f1.a_inf     # ; self.attrs['a_inf']     = a_inf
                U_inf     = f1.U_inf     # ; self.attrs['U_inf']     = U_inf
                cp        = f1.cp        # ; self.attrs['cp']        = cp
                cv        = f1.cv        # ; self.attrs['cv']        = cv
                recov_fac = f1.recov_fac # ; self.attrs['recov_fac'] = recov_fac
                Tw        = f1.Tw        # ; self.attrs['Tw']        = Tw
                Taw       = f1.Taw       # ; self.attrs['Taw']       = Taw
                lchar     = f1.lchar     # ; self.attrs['lchar']     = lchar
                
                tchar = f1.lchar/f1.U_inf # ; self.attrs['tchar'] = tchar
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                Re_mean_total_avg_time       = f1.total_avg_time * tchar
                Re_mean_total_avg_iter_count = f1.total_avg_iter_count
                Re_mean_dt                   = Re_mean_total_avg_time / Re_mean_total_avg_iter_count
                
                self.attrs['Re_mean_total_avg_time'] = Re_mean_total_avg_time
                self.attrs['Re_mean_total_avg_iter_count'] = Re_mean_total_avg_iter_count
                self.attrs['Re_mean_dt'] = Re_mean_dt
                
                t_meas = f1.total_avg_time * (f1.lchar/f1.U_inf) ## dimensional [s]
                #t_meas = f1.total_avg_time ## dimless (char)
                self.attrs['t_meas'] = t_meas
                dset = self.create_dataset('dims/t', data=np.array([t_meas],dtype=np.float64), chunks=None)
                
                # ===
                
                ## from EAS4, dimless (char)
                x = np.copy(f1.x)
                y = np.copy(f1.y)
                
                if (f1.x.ndim==1) and (f1.y.ndim==1): ## rectilinear in [x,y]
                    
                    self.attrs['rectilinear'] = True
                    self.attrs['curvilinear'] = False
                    
                    ## dimensionalize & write
                    x  *= lchar
                    y  *= lchar
                    dz *= lchar
                    dt *= tchar
                    self.create_dataset('dims/x', data=x, chunks=None, dtype=np.float64)
                    self.create_dataset('dims/y', data=y, chunks=None, dtype=np.float64)
                    self.attrs['dz'] = dz
                    self.attrs['dt'] = dt
                    
                    self.attrs['nx'] = nx
                    self.attrs['ny'] = ny
                    #self.attrs['nz'] = 1 ## NO
                    if verbose:
                        even_print('nx' , '%i'%nx )
                        even_print('ny' , '%i'%ny )
                
                elif (f1.x.ndim==3) and (f1.y.ndim==3): ## curvilinear in [x,y]
                    
                    self.attrs['rectilinear'] = False
                    self.attrs['curvilinear'] = True
                    
                    ## 3D coords: confirm that x,y coords are same in [z] direction
                    np.testing.assert_allclose( x[-1,-1,:] , x[-1,-1,0] , rtol=1e-14 , atol=1e-14 )
                    np.testing.assert_allclose( y[-1,-1,:] , y[-1,-1,0] , rtol=1e-14 , atol=1e-14 )
                    
                    ## 3D coords: take only 1 layer in [z]
                    x = np.squeeze( np.copy( x[:,:,0] ) ) ## dimless (char)
                    y = np.squeeze( np.copy( y[:,:,0] ) )
                    
                    if True: ## check against tgg data wall distance file (if it exists)
                        fn_dat = '../tgg/wall_distance.dat'
                        if os.path.isfile(fn_dat):
                            with open(fn_dat,'rb') as f:
                                d_ = pickle.load(f)
                                xy2d_tmp = d_['xy2d']
                                np.testing.assert_allclose(xy2d_tmp[:,:,0], x[:,:,0], rtol=1e-14, atol=1e-14)
                                np.testing.assert_allclose(xy2d_tmp[:,:,1], y[:,:,0], rtol=1e-14, atol=1e-14)
                                if verbose: even_print('check passed' , 'x grid' )
                                if verbose: even_print('check passed' , 'y grid' )
                                d_ = None; del d_
                                xy2d_tmp = None; del xy2d_tmp
                    
                    # ## backup non-dimensional coordinate arrays
                    # dset = self.create_dataset('/dimless/dims/x', data=x.T, chunks=None)
                    # dset = self.create_dataset('/dimless/dims/y', data=y.T, chunks=None)
                    
                    ## dimensionalize & write
                    x  *= lchar
                    y  *= lchar
                    dz *= lchar
                    dt *= tchar
                    
                    self.create_dataset('dims/x', data=x.T, chunks=None, dtype=np.float64)
                    self.create_dataset('dims/y', data=y.T, chunks=None, dtype=np.float64)
                    self.attrs['dz'] = dz
                    self.attrs['dt'] = dt
                    
                    self.attrs['nx'] = nx
                    self.attrs['ny'] = ny
                    #self.attrs['nz'] = 1 ## NO
                    if verbose:
                        even_print('nx' , '%i'%nx )
                        even_print('ny' , '%i'%ny )
                
                else:
                    raise ValueError('case x.ndim=%i , y.ndim=%i not yet accounted for'%(f1.x.ndim,f1.y.ndim))
                
                # === redimensionalize quantities (by sim characteristic quantities)
                
                u   = np.copy( data_mean['u']   ) * U_inf 
                v   = np.copy( data_mean['v']   ) * U_inf
                w   = np.copy( data_mean['w']   ) * U_inf
                rho = np.copy( data_mean['rho'] ) * rho_inf
                p   = np.copy( data_mean['p']   ) * (rho_inf * U_inf**2)
                T   = np.copy( data_mean['T']   ) * T_inf
                mu  = np.copy( data_mean['mu']  ) * mu_inf
                data_mean = None; del data_mean
                
                
                ## derived values from base scalars
                a     = np.sqrt( kappa * R * T )
                nu    = mu / rho
                umag  = np.sqrt( u**2 + v**2 + w**2 )
                M     = umag / np.sqrt(kappa * R * T)
                mflux = umag * rho
                
                ## base scalars [u,v,w,ρ,p,T]
                dset = self.create_dataset('data/u'   , data=u.T   , chunks=None)
                dset = self.create_dataset('data/v'   , data=v.T   , chunks=None)
                dset = self.create_dataset('data/w'   , data=w.T   , chunks=None)
                dset = self.create_dataset('data/rho' , data=rho.T , chunks=None)
                dset = self.create_dataset('data/p'   , data=p.T   , chunks=None)
                dset = self.create_dataset('data/T'   , data=T.T   , chunks=None)
                
                ## derived fields
                dset = self.create_dataset('data/a'     , data=a.T     , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[m/s]'
                dset = self.create_dataset('data/mu'    , data=mu.T    , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[kg/(m·s)]'
                dset = self.create_dataset('data/nu'    , data=nu.T    , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[m²/s]'
                dset = self.create_dataset('data/umag'  , data=umag.T  , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[m/s]'
                dset = self.create_dataset('data/M'     , data=M.T     , chunks=None) #; dset.attrs['dimensional'] = False
                dset = self.create_dataset('data/mflux' , data=mflux.T , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[kg/(m²·s)]'
        
        if fn_Re_fluct.exists():
            even_print('eas4 Re fluct',str(fn_Re_fluct.relative_to(Path())))
            with eas4(str(fn_Re_fluct),'r',verbose=False) as f1:
                
                data_mean = f1.get_mean()
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                Re_fluct_total_avg_time       = f1.total_avg_time
                Re_fluct_total_avg_iter_count = f1.total_avg_iter_count
                Re_fluct_dt                   = Re_fluct_total_avg_time/Re_fluct_total_avg_iter_count
                
                self.attrs['Re_fluct_total_avg_time'] = Re_fluct_total_avg_time
                self.attrs['Re_fluct_total_avg_iter_count'] = Re_fluct_total_avg_iter_count
                self.attrs['Re_fluct_dt'] = Re_fluct_dt
                
                uI_uI = data_mean["u'u'"] * U_inf**2
                vI_vI = data_mean["v'v'"] * U_inf**2
                wI_wI = data_mean["w'w'"] * U_inf**2
                uI_vI = data_mean["u'v'"] * U_inf**2
                uI_wI = data_mean["u'w'"] * U_inf**2
                vI_wI = data_mean["v'w'"] * U_inf**2
                ##
                self.create_dataset('data/uI_uI', data=uI_uI.T, chunks=None)
                self.create_dataset('data/vI_vI', data=vI_vI.T, chunks=None)
                self.create_dataset('data/wI_wI', data=wI_wI.T, chunks=None)
                self.create_dataset('data/uI_vI', data=uI_vI.T, chunks=None)
                self.create_dataset('data/uI_wI', data=uI_wI.T, chunks=None)
                self.create_dataset('data/vI_wI', data=vI_wI.T, chunks=None)
                
                uI_TI = data_mean["u'T'"] * (U_inf*T_inf)
                vI_TI = data_mean["v'T'"] * (U_inf*T_inf)
                wI_TI = data_mean["w'T'"] * (U_inf*T_inf)
                ##
                self.create_dataset('data/uI_TI', data=uI_TI.T, chunks=None)
                self.create_dataset('data/vI_TI', data=vI_TI.T, chunks=None)
                self.create_dataset('data/wI_TI', data=wI_TI.T, chunks=None)
                
                TI_TI = data_mean["T'T'"] * T_inf**2
                pI_pI = data_mean["p'p'"] * (rho_inf * U_inf**2)**2
                rI_rI = data_mean["r'r'"] * rho_inf**2
                muI_muI = data_mean["mu'mu'"] * mu_inf**2
                ##
                self.create_dataset('data/TI_TI',   data=TI_TI.T,   chunks=None)
                self.create_dataset('data/pI_pI',   data=pI_pI.T,   chunks=None)
                self.create_dataset('data/rI_rI',   data=rI_rI.T,   chunks=None)
                self.create_dataset('data/muI_muI', data=muI_muI.T, chunks=None)
                
                tauI_xx = data_mean["tau'_xx"] * mu_inf * U_inf / lchar
                tauI_yy = data_mean["tau'_yy"] * mu_inf * U_inf / lchar
                tauI_zz = data_mean["tau'_zz"] * mu_inf * U_inf / lchar
                tauI_xy = data_mean["tau'_xy"] * mu_inf * U_inf / lchar
                tauI_xz = data_mean["tau'_xz"] * mu_inf * U_inf / lchar
                tauI_yz = data_mean["tau'_yz"] * mu_inf * U_inf / lchar
                ##
                self.create_dataset('data/tauI_xx', data=tauI_xx.T, chunks=None)
                self.create_dataset('data/tauI_yy', data=tauI_yy.T, chunks=None)
                self.create_dataset('data/tauI_zz', data=tauI_zz.T, chunks=None)
                self.create_dataset('data/tauI_xy', data=tauI_xy.T, chunks=None)
                self.create_dataset('data/tauI_xz', data=tauI_xz.T, chunks=None)
                self.create_dataset('data/tauI_yz', data=tauI_yz.T, chunks=None)
                
                # === RMS values
                
                if True: ## dimensional
                    
                    uI_uI_rms = np.sqrt(       data_mean["u'u'"]  * U_inf**2 )
                    vI_vI_rms = np.sqrt(       data_mean["v'v'"]  * U_inf**2 )
                    wI_wI_rms = np.sqrt(       data_mean["w'w'"]  * U_inf**2 )
                    uI_vI_rms = np.sqrt(np.abs(data_mean["u'v'"]) * U_inf**2 ) * np.sign(data_mean["u'v'"]) 
                    uI_wI_rms = np.sqrt(np.abs(data_mean["u'w'"]) * U_inf**2 ) * np.sign(data_mean["u'w'"])
                    vI_wI_rms = np.sqrt(np.abs(data_mean["v'w'"]) * U_inf**2 ) * np.sign(data_mean["v'w'"])
                    
                    uI_TI_rms = np.sqrt(np.abs(data_mean["u'T'"]) * U_inf*T_inf) * np.sign(data_mean["u'T'"])
                    vI_TI_rms = np.sqrt(np.abs(data_mean["v'T'"]) * U_inf*T_inf) * np.sign(data_mean["v'T'"])
                    wI_TI_rms = np.sqrt(np.abs(data_mean["w'T'"]) * U_inf*T_inf) * np.sign(data_mean["w'T'"])
                    
                    rI_rI_rms   = np.sqrt( data_mean["r'r'"]   * rho_inf**2              )
                    TI_TI_rms   = np.sqrt( data_mean["T'T'"]   * T_inf**2                )
                    pI_pI_rms   = np.sqrt( data_mean["p'p'"]   * (rho_inf * U_inf**2)**2 )
                    muI_muI_rms = np.sqrt( data_mean["mu'mu'"] * mu_inf**2               )
                    
                    M_rms = uI_uI_rms / np.sqrt(kappa * R * T)
                
                # if False: ## dimless
                #     
                #     uI_uI_rms = np.sqrt(        data_mean["u'u'"]  )
                #     vI_vI_rms = np.sqrt(        data_mean["v'v'"]  )
                #     wI_wI_rms = np.sqrt(        data_mean["w'w'"]  )
                #     uI_vI_rms = np.sqrt( np.abs(data_mean["u'v'"]) ) * np.sign(data_mean["u'v'"]) 
                #     uI_wI_rms = np.sqrt( np.abs(data_mean["u'w'"]) ) * np.sign(data_mean["u'w'"])
                #     vI_wI_rms = np.sqrt( np.abs(data_mean["v'w'"]) ) * np.sign(data_mean["v'w'"])
                #     
                #     uI_TI_rms = np.sqrt( np.abs(data_mean["u'T'"]) ) * np.sign(data_mean["u'T'"])
                #     vI_TI_rms = np.sqrt( np.abs(data_mean["v'T'"]) ) * np.sign(data_mean["v'T'"])
                #     wI_TI_rms = np.sqrt( np.abs(data_mean["w'T'"]) ) * np.sign(data_mean["w'T'"])
                #     
                #     rI_rI_rms   = np.sqrt( data_mean["r'r'"]   )
                #     TI_TI_rms   = np.sqrt( data_mean["T'T'"]   )
                #     pI_pI_rms   = np.sqrt( data_mean["p'p'"]   )
                #     muI_muI_rms = np.sqrt( data_mean["mu'mu'"] )
                #     
                #     # ...
                #     M_rms = np.sqrt( data_mean["u'u'"] * U_inf**2 ) / np.sqrt(kappa * R * (T*T_inf) )
                
                # ===
                
                self.create_dataset( 'data/uI_uI_rms' , data=uI_uI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_vI_rms' , data=vI_vI_rms.T , chunks=None )
                self.create_dataset( 'data/wI_wI_rms' , data=wI_wI_rms.T , chunks=None )
                self.create_dataset( 'data/uI_vI_rms' , data=uI_vI_rms.T , chunks=None )
                self.create_dataset( 'data/uI_wI_rms' , data=uI_wI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_wI_rms' , data=vI_wI_rms.T , chunks=None )
                ##
                self.create_dataset( 'data/uI_TI_rms' , data=uI_TI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_TI_rms' , data=vI_TI_rms.T , chunks=None )
                self.create_dataset( 'data/wI_TI_rms' , data=wI_TI_rms.T , chunks=None )
                ##
                self.create_dataset( 'data/rI_rI_rms'   , data=rI_rI_rms.T   , chunks=None )
                self.create_dataset( 'data/TI_TI_rms'   , data=TI_TI_rms.T   , chunks=None )
                self.create_dataset( 'data/pI_pI_rms'   , data=pI_pI_rms.T   , chunks=None )
                self.create_dataset( 'data/muI_muI_rms' , data=muI_muI_rms.T , chunks=None )
                ##
                self.create_dataset( 'data/M_rms' , data=M_rms.T , chunks=None )
        
        if fn_Fv_mean.exists():
            #print('--r-> %s'%fn_Fv_mean.relative_to(Path()) )
            even_print('eas4 Fv mean',str(fn_Fv_mean.relative_to(Path())))
            with eas4(str(fn_Fv_mean),'r',verbose=False) as f1:
                
                ## the EAS4 data is still organized by rank in [z], so perform average across ranks
                data_mean = f1.get_mean()
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                Fv_mean_total_avg_time       = f1.total_avg_time * tchar
                Fv_mean_total_avg_iter_count = f1.total_avg_iter_count
                Fv_mean_dt                   = Fv_mean_total_avg_time / Fv_mean_total_avg_iter_count
                
                self.attrs['Fv_mean_total_avg_time'] = Fv_mean_total_avg_time
                self.attrs['Fv_mean_total_avg_iter_count'] = Fv_mean_total_avg_iter_count
                self.attrs['Fv_mean_dt'] = Fv_mean_dt
                
                u_Fv   = np.copy( data_mean['u']   ) * U_inf 
                v_Fv   = np.copy( data_mean['v']   ) * U_inf
                w_Fv   = np.copy( data_mean['w']   ) * U_inf
                rho_Fv = np.copy( data_mean['rho'] ) * rho_inf
                p_Fv   = np.copy( data_mean['p']   ) * (rho_inf * U_inf**2)
                T_Fv   = np.copy( data_mean['T']   ) * T_inf
                mu_Fv  = np.copy( data_mean['mu']  ) * mu_inf
                
                uu_Fv  = np.copy( data_mean['uu'] ) * U_inf**2
                uv_Fv  = np.copy( data_mean['uv'] ) * U_inf**2
                
                data_mean = None; del data_mean
                
                self.create_dataset('data/u_Fv'   , data=u_Fv.T   , chunks=None)
                self.create_dataset('data/v_Fv'   , data=v_Fv.T   , chunks=None)
                self.create_dataset('data/w_Fv'   , data=w_Fv.T   , chunks=None)
                self.create_dataset('data/rho_Fv' , data=rho_Fv.T , chunks=None)
                self.create_dataset('data/p_Fv'   , data=p_Fv.T   , chunks=None)
                self.create_dataset('data/T_Fv'   , data=T_Fv.T   , chunks=None)
                
                self.create_dataset('data/uu_Fv' , data=uu_Fv.T   , chunks=None)
                self.create_dataset('data/uv_Fv' , data=uv_Fv.T   , chunks=None)
        
        if fn_Fv_fluct.exists():
            #print('--r-> %s'%fn_Fv_fluct.relative_to(Path()) )
            even_print('eas4 Fv fluct',str(fn_Fv_fluct.relative_to(Path())))
            with eas4(str(fn_Fv_fluct),'r',verbose=False) as f1:
                
                data_mean = f1.get_mean()
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                Fv_fluct_total_avg_time       = f1.total_avg_time
                Fv_fluct_total_avg_iter_count = f1.total_avg_iter_count
                Fv_fluct_dt                   = Fv_fluct_total_avg_time/Fv_fluct_total_avg_iter_count
                
                self.attrs['Fv_fluct_total_avg_time'] = Fv_fluct_total_avg_time
                self.attrs['Fv_fluct_total_avg_iter_count'] = Fv_fluct_total_avg_iter_count
                self.attrs['Fv_fluct_dt'] = Fv_fluct_dt
                
                r_uII_uII = data_mean["r u''u''"]  * rho_inf * U_inf**2
                r_vII_vII = data_mean["r v''v''"]  * rho_inf * U_inf**2
                r_wII_wII = data_mean["r w''_w''"] * rho_inf * U_inf**2
                r_uII_vII = data_mean["r u''v''"]  * rho_inf * U_inf**2
                r_uII_wII = data_mean["r u''w''"]  * rho_inf * U_inf**2
                r_vII_wII = data_mean["r w''v''"]  * rho_inf * U_inf**2
                ##
                self.create_dataset('data/r_uII_uII', data=r_uII_uII.T, chunks=None)
                self.create_dataset('data/r_vII_vII', data=r_vII_vII.T, chunks=None)
                self.create_dataset('data/r_wII_wII', data=r_wII_wII.T, chunks=None)
                self.create_dataset('data/r_uII_vII', data=r_uII_vII.T, chunks=None)
                self.create_dataset('data/r_uII_wII', data=r_uII_wII.T, chunks=None)
                self.create_dataset('data/r_vII_wII', data=r_vII_wII.T, chunks=None)
                
                r_uII_TII = data_mean["r u''T''"] * rho_inf * U_inf * T_inf
                r_vII_TII = data_mean["r v''T''"] * rho_inf * U_inf * T_inf
                r_wII_TII = data_mean["r w''T''"] * rho_inf * U_inf * T_inf
                ##
                self.create_dataset('data/r_uII_TII', data=r_uII_TII.T, chunks=None)
                self.create_dataset('data/r_vII_TII', data=r_vII_TII.T, chunks=None)
                self.create_dataset('data/r_wII_TII', data=r_wII_TII.T, chunks=None)
                
                r_TII_TII   = data_mean["r T''T''"]   * rho_inf * T_inf**2
                r_pII_pII   = data_mean["r p''p''"]   * rho_inf * (rho_inf * U_inf**2)**2
                r_rII_rII   = data_mean["r r''r''"]   * rho_inf * rho_inf**2
                r_muII_muII = data_mean["r mu''mu''"] * rho_inf * mu_inf**2
                ##
                self.create_dataset('data/r_TII_TII',   data=r_TII_TII.T,   chunks=None)
                self.create_dataset('data/r_pII_pII',   data=r_pII_pII.T,   chunks=None)
                self.create_dataset('data/r_rII_rII',   data=r_rII_rII.T,   chunks=None)
                self.create_dataset('data/r_muII_muII', data=r_muII_muII.T, chunks=None)
                
                # === RMS
                
                if True:
                    
                    r_uII_uII_rms = np.sqrt(       data_mean["r u''u''"]  * rho_inf * U_inf**2 )
                    r_vII_vII_rms = np.sqrt(       data_mean["r v''v''"]  * rho_inf * U_inf**2 )
                    r_wII_wII_rms = np.sqrt(       data_mean["r w''_w''"] * rho_inf * U_inf**2 )
                    r_uII_vII_rms = np.sqrt(np.abs(data_mean["r u''v''"]) * rho_inf * U_inf**2 ) * np.sign(data_mean["r u''v''"]) 
                    r_uII_wII_rms = np.sqrt(np.abs(data_mean["r u''w''"]) * rho_inf * U_inf**2 ) * np.sign(data_mean["r u''w''"])
                    r_vII_wII_rms = np.sqrt(np.abs(data_mean["r w''v''"]) * rho_inf * U_inf**2 ) * np.sign(data_mean["r w''v''"])
                    ## ... rho*u''T''
                    
                    self.create_dataset( 'data/r_uII_uII_rms' , data=r_uII_uII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_vII_vII_rms' , data=r_vII_vII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_wII_wII_rms' , data=r_wII_wII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_uII_vII_rms' , data=r_uII_vII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_uII_wII_rms' , data=r_uII_wII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_vII_wII_rms' , data=r_vII_wII_rms.T , chunks=None )
                    ## ... rho*u''T''
        
        if fn_turb_budget.exists():
            #print('--r-> %s'%fn_turb_budget.relative_to(Path()) )
            even_print('eas4 turb budget',str(fn_turb_budget.relative_to(Path())))
            with eas4(str(fn_turb_budget),'r',verbose=False) as f1:
                
                data_mean = f1.get_mean() ## numpy structured array
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                turb_budget_total_avg_time       = f1.total_avg_time
                turb_budget_total_avg_iter_count = f1.total_avg_iter_count
                turb_budget_dt                   = turb_budget_total_avg_time/turb_budget_total_avg_iter_count
                
                self.attrs['turb_budget_total_avg_time'] = turb_budget_total_avg_time
                self.attrs['turb_budget_total_avg_iter_count'] = turb_budget_total_avg_iter_count
                self.attrs['turb_budget_dt'] = turb_budget_dt
                
                production     = data_mean['prod.']     * U_inf**3 * rho_inf / lchar
                dissipation    = data_mean['dis.']      * U_inf**2 * mu_inf  / lchar**2
                turb_transport = data_mean['t-transp.'] * U_inf**3 * rho_inf / lchar
                visc_diffusion = data_mean['v-diff.']   * U_inf**2 * mu_inf  / lchar**2
                p_diffusion    = data_mean['p-diff.']   * U_inf**3 * rho_inf / lchar
                p_dilatation   = data_mean['p-dilat.']  * U_inf**3 * rho_inf / lchar
                rho_terms      = data_mean['rho-terms'] * U_inf**3 * rho_inf / lchar
                ##
                dset = self.create_dataset('data/production'     , data=production.T     , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/dissipation'    , data=dissipation.T    , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/turb_transport' , data=turb_transport.T , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/visc_diffusion' , data=visc_diffusion.T , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/p_diffusion'    , data=p_diffusion.T    , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/p_dilatation'   , data=p_dilatation.T   , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/rho_terms'      , data=rho_terms.T      , chunks=None) #; dset.attrs['dimensional'] = True
                
                if 'dissipation' in locals():
                    
                    #if not self.get('data/nu').attrs['dimensional']:
                    #    raise ValueError('nu is not dimensional')
                    
                    Kolm_len = (nu**3 / np.abs(dissipation))**(1/4)
                    self.create_dataset('data/Kolm_len', data=Kolm_len.T, chunks=None); dset.attrs['dimensional'] = True
        
        # ===
        
        self.get_header(verbose=True)
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.ztmd.compile_data() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def export_dict(self,**kwargs):
        '''
        pull all data from HDF5 container into memory and pack it into a dictionary
        - convenient for multi-case plotting scripts
        '''
        verbose = kwargs.get('verbose',True)
        
        dd = {} ## the dict to return
        
        ## class-level ZTMD attrs
        attr_exclude_list = ['rank','comm','n_ranks','usingmpi','open_mode',
                             '_libver','_id','requires_wall_norm_interp',
                             'mod_avail_tqdm']
        for attr, val in self.__dict__.items():
            if (attr not in attr_exclude_list):
                
                ## throw error if not a routine type
                if isinstance(val, (int,np.int64,np.int32,float,str,dict,list,tuple,np.ndarray,bool,np.bool_,)):
                    pass
                elif (val is None):
                    pass
                else:
                    print(attr)
                    print(type(val))
                    raise TypeError
                
                dd[attr] = val
        
        ## HDF5 File Group attrs
        for attr, val in self.attrs.items():
            if (attr not in dd.keys()):
                dd[attr] = val
        
        ## Group: dims/
        for dsn in self['dims']:
            if (dsn not in dd.keys()):
                ds = self[f'dims/{dsn}']
                if (ds.ndim==0):
                    dd[dsn] = ds[()]
                elif (ds.ndim>0):
                    dd[dsn] = np.copy(ds[()])
                else:
                    raise ValueError
            else:
                #print(dsn)
                pass
        
        ## Group: data/
        for dsn in self['data']:
            if (dsn not in dd.keys()):
                ds = self[f'data/{dsn}']
                if (ds.ndim==2):
                    dd[dsn] = np.copy(ds[()].T)
                else:
                    raise ValueError
            else:
                #print(dsn)
                pass
        
        ## Group: data_1Dx/
        for dsn in self['data_1Dx']:
            if (dsn not in dd.keys()):
                ds = self[f'data_1Dx/{dsn}']
                if (ds.ndim==1) or (ds.ndim==2):
                    dd[dsn] = np.copy(ds[()])
                else:
                    raise ValueError
            else:
                #print(dsn)
                pass
        
        if verbose:
            print(f'>>> {self.fname}')
        
        return dd
    
    # ===
    
    def calc_gradients(self, acc=6, edge_stencil='full', **kwargs):
        '''
        calculate spatial gradients of averaged quantities
        '''
        
        verbose  = kwargs.get('verbose',True)
        do_favre = kwargs.get('favre',True)
        
        if verbose: print('\n'+'ztmd.calc_gradients()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print('acc','%i'%(acc,))
        if verbose: even_print('edge_stencil','%s'%(edge_stencil,))
        
        ## check
        if not hasattr(self,'rectilinear') and not hasattr(self,'curvilinear'):
            raise AssertionError('neither rectilinear nor curvilinear attr set')
        
        if hasattr(self,'rectilinear'):
            if self.rectilinear:
                if verbose: even_print('grid type','rectilinear')
        if hasattr(self,'curvilinear'):
            if self.curvilinear:
                if verbose: even_print('grid type','curvilinear')
        
        if (self.x.ndim==1) and (self.y.ndim==1):
            if hasattr(self,'rectilinear'):
                if not self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if self.curvilinear:
                    raise AssertionError
        elif (self.x.ndim==2) and (self.y.ndim==2):
            if hasattr(self,'rectilinear'):
                if self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if not self.curvilinear:
                    raise AssertionError
        else:
            raise ValueError
        
        # ===
        
        if self.curvilinear: ## get metric tensor 2D
            
            M = get_metric_tensor_2d(self.x, self.y, acc=acc, edge_stencil=edge_stencil, verbose=False)
            
            ddx_q1 = np.copy( M[:,:,0,0] ) ## ξ_x
            ddx_q2 = np.copy( M[:,:,1,0] ) ## η_x
            ddy_q1 = np.copy( M[:,:,0,1] ) ## ξ_y
            ddy_q2 = np.copy( M[:,:,1,1] ) ## η_y
            
            if verbose: even_print('ξ_x','%s'%str(ddx_q1.shape))
            if verbose: even_print('η_x','%s'%str(ddx_q2.shape))
            if verbose: even_print('ξ_y','%s'%str(ddy_q1.shape))
            if verbose: even_print('η_y','%s'%str(ddy_q2.shape))
            
            M = None; del M
            
            ## the 'computational' grid (unit Cartesian)
            #x_comp = np.arange(nx, dtype=np.float64)
            #y_comp = np.arange(ny, dtype=np.float64)
            x_comp = 1.
            y_comp = 1.
        
        print(72*'-')
        
        # === get gradients of [u,v,p,T,ρ]
        
        if ('data/u' in self):
            
            u = np.copy( self['data/u'][()].T )
            
            if self.rectilinear:
                ddx_u = gradient(u, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_u = gradient(u, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_u_comp = gradient(u, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_u_comp = gradient(u, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_u      = ddx_u_comp*ddx_q1 + ddy_u_comp*ddx_q2
                ddy_u      = ddx_u_comp*ddy_q1 + ddy_u_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_u' in self): del self['data/ddx_u']
            self.create_dataset('data/ddx_u', data=ddx_u.T, chunks=None)
            
            if ('data/ddy_u' in self): del self['data/ddy_u']
            self.create_dataset('data/ddy_u', data=ddy_u.T, chunks=None)
            
            if verbose: even_print('ddx[u]','%s'%str(ddx_u.shape))
            if verbose: even_print('ddy[u]','%s'%str(ddy_u.shape))
        
        if ('data/v' in self):
            
            v = np.copy( self['data/v'][()].T )
            
            if self.rectilinear:
                ddx_v = gradient(v, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_v = gradient(v, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_v_comp = gradient(v, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_v_comp = gradient(v, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_v      = ddx_v_comp*ddx_q1 + ddy_v_comp*ddx_q2
                ddy_v      = ddx_v_comp*ddy_q1 + ddy_v_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_v' in self): del self['data/ddx_v']
            self.create_dataset('data/ddx_v', data=ddx_v.T, chunks=None)
            
            if ('data/ddy_v' in self): del self['data/ddy_v']
            self.create_dataset('data/ddy_v', data=ddy_v.T, chunks=None)
            
            if verbose: even_print('ddx[v]','%s'%str(ddx_v.shape))
            if verbose: even_print('ddy[v]','%s'%str(ddy_v.shape))
        
        if ('data/p' in self):
            
            p = np.copy( self['data/p'][()].T )
            
            if self.rectilinear:
                ddx_p = gradient(p, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_p = gradient(p, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_p_comp = gradient(p, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_p_comp = gradient(p, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_p      = ddx_p_comp*ddx_q1 + ddy_p_comp*ddx_q2
                ddy_p      = ddx_p_comp*ddy_q1 + ddy_p_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_p' in self): del self['data/ddx_p']
            dset = self.create_dataset('data/ddx_p', data=ddx_p.T, chunks=None)
            
            if ('data/ddy_p' in self): del self['data/ddy_p']
            dset = self.create_dataset('data/ddy_p', data=ddy_p.T, chunks=None)
            
            if verbose: even_print('ddx[p]','%s'%str(ddx_p.shape))
            if verbose: even_print('ddy[p]','%s'%str(ddy_p.shape))
        
        if ('data/T' in self):
            
            T = np.copy( self['data/T'][()].T )
            
            if self.rectilinear:
                ddx_T = gradient(T, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_T = gradient(T, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_T_comp = gradient(T, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_T_comp = gradient(T, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_T      = ddx_T_comp*ddx_q1 + ddy_T_comp*ddx_q2
                ddy_T      = ddx_T_comp*ddy_q1 + ddy_T_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_T' in self): del self['data/ddx_T']
            dset = self.create_dataset('data/ddx_T', data=ddx_T.T, chunks=None)
            
            if ('data/ddy_T' in self): del self['data/ddy_T']
            dset = self.create_dataset('data/ddy_T', data=ddy_T.T, chunks=None)
            
            if verbose: even_print('ddx[T]','%s'%str(ddx_T.shape))
            if verbose: even_print('ddy[T]','%s'%str(ddy_T.shape))
        
        if ('data/rho' in self):
            
            r = np.copy( self['data/rho'][()].T )
            
            if self.rectilinear:
                ddx_r = gradient(r, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_r = gradient(r, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_r_comp = gradient(r, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_r_comp = gradient(r, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_r      = ddx_r_comp*ddx_q1 + ddy_r_comp*ddx_q2
                ddy_r      = ddx_r_comp*ddy_q1 + ddy_r_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_r' in self): del self['data/ddx_r']
            dset = self.create_dataset('data/ddx_r', data=ddx_r.T, chunks=None)
            
            if ('data/ddy_r' in self): del self['data/ddy_r']
            dset = self.create_dataset('data/ddy_r', data=ddy_r.T, chunks=None)
            
            if verbose: even_print('ddx[ρ]','%s'%str(ddx_r.shape))
            if verbose: even_print('ddy[ρ]','%s'%str(ddy_r.shape))
        
        # === vorticity
        
        ## z-vorticity :: ω_z
        vort_z = ddx_v - ddy_u
        
        if ('data/vort_z' in self): del self['data/vort_z']
        self.create_dataset('data/vort_z', data=vort_z.T, chunks=None)
        if verbose: even_print('ω_z','%s'%str(vort_z.shape))
        
        ## divergence (in xy-plane)
        div_xy = ddx_u + ddy_v
        
        if ('data/div_xy' in self): del self['data/div_xy']
        self.create_dataset('data/div_xy', data=div_xy.T, chunks=None)
        if verbose: even_print('div_xy','%s'%str(div_xy.shape))
        
        # === 
        
        if ('data/utang' in self):
            
            utang = np.copy( self['data/utang'][()].T )
            
            if self.rectilinear:
                ddx_utang = gradient(utang, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_utang = gradient(utang, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_utang_comp = gradient(utang, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_utang_comp = gradient(utang, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_utang      = ddx_utang_comp*ddx_q1 + ddy_utang_comp*ddx_q2
                ddy_utang      = ddx_utang_comp*ddy_q1 + ddy_utang_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_utang' in self): del self['data/ddx_utang']
            dset = self.create_dataset('data/ddx_utang', data=ddx_utang.T, chunks=None)
            if verbose: even_print('ddx[utang]','%s'%str(ddx_utang.shape))
            
            if ('data/ddy_utang' in self): del self['data/ddy_utang']
            dset = self.create_dataset('data/ddy_utang', data=ddy_utang.T, chunks=None)
            if verbose: even_print('ddy[utang]','%s'%str(ddy_utang.shape))
        
        if ('data/unorm' in self):
            
            unorm = np.copy( self['data/unorm'][()].T )
            
            if self.rectilinear:
                ddx_unorm = gradient(unorm, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_unorm = gradient(unorm, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_unorm_comp = gradient(unorm, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_unorm_comp = gradient(unorm, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_unorm      = ddx_unorm_comp*ddx_q1 + ddy_unorm_comp*ddx_q2
                ddy_unorm      = ddx_unorm_comp*ddy_q1 + ddy_unorm_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_unorm' in self): del self['data/ddx_unorm']
            dset = self.create_dataset('data/ddx_unorm', data=ddx_unorm.T, chunks=None)
            if verbose: even_print('ddx[unorm]','%s'%str(ddx_unorm.shape))
            
            if ('data/ddy_unorm' in self): del self['data/ddy_unorm']
            dset = self.create_dataset('data/ddy_unorm', data=ddy_unorm.T, chunks=None)
            if verbose: even_print('ddy[unorm]','%s'%str(ddy_unorm.shape))
        
        # === Favre
        
        if do_favre:
            
            print(72*'-')
            
            if ('data/u_Fv' in self):
                
                u_Fv = np.copy( self['data/u_Fv'][()].T )
                
                if self.rectilinear:
                    ddx_u_Fv = gradient(u_Fv, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddy_u_Fv = gradient(u_Fv, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                elif self.curvilinear:
                    ddx_u_Fv_comp = gradient(u_Fv, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddy_u_Fv_comp = gradient(u_Fv, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddx_u_Fv      = ddx_u_Fv_comp*ddx_q1 + ddy_u_Fv_comp*ddx_q2
                    ddy_u_Fv      = ddx_u_Fv_comp*ddy_q1 + ddy_u_Fv_comp*ddy_q2
                else:
                    raise ValueError
                
                if ('data/ddx_u_Fv' in self): del self['data/ddx_u_Fv']
                self.create_dataset('data/ddx_u_Fv', data=ddx_u_Fv.T, chunks=None)
                
                if ('data/ddy_u_Fv' in self): del self['data/ddy_u_Fv']
                self.create_dataset('data/ddy_u_Fv', data=ddy_u_Fv.T, chunks=None)
                
                if verbose: even_print('ddx[u_Fv]','%s'%str(ddx_u_Fv.shape))
                if verbose: even_print('ddy[u_Fv]','%s'%str(ddy_u_Fv.shape))
            
            if ('data/v_Fv' in self):
                
                v_Fv = np.copy( self['data/v_Fv'][()].T )
                
                if self.rectilinear:
                    ddx_v_Fv = gradient(v_Fv, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddy_v_Fv = gradient(v_Fv, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                elif self.curvilinear:
                    ddx_v_Fv_comp = gradient(v_Fv, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddy_v_Fv_comp = gradient(v_Fv, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddx_v_Fv      = ddx_v_Fv_comp*ddx_q1 + ddy_v_Fv_comp*ddx_q2
                    ddy_v_Fv      = ddx_v_Fv_comp*ddy_q1 + ddy_v_Fv_comp*ddy_q2
                else:
                    raise ValueError
                
                if ('data/ddx_v_Fv' in self): del self['data/ddx_v_Fv']
                self.create_dataset('data/ddx_v_Fv', data=ddx_v_Fv.T, chunks=None)
                
                if ('data/ddy_v_Fv' in self): del self['data/ddy_v_Fv']
                self.create_dataset('data/ddy_v_Fv', data=ddy_v_Fv.T, chunks=None)
                
                if verbose: even_print('ddx[v_Fv]','%s'%str(ddx_v_Fv.shape))
                if verbose: even_print('ddy[v_Fv]','%s'%str(ddy_v_Fv.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_gradients() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_wall_quantities(self, acc=6, edge_stencil='full', **kwargs):
        '''
        get 1D wall quantities
        -----
        - [ ρ_wall, ν_wall, μ_wall, T_wall ]
        - τ_wall = μ_wall·ddn[utang] :: [kg/(m·s)]·[m/s]/[m] = [kg/(m·s²)] = [N/m²] = [Pa]
        - u_τ = (τ_wall/ρ_wall)^(1/2)
        '''
        
        verbose = kwargs.get('verbose',True)
        if verbose: print('\n'+'ztmd.calc_wall_quantities()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if (self.x.ndim==1) and (self.y.ndim==1):
            if hasattr(self,'rectilinear'):
                if not self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if self.curvilinear:
                    raise AssertionError
        elif (self.x.ndim==2) and (self.y.ndim==2):
            if hasattr(self,'rectilinear'):
                if self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if not self.curvilinear:
                    raise AssertionError
        else:
            raise ValueError
        
        # ===
        
        if self.curvilinear:
            
            if self.requires_wall_norm_interp:
                
                gndata = 'data_2Dw' ## group name for 2D data (interpolated)
                gndims = 'dims_2Dw' ## group name for 2D dims (interpolated)
                
                ## wall-normal interpolation coordinates (2D)
                x_wn = np.copy( self[f'{gndata}/x'][()].T )
                y_wn = np.copy( self[f'{gndata}/y'][()].T )
                s_wn = np.copy( self[f'{gndata}/wall_distance'][()].T )
            
            else:
                
                gndata = 'data' ## group name for 2D data
                gndims = 'dims' ## group name for 2D dims
                
                x_wn = np.copy( self[f'dims/x'][()].T )
                y_wn = np.copy( self[f'dims/y'][()].T )
                s_wn = np.copy( self[f'dims/snorm'][()] )
                #s_wn = np.broadcast_to(s_wn, (self.nx,self.ny))
            
            if (f'{gndata}/utang' not in self):
                raise AssertionError(f'{gndata}/utang not present')
            
            ## wall-normal interpolated scalars (2D)
            utang_wn  = np.copy( self[f'{gndata}/utang'][()].T  )
            T_wn      = np.copy( self[f'{gndata}/T'][()].T      )
            vort_z_wn = np.copy( self[f'{gndata}/vort_z'][()].T )
        
        # === get ρ_wall, ν_wall, μ_wall, T_wall
        
        rho = np.copy( self['data/rho'][()].T )
        rho_wall = np.copy( rho[:,0] )
        if ('data_1Dx/rho_wall' in self): del self['data_1Dx/rho_wall']
        dset = self.create_dataset('data_1Dx/rho_wall', data=rho_wall, chunks=None)
        #dset.attrs['dimensional'] = False
        #dset.attrs['unit'] = '[kg/m³]'
        if verbose: even_print('data_1Dx/rho_wall','%s'%str(rho_wall.shape))
        
        nu = np.copy( self['data/nu'][()].T )
        nu_wall = np.copy( nu[:,0] )
        if ('data_1Dx/nu_wall' in self): del self['data_1Dx/nu_wall']
        dset = self.create_dataset('data_1Dx/nu_wall', data=nu_wall, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[m²/s]'
        if verbose: even_print('data_1Dx/nu_wall','%s'%str(nu_wall.shape))
        
        mu = np.copy( self['data/mu'][()].T )
        mu_wall = np.copy( mu[:,0] )
        if ('data_1Dx/mu_wall' in self): del self['data_1Dx/mu_wall']
        dset = self.create_dataset('data_1Dx/mu_wall', data=mu_wall, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[kg/(m·s)]'
        if verbose: even_print('data_1Dx/mu_wall','%s'%str(mu_wall.shape))
        
        T = np.copy( self['data/T'][()].T )
        T_wall = np.copy( T[:,0] )
        if ('data_1Dx/T_wall' in self): del self['data_1Dx/T_wall']
        dset = self.create_dataset('data_1Dx/T_wall', data=T_wall, chunks=None)
        #dset.attrs['dimensional'] = False
        #dset.attrs['unit'] = '[K]'
        if verbose: even_print('data_1Dx/T_wall','%s'%str(T_wall.shape))
        
        # === get wall ddn[]
        
        if self.rectilinear:
            
            ddy_u = np.copy( self['data/ddy_u'][()].T )
        
        elif self.curvilinear:
            
            if True:
                
                if (s_wn.ndim==2): ## wall-normal distance (s_norm) is a 2D field
                    
                    ddn_utang  = np.zeros((self.nx,self.ny), dtype=np.float64) ## dimensional [m/s]/[m] = [1/s]
                    ddn_vort_z = np.zeros((self.nx,self.ny), dtype=np.float64)
                    
                    progress_bar = tqdm(total=self.nx, ncols=100, desc='get ddn[]', leave=False, file=sys.stdout)
                    for i in range(self.nx):
                        ddn_utang[i,:]  = gradient(utang_wn[i,:]  , s_wn[i,:], axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                        ddn_vort_z[i,:] = gradient(vort_z_wn[i,:] , s_wn[i,:], axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                        progress_bar.update()
                    progress_bar.close()
                
                elif (s_wn.ndim==1): ## wall-normal distance (s_norm) is a 1D vector
                    
                    ddn_utang  = gradient(utang_wn  , s_wn, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddn_vort_z = gradient(vort_z_wn , s_wn, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                
                if (f'{gndata}/ddn_utang' in self): del self[f'{gndata}/ddn_utang']
                dset = self.create_dataset(f'{gndata}/ddn_utang', data=ddn_utang.T, chunks=None)
                #dset.attrs['dimensional'] = True; dset.attrs['unit'] = '1/s'
                
                if (f'{gndata}/ddn_vort_z' in self): del self[f'{gndata}/ddn_vort_z']
                dset = self.create_dataset(f'{gndata}/ddn_vort_z', data=ddn_vort_z.T, chunks=None)
                
                if ('data_1Dx/ddn_utang_wall' in self): del self['data_1Dx/ddn_utang_wall']
                dset = self.create_dataset('data_1Dx/ddn_utang_wall', data=ddn_utang[:,0], chunks=None)
                #dset.attrs['dimensional'] = True; dset.attrs['unit'] = '1/s'
            
            else:
                
                ddn_utang  = np.copy( self[f'{gndata}/ddn_utang'][()].T  )
                ddn_vort_z = np.copy( self[f'{gndata}/ddn_vort_z'][()].T )
        
        else:
            raise ValueError
        
        # === calculate τ_wall & u_τ
        
        ## wall shear stress τ_wall
        if self.rectilinear:
            tau_wall = np.copy( mu_wall * ddy_u[:,0] )
        elif self.curvilinear:
            tau_wall = np.copy( mu_wall * ddn_utang[:,0] )
        else:
            raise ValueError
        
        if ('data_1Dx/tau_wall' in self): del self['data_1Dx/tau_wall']
        dset = self.create_dataset('data_1Dx/tau_wall', data=tau_wall, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[N/m²]' ## = [kg/(m·s²)] = [N/m²] = [Pa]
        if verbose: even_print('data_1Dx/tau_wall','%s'%str(tau_wall.shape))
        
        ## friction velocity u_τ [m/s]
        u_tau = np.copy( np.sqrt( tau_wall / rho_wall ) )
        
        if ('data_1Dx/u_tau' in self): del self['data_1Dx/u_tau']
        dset = self.create_dataset('data_1Dx/u_tau', data=u_tau, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[m/s]'
        if verbose: even_print('data_1Dx/u_tau','%s'%str(u_tau.shape))
        
        # === inner scales: length, velocity & time
        
        sc_u_in = np.copy( u_tau              )
        sc_l_in = np.copy( nu_wall / u_tau    )
        sc_t_in = np.copy( nu_wall / u_tau**2 )
        np.testing.assert_allclose(sc_t_in, sc_l_in/sc_u_in, rtol=1e-6, atol=1e-12)
        
        if ('data_1Dx/sc_u_in' in self): del self['data_1Dx/sc_u_in']
        dset = self.create_dataset('data_1Dx/sc_u_in', data=sc_u_in, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[m/s]'
        if verbose: even_print('data_1Dx/sc_u_in','%s'%str(sc_u_in.shape))
        
        if ('data_1Dx/sc_l_in' in self): del self['data_1Dx/sc_l_in']
        dset = self.create_dataset('data_1Dx/sc_l_in', data=sc_l_in, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[m]'
        if verbose: even_print('data_1Dx/sc_l_in','%s'%str(sc_l_in.shape))
        
        if ('data_1Dx/sc_t_in' in self): del self['data_1Dx/sc_t_in']
        dset = self.create_dataset('data_1Dx/sc_t_in', data=sc_t_in, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[s]'
        if verbose: even_print('data_1Dx/sc_t_in','%s'%str(sc_t_in.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_wall_quantities() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === curvilinear / curved TBL specific
    
    def calc_s_wall(self,**kwargs):
        '''
        calculate wall/top path length 's' (numerically integrated, not continuous!)
        '''
        
        verbose = kwargs.get('verbose',True)
        if verbose: print('\n'+'ztmd.calc_s_wall()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if self.rectilinear:
            
            s_wall = np.copy( self.x - self.x.min() )
            s_top  = np.copy( self.x - self.x.min() )
        
        else: ## curvilinear
            
            ds_wall = np.sqrt( np.diff(self.x[:,0])**2 + np.diff(self.y[:,0])**2 )
            s_wall  = np.cumsum(np.concatenate([[0.],ds_wall]))
            
            ds_top = np.sqrt( np.diff(self.x[:,-1])**2 + np.diff(self.y[:,-1])**2 )
            s_top  = np.cumsum(np.concatenate([[0.],ds_top]))
        
        if verbose: even_print( 's_wall/lchar max' , '%0.8f'%(s_wall.max()/self.lchar) )
        if verbose: even_print( 's_top/lchar max'  , '%0.8f'%(s_top.max()/self.lchar)  )
        
        if ('dims/s_wall' in self):
            del self['dims/s_wall']
        dset = self.create_dataset('dims/s_wall', data=s_wall, chunks=None)
        if verbose: even_print( 'dims/s_wall', str(dset.shape) )
        
        if ('dims/s_top' in self):
            del self['dims/s_top']
        dset = self.create_dataset('dims/s_top', data=s_top, chunks=None)
        if verbose: even_print( 'dims/s_top', str(dset.shape) )
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_s_wall() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_geom_data(self, fn_dat=None, **kwargs):
        '''
        add geom data (from pickled .dat, usually from tgg)
        - dims/stang : (nx,)
        - dims/snorm : (ny,)
        - dims/crv_R : (nx,)
        '''
        
        verbose = kwargs.get('verbose',True)
        if verbose: print('\n'+'ztmd.add_geom_data()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_dat):
            raise FileNotFoundError('file does not exist: %s'%str(fn_dat))
        
        ## open data file from tgg
        with open(fn_dat,'rb') as f:
            dd = pickle.load(f)
        
        if ('xy2d' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain xy2d')
        
        ## check if consistent with [xy] grid in ref file
        xy2d = np.copy( dd['xy2d'] )
        
        if (self.x.ndim==2) and (self.y.ndim==2):
            np.testing.assert_allclose(xy2d[:,:,0], self.x/self.lchar, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,:,1], self.y/self.lchar, rtol=1e-14, atol=1e-14)
        elif (self.x.ndim==1) and (self.y.ndim==1):
            np.testing.assert_allclose(xy2d[:,0,0], self.x/self.lchar, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[0,:,1], self.y/self.lchar, rtol=1e-14, atol=1e-14)
        else:
            raise ValueError
        
        if verbose: even_print('check passed', 'x grid')
        if verbose: even_print('check passed', 'y grid')
        
        ## key names for 'dims' group
        kn_dims = ['stang', 'snorm', 'crv_R'] 
        for k in kn_dims:
            if (k in dd.keys()):
                dsn = f'dims/{k}'
                if (dd[k] is None):
                    continue
                data = np.copy(dd[k])
                data *= self.lchar ## re-dimensionalize (ztmd is dimensional)
                if (dsn in self): del self[dsn]
                ds = self.create_dataset(dsn, data=data, chunks=None)
                if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        kn_dims = ['z', 's', 's_top', 's_wall', 'R_min', 'path_len', 'W', 'H'] 
        for k in kn_dims:
            if (k in dd.keys()):
                dsn = f'dims/{k}'
                if (dd[k] is None):
                    continue
                if isinstance(dd[k], np.ndarray):
                    data = np.copy(dd[k])
                    data *= self.lchar ## re-dimensionalize (ztmd is dimensional)
                if isinstance(dd[k], (int,float)):
                    data = float(dd[k])
                    data *= self.lchar ## re-dimensionalize (ztmd is dimensional)
                if (dsn in self): del self[dsn]
                ds = self.create_dataset(dsn, data=data, chunks=None)
                if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        kn_dims = ['curve_arc_angle']
        for k in kn_dims:
            if (k in dd.keys()):
                dsn = f'dims/{k}'
                if (dd[k] is None):
                    continue
                if (dsn in self): del self[dsn]
                ds = self.create_dataset(dsn, data=dd[k], chunks=None)
                if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        ## check
        if self.rectilinear:
            
            stang_ = self[f'dims/stang'][()]
            x_ = self[f'dims/x'][()]
            np.testing.assert_allclose(stang_, x_-x_.min(), rtol=1e-14, atol=1e-14)
            
            snorm_ = self[f'dims/snorm'][()]
            y_ = self[f'dims/y'][()]
            np.testing.assert_allclose(snorm_, y_-y_.min(), rtol=1e-14, atol=1e-14)
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.add_geom_data() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_csys_vecs_xy(self, fn_dat=None, **kwargs):
        '''
        add csys data (from pickled .dat, usually from tgg)
        - csys/vtang : (nx,ny,2)
        - csys/vnorm : (nx,ny,2)
        '''
        
        verbose = kwargs.get('verbose',True)
        if verbose: print('\n'+'ztmd.add_csys_vecs_xy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_dat):
            raise FileNotFoundError('file does not exist: %s'%str(fn_dat))
        
        ## open data file from tgg
        with open(fn_dat,'rb') as f:
            dd = pickle.load(f)
        
        if ('xy2d' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain xy2d')
        if ('vtang' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain vtang')
        if ('vnorm' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain vnorm')
        
        xy2d  = dd['xy2d']
        vtang = dd['vtang']
        vnorm = dd['vnorm']
        
        #wall_distance = dd['wall_distance']
        #s_wall_2d     = dd['s_wall_2d'] ## curve path length of point on wall (nx,ny)
        #p_wall_2d     = dd['p_wall_2d'] ## projection point on wall (nx,ny,2)
        
        if (self.x.ndim==2) and (self.y.ndim==2):
            np.testing.assert_allclose(xy2d[:,:,0], self.x/self.lchar, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,:,1], self.y/self.lchar, rtol=1e-14, atol=1e-14)
        elif (self.x.ndim==1) and (self.y.ndim==1):
            np.testing.assert_allclose(xy2d[:,0,0], self.x/self.lchar, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[0,:,1], self.y/self.lchar, rtol=1e-14, atol=1e-14)
        else:
            raise ValueError
        
        if verbose: even_print('check passed', 'x grid')
        if verbose: even_print('check passed', 'y grid')
        
        dd = None; del dd
        xy2d = None; del xy2d
        
        ## re-dimensionalize
        #wall_distance *= self.lchar
        
        #if not (wall_distance.shape == (self.nx,self.ny)):
        #    raise ValueError('wall_distance.shape != (self.nx,self.ny)')
        if not (vtang.shape == (self.nx,self.ny,2)):
            raise ValueError('vtang.shape != (self.nx,self.ny,2)')
        if not (vnorm.shape == (self.nx,self.ny,2)):
            raise ValueError('vnorm.shape != (self.nx,self.ny,2)')
        
        ## write wall distance scalar (nx,ny)
        #if ('data/wall_distance' in self): del self['data/wall_distance']
        #self.create_dataset('data/wall_distance', data=wall_distance.T, chunks=None)
        
        ## write wall normal / tangent basis vectors
        if ('csys/vtang' in self): del self['csys/vtang']
        dset = self.create_dataset('csys/vtang', data=vtang, chunks=None)
        if verbose: even_print('csys/vtang',str(dset.shape))
        
        if ('csys/vnorm' in self): del self['csys/vnorm']
        dset = self.create_dataset('csys/vnorm', data=vnorm, chunks=None)
        if verbose: even_print('csys/vnorm',str(dset.shape))
        
        ## write continuous wall point coordinate & wall path length
        #if ('csys/s_wall_2d' in self): del self['csys/s_wall_2d']
        #self.create_dataset('csys/s_wall_2d', data=s_wall_2d, chunks=None)
        #if ('csys/p_wall_2d' in self): del self['csys/p_wall_2d']
        #self.create_dataset('csys/p_wall_2d', data=p_wall_2d, chunks=None)
        
        ## check
        if self.rectilinear:
            
            vtang_        = np.zeros((self.nx,self.ny,2),dtype=np.float64)
            vtang_[:,:,:] = np.array([1,0],dtype=np.float64)
            np.testing.assert_allclose(vtang_, vtang, rtol=1e-14, atol=1e-14)
            
            vnorm_        = np.zeros((self.nx,self.ny,2),dtype=np.float64)
            vnorm_[:,:,:] = np.array([0,1],dtype=np.float64)
            np.testing.assert_allclose(vnorm_, vnorm, rtol=1e-14, atol=1e-14)
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.add_csys_vecs_xy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vel_tangnorm(self, **kwargs):
        '''
        add tangent & normal velocity [utang,unorm] to file
        '''
        
        verbose = kwargs.get('verbose',True)
        if verbose: print('\n'+'ztmd.calc_vel_tangnorm()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('csys/vtang' in self):
            raise ValueError('csys/vtang not in hdf5')
        if not ('csys/vnorm' in self):
            raise ValueError('csys/vnorm not in hdf5')
        if not (self.open_mode=='a') or (self.open_mode=='w'):
            raise ValueError('not able to write to hdf5 file')
        
        ## read unit vectors (wall tangent, wall norm) from HDF5
        vtang = np.copy( self['csys/vtang'][()] )
        vnorm = np.copy( self['csys/vnorm'][()] )
        
        # === Reynolds
        
        if ('data/u' in self) and ('data/v' in self):
            
            ## read 2D velocities
            u  = np.copy( self['data/u'][()].T )
            v  = np.copy( self['data/v'][()].T )
            uv = np.stack((u,v), axis=-1)
            
            umag1 = np.copy( np.sqrt( u**2 + v**2 ) )
            
            ## inner product of velocity vector and basis vector (csys transform)
            utang = np.einsum('xyi,xyi->xy', vtang, uv)
            unorm = np.einsum('xyi,xyi->xy', vnorm, uv)
            
            umag2 = np.copy( np.sqrt( utang**2 + unorm**2 ) )
            
            if (umag1.dtype==np.dtype(np.float32)):
                np.testing.assert_allclose(umag1, umag2, rtol=1e-6) ## single precision
            elif (umag2.dtype==np.dtype(np.float64)):
                np.testing.assert_allclose(umag1, umag2, rtol=1e-12) ## double precision
            else:
                raise ValueError
            
            # if self.get('data/u').attrs['dimensional']:
            #     raise AssertionError('u is dimensional')
            # if self.get('data/v').attrs['dimensional']:
            #     raise AssertionError('v is dimensional')
            
            if ('data/utang' in self): del self['data/utang']
            dset = self.create_dataset('data/utang', data=utang.T, chunks=None)
            #dset.attrs['dimensional'] = False
            if verbose: even_print('utang','%s'%str(utang.shape))
            
            if ('data/unorm' in self): del self['data/unorm']
            dset = self.create_dataset('data/unorm', data=unorm.T, chunks=None)
            #dset.attrs['dimensional'] = False
            if verbose: even_print('unorm','%s'%str(unorm.shape))
            
            ## assert that in rectilinear case that u==utang & v==unorm
            if self.rectilinear:
                np.testing.assert_allclose(u, utang, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(v, unorm, rtol=1e-14, atol=1e-14)
        
        # === Favre
        
        if ('data/u_Fv' in self) and ('data/v_Fv' in self):
            
            ## read 2D velocities
            u_Fv     = np.copy( self['data/u_Fv'][()].T )
            v_Fv     = np.copy( self['data/v_Fv'][()].T )
            uv_Fv    = np.stack((u_Fv,v_Fv), axis=-1)
            umag1_Fv = np.copy( np.sqrt( u_Fv**2 + v_Fv**2 ) )
            
            ## inner product of velocity vector and basis vector (csys transform)
            utang_Fv = np.einsum('xyi,xyi->xy', vtang, uv_Fv)
            unorm_Fv = np.einsum('xyi,xyi->xy', vnorm, uv_Fv)
            
            umag2_Fv = np.copy( np.sqrt( utang_Fv**2 + unorm_Fv**2 ) )
            
            if (umag1_Fv.dtype==np.dtype(np.float32)):
                np.testing.assert_allclose(umag1_Fv, umag2_Fv, rtol=1e-6) ## single precision
            elif (umag2_Fv.dtype==np.dtype(np.float64)):
                np.testing.assert_allclose(umag1_Fv, umag2_Fv, rtol=1e-12) ## double precision
            else:
                raise ValueError
            
            if ('data/utang_Fv' in self): del self['data/utang_Fv']
            dset = self.create_dataset('data/utang_Fv', data=utang_Fv.T, chunks=None)
            if verbose: even_print('utang_Fv','%s'%str(utang_Fv.shape))
            
            if ('data/unorm_Fv' in self): del self['data/unorm_Fv']
            dset = self.create_dataset('data/unorm_Fv', data=unorm_Fv.T, chunks=None)
            #dset.attrs['dimensional'] = False
            if verbose: even_print('unorm_Fv','%s'%str(unorm_Fv.shape))
            
            ## assert that in rectilinear case that u==utang & v==unorm
            if self.rectilinear:
                np.testing.assert_allclose(u_Fv, utang_Fv, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(v_Fv, unorm_Fv, rtol=1e-14, atol=1e-14)
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_vel_tangnorm() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vel_tangnorm_mean_removed(self, **kwargs):
        '''
        calculate utangI_utangI, unormI_unormI
        '''
        
        verbose = kwargs.get('verbose',True)
        if verbose: print('\n'+'ztmd.calc_vel_tangnorm_mean_removed()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        #if not ('data/wall_distance' in self):
        #    raise ValueError('data/wall_distance not in hdf5')
        if not ('dims/snorm' in self):
            raise ValueError('dims/snorm not in hdf5')
        if not ('data/uI_uI' in self):
            raise ValueError('data/uI_uI not in hdf5')
        if not ('data/vI_vI' in self):
            raise ValueError('data/vI_vI not in hdf5')
        if not ('csys/vtang' in self):
            raise ValueError('csys/vtang not in hdf5')
        if not ('csys/vnorm' in self):
            raise ValueError('csys/vnorm not in hdf5')
        if not (self.open_mode=='a') or (self.open_mode=='w'):
            raise ValueError('not able to write to hdf5 file')
        
        ## read unit vectors (wall tangent, wall norm) from HDF5
        vtang = np.copy( self['csys/vtang'][()] )
        vnorm = np.copy( self['csys/vnorm'][()] )
        trafo = np.stack((vtang,vnorm),axis=-1)
        
        ## transpose [2,2] matrix at every [x,y]
        trafoT = np.transpose(trafo,(0,1,3,2))
        
        # === Reynolds
        
        if ('data/uI_uI' in self) and ('data/uI_vI' in self) and ('data/vI_vI' in self):
            
            uI_uI = np.copy( self['data/uI_uI'][()].T )
            vI_vI = np.copy( self['data/vI_vI'][()].T )
            uI_vI = np.copy( self['data/uI_vI'][()].T )
            vI_uI = np.copy( uI_vI )
            
            ## construct 2D tensor --> (nx,ny,2,2)
            uIuI_ij = np.stack( (np.stack( [uI_uI, uI_vI], axis=-1 ),
                                 np.stack( [vI_uI, vI_vI], axis=-1 )), axis=-1 )
            
            #uIuI_ij = np.transpose(uIuI_ij, (0,1,3,2))
            
            uI_uI_rms = np.copy( self['data/uI_uI_rms'][()].T )
            vI_vI_rms = np.copy( self['data/vI_vI_rms'][()].T )
            uI_vI_rms = np.copy( self['data/uI_vI_rms'][()].T )
            vI_uI_rms = np.copy( uI_vI_rms )
            
            ## construct 2D tensor --> (nx,ny,2,2)
            uIuI_rms_ij = np.stack( (np.stack( [uI_uI_rms, uI_vI_rms], axis=-1 ),
                                     np.stack( [vI_uI_rms, vI_vI_rms], axis=-1 )), axis=-1 )
            
            #uIuI_rms_ij = np.transpose(uIuI_rms_ij, (0,1,3,2))
            
            ## check
            np.testing.assert_allclose(uI_uI, uI_uI_rms**2, rtol=1e-6, atol=1e-6)
            np.testing.assert_allclose(vI_vI, vI_vI_rms**2, rtol=1e-6, atol=1e-6)
            
            ## these three ops are the same
            #uIuI_ij_tn_A = np.einsum('xyij,xyjk,xykl->xyil', trafoT, uIuI_ij, trafo)
            #uIuI_ij_tn_B = np.einsum('xyji,xyjk,xykl->xyil', trafo,  uIuI_ij, trafo)
            #uIuI_ij_tn_C = np.matmul(np.matmul(trafoT, uIuI_ij), trafo)
            #np.testing.assert_allclose(uIuI_ij_tn_A, uIuI_ij_tn_B, atol=1e-14, rtol=1e-14)
            #np.testing.assert_allclose(uIuI_ij_tn_B, uIuI_ij_tn_C, atol=1e-14, rtol=1e-14)
            
            uIuI_ij_tn     = np.einsum('xyji,xyjk,xykl->xyil', trafo, uIuI_ij,     trafo)
            uIuI_rms_ij_tn = np.einsum('xyji,xyjk,xykl->xyil', trafo, uIuI_rms_ij, trafo)
            
            utI_utI = np.copy(uIuI_ij_tn[:,:,0,0])
            unI_unI = np.copy(uIuI_ij_tn[:,:,1,1])
            utI_unI = np.copy(uIuI_ij_tn[:,:,0,1])
            unI_utI = np.copy(uIuI_ij_tn[:,:,1,0])
            
            np.testing.assert_allclose( utI_unI, unI_utI, rtol=1e-14, atol=1e-14 )
            
            utI_utI_rms = np.copy(uIuI_rms_ij_tn[:,:,0,0])
            unI_unI_rms = np.copy(uIuI_rms_ij_tn[:,:,1,1])
            utI_unI_rms = np.copy(uIuI_rms_ij_tn[:,:,0,1])
            unI_utI_rms = np.copy(uIuI_rms_ij_tn[:,:,1,0])
            
            np.testing.assert_allclose( utI_unI_rms, unI_utI_rms, rtol=1e-14, atol=1e-14 )
            
            ## fails
            #np.testing.assert_allclose( utI_utI, utI_utI_rms**2, rtol=1e-6, atol=1e-6 )
            #np.testing.assert_allclose( unI_unI, unI_unI_rms**2, rtol=1e-6, atol=1e-6 )
            
            ## only write full covariances, not RMSes
            
            if ('data/utI_utI' in self): del self['data/utI_utI']
            dset = self.create_dataset('data/utI_utI', data=utI_utI.T, chunks=None)
            if verbose: even_print('utI_utI','%s'%str(utI_utI.shape))
            
            if ('data/unI_unI' in self): del self['data/unI_unI']
            dset = self.create_dataset('data/unI_unI', data=unI_unI.T, chunks=None)
            if verbose: even_print('unI_unI','%s'%str(unI_unI.shape))
            
            if ('data/utI_unI' in self): del self['data/utI_unI']
            dset = self.create_dataset('data/utI_unI', data=utI_unI.T, chunks=None)
            if verbose: even_print('utI_unI','%s'%str(utI_unI.shape))
            
            ## check
            if self.rectilinear:
                np.testing.assert_allclose(utI_utI, uI_uI, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(unI_unI, vI_vI, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(utI_unI, uI_vI, rtol=1e-14, atol=1e-14)
        
        # === Favre
        
        if ('data/r_uII_uII' in self) and ('data/r_uII_vII' in self) and ('data/r_vII_vII' in self):
            
            r_uII_uII = np.copy( self['data/r_uII_uII'][()].T )
            r_vII_vII = np.copy( self['data/r_vII_vII'][()].T )
            r_uII_vII = np.copy( self['data/r_uII_vII'][()].T )
            r_vII_uII = np.copy( r_uII_vII )
            
            ## construct 2D tensor --> (nx,ny,2,2)
            r_uIIuII_ij = np.stack( (np.stack( [r_uII_uII, r_uII_vII], axis=-1 ),
                                     np.stack( [r_vII_uII, r_vII_vII], axis=-1 )), axis=-1 )
            
            r_uII_uII_rms = np.copy( self['data/r_uII_uII_rms'][()].T )
            r_vII_vII_rms = np.copy( self['data/r_vII_vII_rms'][()].T )
            r_uII_vII_rms = np.copy( self['data/r_uII_vII_rms'][()].T )
            r_vII_uII_rms = np.copy( r_uII_vII_rms )
            
            ## construct 2D tensor --> (nx,ny,2,2)
            r_uIIuII_rms_ij = np.stack( (np.stack( [r_uII_uII_rms, r_uII_vII_rms], axis=-1 ),
                                         np.stack( [r_vII_uII_rms, r_vII_vII_rms], axis=-1 )), axis=-1 )
            
            ## check
            np.testing.assert_allclose(r_uII_uII, r_uII_uII_rms**2, rtol=1e-6, atol=1e-6)
            np.testing.assert_allclose(r_vII_vII, r_vII_vII_rms**2, rtol=1e-6, atol=1e-6)
            
            r_uIIuII_ij_tn     = np.einsum('xyji,xyjk,xykl->xyil', trafo, r_uIIuII_ij,     trafo)
            r_uIIuII_rms_ij_tn = np.einsum('xyji,xyjk,xykl->xyil', trafo, r_uIIuII_rms_ij, trafo)
            
            r_utII_utII = np.copy(r_uIIuII_ij_tn[:,:,0,0])
            r_unII_unII = np.copy(r_uIIuII_ij_tn[:,:,1,1])
            r_utII_unII = np.copy(r_uIIuII_ij_tn[:,:,0,1])
            r_unII_utII = np.copy(r_uIIuII_ij_tn[:,:,1,0])
            
            np.testing.assert_allclose( r_utII_unII, r_unII_utII, rtol=1e-14, atol=1e-14 )
            
            r_utII_utII_rms = np.copy(uIuI_rms_ij_tn[:,:,0,0])
            r_unII_unII_rms = np.copy(uIuI_rms_ij_tn[:,:,1,1])
            r_utII_unII_rms = np.copy(uIuI_rms_ij_tn[:,:,0,1])
            r_unII_utII_rms = np.copy(uIuI_rms_ij_tn[:,:,1,0])
            
            np.testing.assert_allclose( r_utII_unII_rms, r_unII_utII_rms, rtol=1e-14, atol=1e-14 )
            
            ## fails!
            #np.testing.assert_allclose( r_utII_utII, r_utII_utII_rms**2, rtol=1e-6, atol=1e-6 )
            #np.testing.assert_allclose( r_unII_unII, r_unII_unII_rms**2, rtol=1e-6, atol=1e-6 )
            
            ## only write full covariances, not RMSes
            
            if ('data/r_utII_utII' in self): del self['data/r_utII_utII']
            dset = self.create_dataset('data/r_utII_utII', data=r_utII_utII.T, chunks=None)
            if verbose: even_print('r_utII_utII','%s'%str(r_utII_utII.shape))
            
            if ('data/r_unII_unII' in self): del self['data/r_unII_unII']
            dset = self.create_dataset('data/r_unII_unII', data=r_unII_unII.T, chunks=None)
            if verbose: even_print('r_unII_unII','%s'%str(r_unII_unII.shape))
            
            if ('data/r_utII_unII' in self): del self['data/r_utII_unII']
            dset = self.create_dataset('data/r_utII_unII', data=r_utII_unII.T, chunks=None)
            if verbose: even_print('r_utII_unII','%s'%str(r_utII_unII.shape))
            
            ## check
            if self.rectilinear:
                np.testing.assert_allclose(r_utII_utII, r_uII_uII, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(r_unII_unII, r_vII_vII, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(r_utII_unII, r_uII_vII, rtol=1e-14, atol=1e-14)
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_vel_tangnorm_mean_removed() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing: mass & momentum balance
    
    @staticmethod
    def mass_balance_kernel(x,y, u,v,rho, transformation_matrix=None, acc=8, edge_stencil='half'):
        '''
        get Cartesian mass balance terms in 2D plane
        '''
        
        ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
        uv2d = np.copy(np.stack((u,v), axis=-1))
        
        ## rotate & translate [x,y]
        if (transformation_matrix is not None):
            
            ## rotate & translate [x,y]
            xy2d_tmp = np.copy( np.stack( ( x, y, np.ones_like(x) ), axis=-1) )
            xy2d_tmp = np.copy( np.einsum( 'ij,xyj->xyi', transformation_matrix , xy2d_tmp ) )
            xy2d     = np.copy( xy2d_tmp[:,:,:2] )
            xy2d_tmp = None; del xy2d_tmp
            
            ## overwrite
            x = np.squeeze(np.copy(xy2d[:,:,0]))
            y = np.squeeze(np.copy(xy2d[:,:,1]))
            
            ## make a copy of the [2x2] matrix
            rotation_matrix_2x2 = np.copy(transformation_matrix[:2,:2])
            
            ## rotate [u,v]
            uv2d = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d ) )
            
            ## overwrite
            u = np.squeeze(np.copy(uv2d[:,:,0]))
            v = np.squeeze(np.copy(uv2d[:,:,1]))
        
        else:
            
            ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
            xy2d = np.copy(np.stack((x,y), axis=-1))
        
        ## get metric tensor
        M = get_metric_tensor_2d(x, y, acc=acc, edge_stencil=edge_stencil, verbose=False)
        ddx1_q1 = np.copy( M[:,:,0,0] ) ## ∂q1/∂x1 --> ∂q1/∂x or ∂q1/∂r
        ddx1_q2 = np.copy( M[:,:,1,0] ) ## ∂q2/∂x1 --> ∂q2/∂x or ∂q2/∂r
        ddx2_q1 = np.copy( M[:,:,0,1] ) ## ∂q1/∂x2 --> ∂q1/∂y or ∂q1/∂θ
        ddx2_q2 = np.copy( M[:,:,1,1] ) ## ∂q2/∂x2 --> ∂q2/∂y or ∂q2/∂θ
        M = None ; del M
        
        # ===
        
        A = np.copy( rho * u )
        ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
        ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
        ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
        
        ddx_r_u = np.copy(ddx_A)
        ddy_r_u = np.copy(ddy_A)
        
        ddx_A = None; del ddx_A
        ddy_A = None; del ddy_A
        
        
        A = np.copy( rho * v )
        ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
        ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
        ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
        
        ddx_r_v = np.copy(ddx_A)
        ddy_r_v = np.copy(ddy_A)
        
        ddx_A = None; del ddx_A
        ddy_A = None; del ddy_A
        
        # ===
        
        data = {'t1':ddx_r_u,
                't2':ddy_r_v,
               }
        
        return data
    
    def calc_mass_balance_terms(self, **kwargs):
        '''
        calculate terms of time-averaged mass conservation equation
        '''
        
        verbose      = kwargs.get('verbose',True)
        local_csys   = kwargs.get('local_csys',False)
        gn           = kwargs.get('gn','data_mass') ## group name
        acc          = kwargs.get('acc',8)
        edge_stencil = kwargs.get('edge_stencil','half')
        xis          = kwargs.get('xis',None)
        
        csys_type = 'cartesian' ## dummy for now... mass balance in cylindrical not yet implemented
        
        if verbose: print('\n'+'ztmd.calc_mass_balance_terms()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print('local_csys',str(local_csys))
        if verbose: even_print('csys_type',csys_type)
        if verbose: even_print('group name',f'{gn}/')
        
        if local_csys:
            if self.rectilinear:
                print('>>> WARNING: this grid is rectilinear. Using local csys is not needed.')
            if not ('csys/vtang' in self):
                raise ValueError('csys/vtang not in hdf5')
            if not ('csys/vnorm' in self):
                raise ValueError('csys/vnorm not in hdf5')
            
            ## read unit vectors (wall tangent, wall norm) from HDF5
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            if verbose: even_print('vtang',str(vtang.shape))
            if verbose: even_print('vnorm',str(vnorm.shape))
        
        ## if group already exists, delete it entirely
        if (gn in self):
            del self[gn]
            if verbose: even_print('deleted group',f'/{gn}/')
        
        ## create group
        grp = self.create_group(gn)
        if verbose: even_print('created group',f'{grp.name}/')
        
        ## set group attributes
        grp.attrs['local_csys'] = local_csys
        if verbose: even_print(f'{gn}.local_csys',str(grp.attrs['local_csys']))
        
        if verbose: print(72*'-')
        
        terms = {'t1':r'$\partial_x [\bar{\rho} \tilde{u}]$',
                 't2':r'$\partial_y [\bar{\rho} \tilde{v}]$',
                 }
        
        ## initialize dsets
        for dsn, tag in terms.items():
            shape = (self.ny,self.nx)
            if (f'{gn}/{dsn}' in self): del self[f'{gn}/{dsn}']
            dset = self.create_dataset(f'{gn}/{dsn}', 
                                       shape=shape, 
                                       dtype=np.float32,
                                       chunks=None)
            if verbose: even_print(f'{gn}/{dsn}',str(shape))
            dset.attrs['latex'] = tag
            if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
        
        if verbose: print(72*'-')
        
        ## copy data
        u   = np.copy( self['data/u'][()].T   )
        v   = np.copy( self['data/v'][()].T   )
        #w   = np.copy( self['data/w'][()].T   )
        rho = np.copy( self['data/rho'][()].T )
        #p   = np.copy( self['data/p'][()].T   )
        #T   = np.copy( self['data/T'][()].T   )
        #mu  = np.copy( self['data/mu'][()].T  )
        if verbose: even_print('u',str(u.shape))
        if verbose: even_print('v',str(v.shape))
        if verbose: even_print('ρ',str(rho.shape))
        
        u_Fv = np.copy( self['data/u_Fv'][()].T )
        v_Fv = np.copy( self['data/v_Fv'][()].T )
        if verbose: even_print('u_Fv',str(u_Fv.shape))
        if verbose: even_print('v_Fv',str(v_Fv.shape))
        
        x = np.copy(self.x)
        y = np.copy(self.y)
        
        if (x.ndim==1) and (y.ndim==1):
            x,y = np.meshgrid(x,y, indexing='ij')
        if verbose: even_print('x',str(x.shape))
        if verbose: even_print('y',str(y.shape))
        
        if local_csys:
            
            # ## calculate grid points for specific postions in char. lengths
            # s_vert_locs = [-100,-50,-25,0,+25,+50,+100]
            # svert  = (self.stang-250*self.lchar)/self.lchar
            # xis    = [ np.abs(s-svert).argmin() for s in s_vert_locs ]
            
            if (xis is None):
                xis = np.arange(self.nx)
                n_prog = self.nx
            else:
                n_prog = len(xis)
            
            if verbose: progress_bar = tqdm(total=n_prog, ncols=100, desc='mass balance', leave=False, file=sys.stdout)
            
            ## iterate through axis=0 indices
            for xi in xis:
                
                rotation_angle_z = np.arcsin(vnorm[xi,0,0])
                #rotation_angle_z = np.arcsin(-vtang[xi,0,1])
                #if verbose: tqdm.write( even_print('rotation_angle_z', f'{np.degrees(rotation_angle_z):0.14f} [deg]', s=True) )
                
                rotation_matrix_1 = np.array( [[ np.cos(rotation_angle_z) , -np.sin(rotation_angle_z) , 0 ],
                                               [ np.sin(rotation_angle_z) ,  np.cos(rotation_angle_z) , 0 ],
                                               [ 0                        ,  0                        , 1 ]], dtype=np.float64 )
                
                rotation_matrix_2 = np.array( [[ vtang[xi,0,0] ,  vtang[xi,0,1] , 0 ],
                                               [ vnorm[xi,0,0] ,  vnorm[xi,0,1] , 0 ],
                                               [ 0             ,  0             , 1 ]], dtype=np.float64 )
                
                ## just showing that these are the same
                np.testing.assert_allclose( rotation_matrix_1 , rotation_matrix_2 , rtol=1e-14 , atol=1e-14 )
                rotation_matrix = np.copy(rotation_matrix_1)
                
                ## translate to [cx,cy]
                cx = x[xi,0]
                cy = y[xi,0]
                translation_matrix_A = np.array( [[ 1 , 0 , -cx ],
                                                  [ 0 , 1 , -cy ],
                                                  [ 0 , 0 , 1   ]], dtype=np.float64 )
                
                ## translate in [y] by -R (the local curvature radius)
                if (csys_type=='cylindrical'):
                    cx = 0.
                    cy = crv_R[xi]
                    translation_matrix_B = np.array( [[ 1 , 0 , 0.  ],
                                                      [ 0 , 1 , -cy ],
                                                      [ 0 , 0 , 1   ]], dtype=np.float64 )
                else:
                    translation_matrix_B = np.identity(3, dtype=np.float64)
                
                ## form the full [3x3] matrix by multiplying together --> order matters!
                transformation_matrix = np.einsum( 'ij,jk,kl->il', translation_matrix_B , rotation_matrix, translation_matrix_A )
                
                # ===
                
                ## send to kernel
                data = ztmd.mass_balance_kernel(x=x,y=y,
                                                u=u_Fv, v=v_Fv, rho=rho,
                                                transformation_matrix=transformation_matrix,
                                                acc=acc,edge_stencil=edge_stencil,
                                                )
                
                ## write to HDF5 (only data from single axis=0 index)
                for dsn, arr in data.items():
                    dset = self[f'{gn}/{dsn}']
                    dset[:,xi] = arr[xi,:] ## arrays in HDF5 are transposed
                
                if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        else:
            
            ## send to kernel
            data = ztmd.mass_balance_kernel(x=x,y=y,
                                            u=u_Fv, v=v_Fv, rho=rho,
                                            transformation_matrix=None,
                                            acc=acc,edge_stencil=edge_stencil,
                                            )
            
            ## write to HDF5 (only data from single axis=0 index)
            for dsn, arr in data.items():
                dset = self[f'{gn}/{dsn}']
                dset[:,:] = arr.T ## arrays in HDF5 are transposed
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_mass_balance_terms() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    @staticmethod
    def momentum_balance_kernel_cartesian(x,y, u,v, u_Fv,v_Fv, p,rho,mu, r_uII_uII,r_uII_vII,r_vII_vII, transformation_matrix=None, acc=8, edge_stencil='half'):
        '''
        get Cartesian [x] momentum balance terms in 2D plane
        '''
        
        nx,ny = x.shape
        
        ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
        uv2d    = np.copy(np.stack((u,v),       axis=-1))
        uv2d_Fv = np.copy(np.stack((u_Fv,v_Fv), axis=-1))
        
        ## rotate & translate [x,y]
        if (transformation_matrix is not None):
            
            ## rotate & translate [x,y]
            xy2d_tmp = np.copy( np.stack( ( x, y, np.ones_like(x) ), axis=-1) )
            xy2d_tmp = np.copy( np.einsum( 'ij,xyj->xyi', transformation_matrix , xy2d_tmp ) )
            xy2d     = np.copy( xy2d_tmp[:,:,:2] )
            xy2d_tmp = None; del xy2d_tmp
            
            ## overwrite
            x = np.squeeze(np.copy(xy2d[:,:,0]))
            y = np.squeeze(np.copy(xy2d[:,:,1]))
            
            ## make a copy of the [2x2] matrix
            rotation_matrix_2x2 = np.copy(transformation_matrix[:2,:2])
            
            ## rotate [u,v]
            uv2d = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d ) )
            
            ## overwrite
            u = np.squeeze(np.copy(uv2d[:,:,0]))
            v = np.squeeze(np.copy(uv2d[:,:,1]))
            
            ## rotate [u_Fv,v_Fv]
            uv2d_Fv = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d_Fv ) )
            
            ## overwrite
            u_Fv = np.squeeze(np.copy(uv2d_Fv[:,:,0]))
            v_Fv = np.squeeze(np.copy(uv2d_Fv[:,:,1]))
            
            if True: ## Re-Stress tensor: global Cartesian rotate
                
                r_uiIIujII = np.zeros((nx,ny,2,2), dtype=np.float64)
                r_uiIIujII[:,:,0,0] = np.copy( r_uII_uII )
                r_uiIIujII[:,:,0,1] = np.copy( r_uII_vII )
                r_uiIIujII[:,:,1,0] = np.copy( r_uII_vII )
                r_uiIIujII[:,:,1,1] = np.copy( r_vII_vII )
                
                ## assert symmetric
                np.testing.assert_allclose(r_uiIIujII, np.transpose(r_uiIIujII, (0,1,3,2)), atol=1e-14, rtol=1e-14)
                
                ## do tensor rotation : [A'] = [R][A][R]^T
                r_uiIIujII = np.copy( np.einsum( 'ij,xyjk,kl->xyil' , rotation_matrix_2x2 , r_uiIIujII , rotation_matrix_2x2.T ) )
                
                r_uII_uII = np.copy(r_uiIIujII[:,:,0,0])
                r_uII_vII = np.copy(r_uiIIujII[:,:,0,1])
                r_vII_uII = np.copy(r_uiIIujII[:,:,1,0])
                r_vII_vII = np.copy(r_uiIIujII[:,:,1,1])
                
                r_uiIIujII = None ; del r_uiIIujII
                
                np.testing.assert_allclose(r_uII_vII, r_vII_uII, atol=1e-7, rtol=1e-7)
        
        else:
            
            ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
            xy2d = np.copy(np.stack((x,y), axis=-1))
        
        # ===
        
        ## get metric tensor M = δ[q1,q2]/δ[x1,x2] : δ[q1,q2]/δ[x,y] or δ[q1,q2]/δ[r,θ]
        M = get_metric_tensor_2d(x, y, acc=acc, edge_stencil=edge_stencil, verbose=False)
        ddx1_q1 = np.copy( M[:,:,0,0] ) ## ∂q1/∂x1 --> ∂q1/∂x or ∂q1/∂r
        ddx1_q2 = np.copy( M[:,:,1,0] ) ## ∂q2/∂x1 --> ∂q2/∂x or ∂q2/∂r
        ddx2_q1 = np.copy( M[:,:,0,1] ) ## ∂q1/∂x2 --> ∂q1/∂y or ∂q1/∂θ
        ddx2_q2 = np.copy( M[:,:,1,1] ) ## ∂q2/∂x2 --> ∂q2/∂y or ∂q2/∂θ
        M = None ; del M
        
        # ===
        
        if True: ## calculate gradients : Cartesian [x,y]
            
            ## [x1,x2] = [x,y]
            ## ∂(A)/∂x1 = ∂(A)/∂(q1)·∂(q1)/∂(x1) + ∂(A)/∂(q2)·∂(q2)/∂(x1)
            ## ∂(A)/∂x2 = ∂(A)/∂(q1)·∂(q1)/∂(x2) + ∂(A)/∂(q2)·∂(q2)/∂(x2)
            
            A = np.copy( u )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_u = np.copy(ddx_A)
            ddy_u = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( v )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_v = np.copy(ddx_A)
            ddy_v = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( rho )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_rho = np.copy(ddx_A)
            ddy_rho = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( p )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_p = np.copy(ddx_A)
            ddy_p = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( u_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_u_Fv = np.copy(ddx_A)
            ddy_u_Fv = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( v_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_v_Fv = np.copy(ddx_A)
            ddy_v_Fv = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
        
        if True: ## construct momentum balance : Cartesian [x,y]
            
            tau_xx = np.copy( +(4/3)*mu*ddx_u - (2/3)*mu*ddy_v )
            tau_yy = np.copy( -(2/3)*mu*ddx_u + (4/3)*mu*ddy_v )
            
            tau_xy = np.copy( mu*(ddx_v + ddy_u) )
            #tau_yx = np.copy( tau_xy )
            
            # === δx[ρ·u·u]
            
            A = np.copy( rho * u_Fv * u_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_r_uFv_uFv = np.copy(ddx_A)
            t1x = np.copy( ddx_r_uFv_uFv )
            t1x_lbl = r'$\partial_{x}[\bar{\rho} \tilde{u} \tilde{u}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === δx[ρ·u·v] & δy[ρ·u·v]
            
            A = np.copy( rho * u_Fv * v_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_r_uFv_vFv = np.copy(ddx_A)
            ddy_r_uFv_vFv = np.copy(ddy_A)
            
            t1y = np.copy( ddx_r_uFv_vFv )
            t1y_lbl = r'$\partial_{x}[\bar{\rho} \tilde{u} \tilde{v}]$'
            
            t2x = np.copy( ddy_r_uFv_vFv )
            t2x_lbl = r'$\partial_{y}[\bar{\rho} \tilde{u} \tilde{v}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === δy[ρ·v·v]
            
            A = np.copy( rho * v_Fv * v_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddy_r_vFv_vFv = np.copy(ddy_A)
            t2y = np.copy( ddy_r_vFv_vFv )
            t2y_lbl = r'$\partial_{y}[\bar{\rho} \tilde{v} \tilde{v}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === δx[ρ·u''·u'']
            
            A = np.copy(r_uII_uII)
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_r_uII_uII = np.copy(ddx_A)
            t5x = np.copy( ddx_r_uII_uII )
            t5x_lbl = r'$\partial_{x}[ \overline{ \rho u^{\prime\prime} u^{\prime\prime}} ]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === δx[ρ·u''·v''] & δy[ρ·u''·v'']
            
            A = np.copy(r_uII_vII)
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_r_uII_vII = np.copy(ddx_A)
            ddy_r_uII_vII = np.copy(ddy_A)
            
            t5y = np.copy( ddx_r_uII_vII )
            t5y_lbl = r'$\partial_{x}[ \overline{ \rho u^{\prime\prime} v^{\prime\prime}} ]$'
            t7x = np.copy( ddy_r_uII_vII )
            t7x_lbl = r'$\partial_{y}[ \overline{ \rho u^{\prime\prime} v^{\prime\prime}} ]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === δy[ρ·v''·v'']
            
            A = np.copy(r_vII_vII)
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddy_r_vII_vII = np.copy(ddy_A)
            t7y = np.copy( ddy_r_vII_vII )
            t7y_lbl = r'$\partial_{y}[ \overline{\rho v^{\prime\prime} v^{\prime\prime}} ]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === δx[p] & δy[p]
            
            t3x = np.copy( ddx_p )
            t3x_lbl = r'$\partial_{x}[p]$'
            
            t3y = np.copy( ddy_p )
            t3y_lbl = r'$\partial_{y}[p]$'
            
            # === δx[τxx]
            
            ddq1_A = gradient(tau_xx, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(tau_xx, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_tau_xx = np.copy(ddx_A)
            t4x = np.copy( -1 * ddx_tau_xx )
            t4x_lbl = r'$-\partial_{x}[\overline{\tau_{xx}}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === δx[τxy] & δy[τxy]
            
            ddq1_A = gradient(tau_xy, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(tau_xy, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_tau_xy = np.copy(ddx_A)
            ddy_tau_xy = np.copy(ddy_A)
            
            t4y = np.copy( -1 * ddx_tau_xy )
            t4y_lbl = r'$-\partial_{x}[\overline{\tau_{xy}}]$'
            
            t6x = np.copy( -1 * ddy_tau_xy )
            t6x_lbl = r'$-\partial_{y}[\overline{\tau_{xy}}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === δy[τyy]
            
            ddq1_A = gradient(tau_yy, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(tau_yy, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddy_tau_yy = np.copy(ddy_A)
            t6y = np.copy( -1 * ddy_tau_yy )
            t6y_lbl = r'$-\partial_{y}[\overline{\tau_{yy}}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
        
        # ===
        
        data_x = {'t1':np.copy(t1x),
                  't2':np.copy(t2x),
                  't3':np.copy(t3x),
                  't4':np.copy(t4x),
                  't5':np.copy(t5x),
                  't6':np.copy(t6x),
                  't7':np.copy(t7x),
                  }
        
        data_y = {'t1':np.copy(t1y),
                  't2':np.copy(t2y),
                  't3':np.copy(t3y),
                  't4':np.copy(t4y),
                  't5':np.copy(t5y),
                  't6':np.copy(t6y),
                  't7':np.copy(t7y),
                  }
        
        return data_x, data_y
    
    @staticmethod
    def momentum_balance_kernel_cylindrical(x,y, u,v, u_Fv,v_Fv, p,rho,mu, r_uII_uII,r_uII_vII,r_vII_vII, transformation_matrix=None, acc=8, edge_stencil='half'):
        '''
        get cylindrical [r,θ] momentum balance terms in 2D plane
        '''
        
        nx,ny = x.shape
        
        ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
        uv2d    = np.copy(np.stack((u,v),       axis=-1))
        uv2d_Fv = np.copy(np.stack((u_Fv,v_Fv), axis=-1))
        
        ## rotate & translate [x,y]
        if (transformation_matrix is not None):
            
            ## rotate & translate [x,y]
            xy2d_tmp = np.copy( np.stack( ( x, y, np.ones_like(x) ), axis=-1) )
            xy2d_tmp = np.copy( np.einsum( 'ij,xyj->xyi', transformation_matrix , xy2d_tmp ) )
            xy2d     = np.copy( xy2d_tmp[:,:,:2] )
            xy2d_tmp = None; del xy2d_tmp
            
            ## overwrite
            x = np.squeeze(np.copy(xy2d[:,:,0]))
            y = np.squeeze(np.copy(xy2d[:,:,1]))
            
            ## make a copy of the [2x2] matrix
            rotation_matrix_2x2 = np.copy(transformation_matrix[:2,:2])
            
            ## rotate [u,v]
            uv2d = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d ) )
            
            ## overwrite
            u = np.squeeze(np.copy(uv2d[:,:,0]))
            v = np.squeeze(np.copy(uv2d[:,:,1]))
            
            ## rotate [u_Fv,v_Fv]
            uv2d_Fv = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d_Fv ) )
            
            ## overwrite
            u_Fv = np.squeeze(np.copy(uv2d_Fv[:,:,0]))
            v_Fv = np.squeeze(np.copy(uv2d_Fv[:,:,1]))
            
            if True: ## Re-Stress tensor: global Cartesian rotate
                
                r_uiIIujII = np.zeros((nx,ny,2,2), dtype=np.float64)
                r_uiIIujII[:,:,0,0] = np.copy( r_uII_uII )
                r_uiIIujII[:,:,0,1] = np.copy( r_uII_vII )
                r_uiIIujII[:,:,1,0] = np.copy( r_uII_vII )
                r_uiIIujII[:,:,1,1] = np.copy( r_vII_vII )
                
                ## assert symmetric
                np.testing.assert_allclose(r_uiIIujII, np.transpose(r_uiIIujII, (0,1,3,2)), atol=1e-14, rtol=1e-14)
                
                ## do tensor rotation : [A'] = [R][A][R]^T
                r_uiIIujII = np.copy( np.einsum( 'ij,xyjk,kl->xyil' , rotation_matrix_2x2 , r_uiIIujII , rotation_matrix_2x2.T ) )
                
                r_uII_uII = np.copy(r_uiIIujII[:,:,0,0])
                r_uII_vII = np.copy(r_uiIIujII[:,:,0,1])
                r_vII_uII = np.copy(r_uiIIujII[:,:,1,0])
                r_vII_vII = np.copy(r_uiIIujII[:,:,1,1])
                
                r_uiIIujII = None ; del r_uiIIujII
                
                np.testing.assert_allclose(r_uII_vII, r_vII_uII, atol=1e-7, rtol=1e-7)
        
        else:
            
            ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
            xy2d = np.copy(np.stack((x,y), axis=-1))
        
        # ===
        
        ## calculate coordinate arrays [r,θ] from [x,y]
        r     = np.copy( np.sqrt(x**2 + y**2) )
        theta = np.copy( np.arctan2(y,x) )
        
        ## calculate [ur,uθ] from [u,v]
        vel_cart2cyl_trans_mat = np.zeros((nx,ny,2,2), dtype=np.float64)
        for i in range(nx):
            for j in range(ny):
                vel_cart2cyl_trans_mat[i,j,:,:] = np.array( [[  np.cos(theta[i,j]) , np.sin(theta[i,j]) ],
                                                             [ -np.sin(theta[i,j]) , np.cos(theta[i,j]) ]], dtype=np.float64 )
        
        ## ur, uθ
        urut2d = np.einsum( 'xyji,xyi->xyj', vel_cart2cyl_trans_mat, uv2d )
        ur = np.squeeze(np.copy(urut2d[:,:,0]))
        ut = np.squeeze(np.copy(urut2d[:,:,1]))
        
        ## ur_Fv, uθ_Fv
        urut2d_Fv = np.einsum( 'xyji,xyi->xyj', vel_cart2cyl_trans_mat, uv2d_Fv )
        ur_Fv = np.squeeze(np.copy(urut2d_Fv[:,:,0]))
        ut_Fv = np.squeeze(np.copy(urut2d_Fv[:,:,1]))
        
        ## Re-Stress tensor : convert from Cartesian (rotated) to cylindrical components
        r_uiIIujII = np.zeros((nx,ny,2,2), dtype=np.float64)
        r_uiIIujII[:,:,0,0] = np.copy( r_uII_uII )
        r_uiIIujII[:,:,0,1] = np.copy( r_uII_vII )
        r_uiIIujII[:,:,1,0] = np.copy( r_uII_vII )
        r_uiIIujII[:,:,1,1] = np.copy( r_vII_vII )
        
        r_uII_uII = None; del r_uII_uII
        r_uII_vII = None; del r_uII_vII
        r_vII_uII = None; del r_vII_uII
        r_vII_vII = None; del r_vII_vII
        
        ## assert symmetric
        np.testing.assert_allclose(r_uiIIujII, np.transpose(r_uiIIujII, (0,1,3,2)), atol=1e-14, rtol=1e-14)
        
        ## csys conversion : Cartesian --> cylindrical
        r_uiIIujII = np.copy( np.einsum( 'xyij,xyjk,xykl->xyil' , vel_cart2cyl_trans_mat , r_uiIIujII , np.transpose(vel_cart2cyl_trans_mat,(0,1,3,2)) ) )
        
        r_urII_urII = np.copy(r_uiIIujII[:,:,0,0])
        r_urII_utII = np.copy(r_uiIIujII[:,:,0,1])
        r_utII_urII = np.copy(r_uiIIujII[:,:,1,0])
        r_utII_utII = np.copy(r_uiIIujII[:,:,1,1])
        
        r_uiIIujII = None ; del r_uiIIujII
        
        ## assert symmetric again
        np.testing.assert_allclose(r_urII_utII, r_utII_urII, atol=1e-7, rtol=1e-7)
        
        # ===
        
        ## get metric tensor M = δ[q1,q2]/δ[x1,x2] : δ[q1,q2]/δ[x,y] or δ[q1,q2]/δ[r,θ]
        M = get_metric_tensor_2d(r, theta, acc=acc, edge_stencil=edge_stencil, verbose=False)
        ddx1_q1 = np.copy( M[:,:,0,0] ) ## ∂q1/∂x1 --> ∂q1/∂x or ∂q1/∂r
        ddx1_q2 = np.copy( M[:,:,1,0] ) ## ∂q2/∂x1 --> ∂q2/∂x or ∂q2/∂r
        ddx2_q1 = np.copy( M[:,:,0,1] ) ## ∂q1/∂x2 --> ∂q1/∂y or ∂q1/∂θ
        ddx2_q2 = np.copy( M[:,:,1,1] ) ## ∂q2/∂x2 --> ∂q2/∂y or ∂q2/∂θ
        M = None ; del M
        
        # ===
        
        if True: ## calculate gradients : cylindrical [r,θ]
            
            ## [x1,x2] = [r,θ]
            ## ∂(A)/∂x1 = ∂(A)/∂(q1)·∂(q1)/∂(x1) + ∂(A)/∂(q2)·∂(q2)/∂(x1)
            ## ∂(A)/∂x2 = ∂(A)/∂(q1)·∂(q1)/∂(x2) + ∂(A)/∂(q2)·∂(q2)/∂(x2)
            
            A = np.copy( ur )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_ur = np.copy(ddx1_A)
            ddt_ur = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( ut )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_ut = np.copy(ddx1_A)
            ddt_ut = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( rho )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_rho = np.copy(ddx1_A)
            ddt_rho = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( p )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_p = np.copy(ddx1_A)
            ddt_p = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( ur_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_ur_Fv = np.copy(ddx1_A)
            ddt_ur_Fv = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( ut_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_ut_Fv = np.copy(ddx1_A)
            ddt_ut_Fv = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
        
        if True: ## construct momentum balance : cylindrical [r,θ]
            
            ## τ_rr, τ_θθ, τ_rθ
            tau_rr = np.copy( +(4/3)*mu*ddr_ur - (2/3)*(1/r)*mu*ddt_ut - (2/3)*mu*(ur/r) )
            tau_tt = np.copy( -(2/3)*mu*ddr_ur + (4/3)*(1/r)*mu*ddt_ut + (4/3)*mu*(ur/r) )
            tau_rt = np.copy( mu*( (1/r)*ddt_ur + ddr_ut - (ut/r) ) )
            
            # === δr[ρ·ur·ur]
            
            A = np.copy( rho * ur_Fv * ur_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_rho_urFv_urFv = np.copy(ddx1_A)
            t1x1 = np.copy( ddr_rho_urFv_urFv )
            t1x1_lbl = r'$\partial_{r}[ \bar{\rho} \tilde{u}_r \tilde{u}_r ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === (1/r)·δθ[ρ·ur·uθ]
            
            A = np.copy( rho * ur_Fv * ut_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_rho_urFv_utFv = np.copy(ddx2_A)
            t2x1 = np.copy( (1/r)*ddt_rho_urFv_utFv )
            t2x1_lbl = r'$\frac{1}{r} \partial_{\theta}[ \bar{\rho} \tilde{u}_r \tilde{u}_\theta ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === δr[ρ·ur·uθ]
            
            A = np.copy( rho * ur_Fv * ut_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_rho_urFv_utFv = np.copy(ddx1_A)
            t1x2 = np.copy( ddr_rho_urFv_utFv )
            t1x2_lbl = r'$\partial_{r}[ \bar{\rho} \tilde{u}_r \tilde{u}_\theta ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === δθ[ρ·uθ·uθ]
            
            A = np.copy( rho * ut_Fv * ut_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_rho_utFv_utFv = np.copy(ddx2_A)
            t2x2 = np.copy( (1/r)*ddt_rho_utFv_utFv )
            t2x2_lbl = r'$\frac{1}{r} \partial_{\theta}[ \bar{\rho} \tilde{u}_\theta \tilde{u}_\theta ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === δr[p] & δθ[p]
            
            t3x1 = np.copy( ddr_p )
            t3x1_lbl = r'$\partial_{r}[\bar{p}]$'
            
            t3x2 = np.copy( (1/r)*ddt_p )
            t3x2_lbl = r'$\frac{1}{r} \partial_{\theta}[\bar{p}]$'
            
            # === δr[τ_rr]
            
            A = np.copy( tau_rr )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_tau_rr = np.copy( ddx1_A )
            t4x1 = np.copy( -ddr_tau_rr )
            t4x1_lbl = r'$-\partial_{r}[ \overline{\tau}_{r r} ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === -(1/r)·δθ[τ_θθ]
            
            A = np.copy( tau_tt )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_tau_tt = np.copy( ddx2_A )
            t6x2 = np.copy( -(1/r)*ddt_tau_tt )
            t6x2_lbl = r'$-\frac{1}{r} \partial_{\theta}[ \overline{\tau}_{\theta \theta} ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === δr[ρ·ur''·ur'']
            
            A = np.copy( r_urII_urII )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_r_urII_urII = np.copy( ddx1_A )
            t5x1 = np.copy( ddr_r_urII_urII )
            t5x1_lbl = r'$\partial_{r}[ \overline{ \rho u^{\prime \prime}_r  u^{\prime \prime}_r } ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === -(1/r)·δθ[τ_rθ]  &  -δr[τ_rθ]
            
            A = np.copy( tau_rt )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_tau_rt = np.copy( ddx1_A )
            t4x2 = np.copy( -ddr_tau_rt )
            t4x2_lbl = r'$-\partial_{r}[ \overline{\tau}_{r \theta} ]$'
            
            ddt_tau_rt = np.copy( ddx2_A )
            t6x1 = np.copy( -(1/r)*ddt_tau_rt )
            t6x1_lbl = r'$-\frac{1}{r} \partial_{\theta}[ \overline{\tau}_{r \theta} ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === (1/r)·δθ[ρ·ur''·uθ''] & δr[ρ·ur''·uθ'']
            
            A = np.copy( r_urII_utII )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_r_urII_utII = np.copy( ddx2_A )
            t7x1 = np.copy( (1/r)*ddt_r_urII_utII )
            t7x1_lbl = r'$\frac{1}{r} \partial_{\theta}[ \overline{\rho u^{\prime \prime}_r  u^{\prime \prime}_\theta } ]$'
            
            ddr_r_urII_utII = np.copy( ddx1_A )
            t5x2 = np.copy( ddr_r_urII_utII )
            t5x2_lbl = r'$\partial_{r}[ \overline{\rho u^{\prime \prime}_r  u^{\prime \prime}_\theta } ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === (1/r)·δθ[ρ·uθ''·uθ'']
            
            A = np.copy( r_utII_utII )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_r_utII_utII = np.copy( ddx2_A )
            t7x2 = np.copy( (1/r)*ddt_r_utII_utII )
            t7x2_lbl = r'$\frac{1}{r} \partial_{\theta}[ \overline{\rho u^{\prime \prime}_\theta  u^{\prime \prime}_\theta } ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # ===
            
            ## +(1/r)·ρ·ur·ur
            t8x1 = np.copy( (1/r) * rho * ur_Fv * ur_Fv )
            t8x1_lbl = r'$\frac{1}{r} \bar{\rho} \tilde{u}_{r} \tilde{u}_{r}$'
            
            ## +(1/r)·ρ·ur''·ur''
            t9x1 = np.copy( (1/r) * r_urII_urII )
            t9x1_lbl = r'$\frac{1}{r} \overline{ \rho u^{\prime \prime}_r u^{\prime \prime}_r }$'
            
            ## -(1/r)·ρ·uθ·uθ
            t10x1 = np.copy( -1 * (1/r) * rho * ut_Fv * ut_Fv )
            t10x1_lbl = r'$-\frac{1}{r} \bar{\rho} \tilde{u}_{\theta} \tilde{u}_{\theta}$'
            
            ## -(1/r)·ρ·uθ''·uθ''
            t11x1 = np.copy( -1 * (1/r) * r_utII_utII )
            t11x1_lbl = r'$-\frac{1}{r} \overline{ \rho u^{\prime \prime}_\theta u^{\prime \prime}_\theta }$'
            
            ## -(1/r)·τ_rr
            t12x1 = np.copy( -1 * (1/r) * tau_rr )
            t12x1_lbl = r'$-\frac{1}{r} \overline{\tau}_{r r}$'
            
            ## +(1/r)·τ_θθ
            t13x1 = np.copy( (1/r) * tau_tt )
            t13x1_lbl = r'$\frac{1}{r} \overline{\tau}_{\theta \theta}$'
            
            # ===
            
            ## +(2/r)·ρ·ur·uθ
            t8x2 = np.copy( (2/r) * rho * ur_Fv * ut_Fv )
            t8x2_lbl = r'$\frac{2}{r} \bar{\rho} \tilde{u}_{r} \tilde{u}_{\theta}$'
            
            ## +(2/r)·ρ·ur·uθ
            t9x2 = np.copy( (2/r) * r_urII_utII  )
            t9x2_lbl = r'$\frac{2}{r} \overline{ \rho u^{\prime \prime}_r u^{\prime \prime}_\theta }$'
            
            ## -(2/r)·τ_rθ
            t10x2 = np.copy( -(2/r) * tau_rt )
            t10x2_lbl = r'$-\frac{2}{r} \overline{\tau}_{r \theta}$'
        
        # ===
        
        data_r = {'t1':np.copy(t1x1),
                  't2':np.copy(t2x1),
                  't3':np.copy(t3x1),
                  't4':np.copy(t4x1),
                  't5':np.copy(t5x1),
                  't6':np.copy(t6x1),
                  't7':np.copy(t7x1),
                  't8':np.copy(t8x1),
                  't9':np.copy(t9x1),
                  't10':np.copy(t10x1),
                  't11':np.copy(t11x1),
                  't12':np.copy(t12x1),
                  't13':np.copy(t13x1),
                  }
        
        data_t = {'t1':np.copy(t1x2),
                  't2':np.copy(t2x2),
                  't3':np.copy(t3x2),
                  't4':np.copy(t4x2),
                  't5':np.copy(t5x2),
                  't6':np.copy(t6x2),
                  't7':np.copy(t7x2),
                  't8':np.copy(t8x2),
                  't9':np.copy(t9x2),
                  't10':np.copy(t10x2),
                  }
        
        return data_r, data_t
    
    def calc_momentum_balance_terms(self, **kwargs):
        '''
        calculate terms of the 2D time-averaged momentum conservation equation in Cartesian coords
        '''
        
        verbose      = kwargs.get('verbose',True)
        local_csys   = kwargs.get('local_csys',False)
        acc          = kwargs.get('acc',8)
        edge_stencil = kwargs.get('edge_stencil','half')
        xis          = kwargs.get('xis',None)
        csys_type    = kwargs.get('csys_type','cartesian')
        
        if verbose: print('\n'+'ztmd.calc_momentum_balance_terms()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        
        if verbose: even_print('local_csys',str(local_csys))
        if verbose: even_print('csys_type',csys_type)
        
        if local_csys:
            if self.rectilinear:
                print('>>> WARNING: this grid is rectilinear. Using local csys is not needed.')
            if not ('csys/vtang' in self):
                raise ValueError('csys/vtang not in hdf5')
            if not ('csys/vnorm' in self):
                raise ValueError('csys/vnorm not in hdf5')
            
            ## read unit vectors (wall tangent, wall norm) from HDF5
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            if verbose: even_print('vtang',str(vtang.shape))
            if verbose: even_print('vnorm',str(vnorm.shape))
        
        if (csys_type=='cartesian'):
            gnx = 'data_momentum_x'
            gny = 'data_momentum_y'
            gns = [gnx,gny]
        elif (csys_type=='cylindrical'):
            gnr = 'data_momentum_r'
            gnt = 'data_momentum_theta'
            gns = [gnr,gnt]
        else:
            raise NotImplementedError
        
        terms_x = {'t1':r'$\partial_{x}[\bar{\rho} \tilde{u} \tilde{u}]$',
                   't2':r'$\partial_{y}[\bar{\rho} \tilde{u} \tilde{v}]$',
                   't3':r'$\partial_{x}[\bar{p}]$',
                   't4':r'$-\partial_{x}[\overline{\tau}_xx}]$',
                   't5':r'$\partial_{x}[\overline{\rho u^{\prime\prime} u^{\prime\prime}}]$',
                   't6':r'$-\partial_{y}[\overline{\tau_{xy}}]$',
                   't7':r'$\partial_{y}[\overline{\rho u^{\prime\prime} v^{\prime\prime}}]$',
                   }
        
        terms_y = {'t1':r'$\partial_{x}[\bar{\rho} \tilde{u} \tilde{v}]$',
                   't2':r'$\partial_{y}[\bar{\rho} \tilde{v} \tilde{v}]$',
                   't3':r'$\partial_{y}[\bar{p}]$',
                   't4':r'$-\partial_{x}[\overline{\tau_{xy}}]$',
                   't5':r'$\partial_{x}[\overline{\rho u^{\prime\prime} v^{\prime\prime}}]$',
                   't6':r'$-\partial_{y}[\overline{\tau_{yy}}]$',
                   't7':r'$\partial_{y}[\overline{\rho v^{\prime\prime} v^{\prime\prime}}]$',
                   }
        
        terms_r = {'t1':r'$\partial_{r}[ \bar{\rho} \tilde{u}_r \tilde{u}_r ]$',
                   't2':r'$\frac{1}{r} \partial_{\theta}[ \bar{\rho} \tilde{u}_r \tilde{u}_\theta ]$',
                   't3':r'$\partial_{r}[\bar{p}]$',
                   't4':r'$-\partial_{r}[ \overline{\tau}_{r r} ]$',
                   't5':r'$\partial_{r}[ \overline{ \rho u^{\prime \prime}_r  u^{\prime \prime}_r } ]$',
                   't6':r'$-\frac{1}{r} \partial_{\theta}[ \overline{\tau}_{r \theta} ]$',
                   't7':r'$\frac{1}{r} \partial_{\theta}[ \overline{\rho u^{\prime \prime}_r  u^{\prime \prime}_\theta } ]$',
                   't8':r'$\frac{1}{r} \bar{\rho} \tilde{u}_{r} \tilde{u}_{r}$',
                   't9':r'$\frac{1}{r} \overline{ \rho u^{\prime \prime}_r u^{\prime \prime}_r }$',
                   't10':r'$-\frac{1}{r} \bar{\rho} \tilde{u}_{\theta} \tilde{u}_{\theta}$',
                   't11':r'$-\frac{1}{r} \overline{ \rho u^{\prime \prime}_\theta u^{\prime \prime}_\theta }$',
                   't12':r'$-\frac{1}{r} \overline{\tau}_{r r}$',
                   't13':r'$\frac{1}{r} \overline{\tau}_{\theta \theta}$',
                   }
        
        terms_theta = {'t1':r'$\partial_{r}[ \bar{\rho} \tilde{u}_r \tilde{u}_\theta ]$',
                       't2':r'$\frac{1}{r} \partial_{\theta}[ \bar{\rho} \tilde{u}_\theta \tilde{u}_\theta ]$',
                       't3':r'$\frac{1}{r} \partial_{\theta}[\bar{p}]$',
                       't4':r'$-\partial_{r}[ \overline{\tau}_{r \theta} ]$',
                       't5':r'$\partial_{r}[ \overline{\rho u^{\prime \prime}_r  u^{\prime \prime}_\theta } ]$',
                       't6':r'$-\frac{1}{r} \partial_{\theta}[ \overline{\tau}_{\theta \theta} ]$',
                       't7':r'$\frac{1}{r} \partial_{\theta}[ \overline{\rho u^{\prime \prime}_\theta  u^{\prime \prime}_\theta } ]$',
                       't8':r'$\frac{2}{r} \bar{\rho} \tilde{u}_{r} \tilde{u}_{\theta}$',
                       't9':r'$\frac{2}{r} \overline{ \rho u^{\prime \prime}_r u^{\prime \prime}_\theta }$',
                       't10':r'$-\frac{2}{r} \overline{\tau}_{r \theta}$',
                       }
        
        ## create groups
        for gn in gns:
            
            if verbose: print(72*'-')
            
            if (gn in self):
                del self[gn]
                if verbose: even_print('deleted group',f'/{gn}/')
            
            grp = self.create_group(gn)
            if verbose: even_print('created group',f'{grp.name}/')
            
            ## set group attributes
            grp.attrs['local_csys'] = local_csys
            if verbose: even_print(f'{gn}.local_csys',str(grp.attrs['local_csys']))
        
        gn=None; del gn
        
        if verbose: print(72*'-')
        
        ## initialize dsets for [x] terms
        if (csys_type=='cartesian'):
            for dsn, tag in terms_x.items():
                shape = (self.ny,self.nx)
                if (f'{gnx}/{dsn}' in self): del self[f'{gnx}/{dsn}']
                dset = self.create_dataset(f'{gnx}/{dsn}', 
                                           shape=shape, 
                                           dtype=np.float32,
                                           chunks=None)
                if verbose: even_print(f'{gnx}/{dsn}',str(shape))
                dset.attrs['latex'] = tag
                if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
            if verbose: print(72*'-')
        
        ## initialize dsets for [y] terms
        if (csys_type=='cartesian'):
            for dsn, tag in terms_y.items():
                shape = (self.ny,self.nx)
                if (f'{gny}/{dsn}' in self): del self[f'{gny}/{dsn}']
                dset = self.create_dataset(f'{gny}/{dsn}', 
                                           shape=shape, 
                                           dtype=np.float32,
                                           chunks=None)
                if verbose: even_print(f'{gny}/{dsn}',str(shape))
                dset.attrs['latex'] = tag
                if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
            if verbose: print(72*'-')
        
        ## initialize dsets for [r] terms
        if (csys_type=='cylindrical'):
            for dsn, tag in terms_r.items():
                shape = (self.ny,self.nx)
                if (f'{gnr}/{dsn}' in self): del self[f'{gnr}/{dsn}']
                dset = self.create_dataset(f'{gnr}/{dsn}', 
                                           shape=shape, 
                                           dtype=np.float32,
                                           chunks=None)
                if verbose: even_print(f'{gnr}/{dsn}',str(shape))
                dset.attrs['latex'] = tag
                if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
            if verbose: print(72*'-')
        
        ## initialize dsets for [θ] terms
        if (csys_type=='cylindrical'):
            for dsn, tag in terms_theta.items():
                shape = (self.ny,self.nx)
                if (f'{gnt}/{dsn}' in self): del self[f'{gnt}/{dsn}']
                dset = self.create_dataset(f'{gnt}/{dsn}', 
                                           shape=shape, 
                                           dtype=np.float32,
                                           chunks=None)
                if verbose: even_print(f'{gnt}/{dsn}',str(shape))
                dset.attrs['latex'] = tag
                if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
            if verbose: print(72*'-')
        
        ## copy data
        
        if (csys_type=='cylindrical') and hasattr(self,'crv_R'):
            crv_R = np.copy(self.crv_R)
            if verbose: even_print('crv_R',str(crv_R.shape))
        
        u   = np.copy( self['data/u'][()].T   )
        v   = np.copy( self['data/v'][()].T   )
        #w   = np.copy( self['data/w'][()].T   )
        rho = np.copy( self['data/rho'][()].T )
        p   = np.copy( self['data/p'][()].T   )
        #T   = np.copy( self['data/T'][()].T   )
        mu  = np.copy( self['data/mu'][()].T  )
        
        if verbose: even_print('u',str(u.shape))
        if verbose: even_print('v',str(v.shape))
        if verbose: even_print('ρ',str(rho.shape))
        if verbose: even_print('p',str(p.shape))
        if verbose: even_print('μ',str(mu.shape))
        
        u_Fv = np.copy( self['data/u_Fv'][()].T )
        v_Fv = np.copy( self['data/v_Fv'][()].T )
        if verbose: even_print('u_Fv',str(u_Fv.shape))
        if verbose: even_print('v_Fv',str(v_Fv.shape))
        
        r_uII_uII = np.copy( self['data/r_uII_uII'][()].T )
        r_vII_vII = np.copy( self['data/r_vII_vII'][()].T )
        #r_wII_wII = np.copy( self['data/r_wII_wII'][()].T )
        r_uII_vII = np.copy( self['data/r_uII_vII'][()].T )
        #r_uII_wII = np.copy( self['data/r_uII_wII'][()].T )
        #r_vII_wII = np.copy( self['data/r_vII_wII'][()].T )
        if verbose: even_print('ρu′′u′′',str(r_uII_uII.shape))
        if verbose: even_print('ρu′′v′′',str(r_uII_vII.shape))
        if verbose: even_print('ρv′′v′′',str(r_vII_vII.shape))
        
        x = np.copy(self.x)
        y = np.copy(self.y)
        
        if (x.ndim==1) and (y.ndim==1):
            x,y = np.meshgrid(x,y, indexing='ij')
        if verbose: even_print('x',str(x.shape))
        if verbose: even_print('y',str(y.shape))
        
        nx,ny = x.shape
        
        if local_csys:
            
            # ## calculate grid points for specific postions in char. lengths
            # s_vert_locs = [-100,-50,-25,0,+25,+50,+100]
            # svert  = (self.stang-250*self.lchar)/self.lchar
            # xis    = [ np.abs(s-svert).argmin() for s in s_vert_locs ]
            
            if (xis is None):
                xis = np.arange(self.nx)
                n_prog = self.nx
            else:
                n_prog = len(xis)
            
            if (csys_type=='cartesian'):
                desc='momentum balance [x,y]'
            elif (csys_type=='cylindrical'):
                desc='momentum balance [r,θ]'
            else:
                raise ValueError
            
            if verbose: progress_bar = tqdm(total=n_prog, ncols=100, desc=desc, leave=False, file=sys.stdout)
            
            ## iterate through axis=0 indices
            for xi in xis:
                
                rotation_angle_z = np.arcsin(vnorm[xi,0,0])
                #rotation_angle_z = np.arcsin(-vtang[xi,0,1])
                #if verbose: tqdm.write( even_print('rotation_angle_z', f'{np.degrees(rotation_angle_z):0.14f} [deg]', s=True) )
                
                rotation_matrix_1 = np.array( [[ np.cos(rotation_angle_z) , -np.sin(rotation_angle_z) , 0 ],
                                               [ np.sin(rotation_angle_z) ,  np.cos(rotation_angle_z) , 0 ],
                                               [ 0                        ,  0                        , 1 ]], dtype=np.float64 )
                
                ## rotation_matrix_2 = np.array( [[ vtang[xi,0,0] ,  vtang[xi,0,1] , 0 ],
                ##                                [ vnorm[xi,0,0] ,  vnorm[xi,0,1] , 0 ],
                ##                                [ 0             ,  0             , 1 ]], dtype=np.float64 )
                ## 
                ## ## just showing that these are the same
                ## np.testing.assert_allclose( rotation_matrix_1 , rotation_matrix_2 , rtol=1e-14 , atol=1e-14 )
                
                rotation_matrix = np.copy(rotation_matrix_1)
                
                ## translate to [cx,cy]
                cx = x[xi,0]
                cy = y[xi,0]
                translation_matrix_A = np.array( [[ 1 , 0 , -cx ],
                                                  [ 0 , 1 , -cy ],
                                                  [ 0 , 0 ,  1  ]], dtype=np.float64 )
                
                ## translate in [y] by -R (the local curvature radius)
                if (csys_type=='cylindrical'):
                    cx = 0.
                    cy = crv_R[xi]
                    translation_matrix_B = np.array( [[ 1 , 0 ,  0  ],
                                                      [ 0 , 1 , -cy ],
                                                      [ 0 , 0 ,  1  ]], dtype=np.float64 )
                else:
                    translation_matrix_B = np.identity(3, dtype=np.float64)
                
                ## form the full [3x3] matrix by multiplying together --> order matters!
                transformation_matrix = np.einsum( 'ij,jk,kl->il', translation_matrix_B , rotation_matrix, translation_matrix_A )
                
                # ===
                
                ## send to kernel
                if (csys_type=='cartesian'):
                    
                    data_x, data_y = ztmd.momentum_balance_kernel_cartesian(x=x,y=y,
                                                                            u=u,v=v,
                                                                            u_Fv=u_Fv,v_Fv=v_Fv,
                                                                            p=p, rho=rho, mu=mu,
                                                                            r_uII_uII=r_uII_uII,
                                                                            r_uII_vII=r_uII_vII,
                                                                            r_vII_vII=r_vII_vII,
                                                                            transformation_matrix=transformation_matrix,
                                                                            acc=acc,edge_stencil=edge_stencil )
                    
                    ## write to HDF5 (only data from single axis=0 index)
                    for dsn, arr in data_x.items():
                        dset = self[f'{gnx}/{dsn}']
                        dset[:,xi] = arr[xi,:] ## arrays in HDF5 are transposed
                    for dsn, arr in data_y.items():
                        dset = self[f'{gny}/{dsn}']
                        dset[:,xi] = arr[xi,:]
                
                elif (csys_type=='cylindrical'):
                    
                    data_r, data_t = ztmd.momentum_balance_kernel_cylindrical(x=x,y=y,
                                                                              u=u,v=v,
                                                                              u_Fv=u_Fv,v_Fv=v_Fv,
                                                                              p=p, rho=rho, mu=mu,
                                                                              r_uII_uII=r_uII_uII,
                                                                              r_uII_vII=r_uII_vII,
                                                                              r_vII_vII=r_vII_vII,
                                                                              transformation_matrix=transformation_matrix,
                                                                              acc=acc,edge_stencil=edge_stencil )
                    
                    ## write to HDF5 (only data from single axis=0 index)
                    for dsn, arr in data_r.items():
                        dset = self[f'{gnr}/{dsn}']
                        dset[:,xi] = arr[xi,:] ## arrays in HDF5 are transposed
                    for dsn, arr in data_t.items():
                        dset = self[f'{gnt}/{dsn}']
                        dset[:,xi] = arr[xi,:]
                
                else:
                    raise ValueError
                
                if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        else:
            
            ## send to kernel
            if (csys_type=='cartesian'):
                
                data_x, data_y = ztmd.momentum_balance_kernel_cartesian(x=x,y=y,
                                                                        u=u,v=v,
                                                                        u_Fv=u_Fv,v_Fv=v_Fv,
                                                                        p=p, rho=rho, mu=mu,
                                                                        r_uII_uII=r_uII_uII,
                                                                        r_uII_vII=r_uII_vII,
                                                                        r_vII_vII=r_vII_vII,
                                                                        #transformation_matrix=transformation_matrix,
                                                                        acc=acc,edge_stencil=edge_stencil )
                
                ## write to HDF5 (only data from single axis=0 index)
                for dsn, arr in data_x.items():
                    dset = self[f'{gnx}/{dsn}']
                    dset[:,:] = arr.T ## arrays in HDF5 are transposed
                for dsn, arr in data_y.items():
                    dset = self[f'{gny}/{dsn}']
                    dset[:,:] = arr.T ## arrays in HDF5 are transposed
            
            elif (csys_type=='cylindrical'):
                
                data_r, data_t = ztmd.momentum_balance_kernel_cylindrical(x=x,y=y,
                                                                          u=u,v=v,
                                                                          u_Fv=u_Fv,v_Fv=v_Fv,
                                                                          p=p, rho=rho, mu=mu,
                                                                          r_uII_uII=r_uII_uII,
                                                                          r_uII_vII=r_uII_vII,
                                                                          r_vII_vII=r_vII_vII,
                                                                          #transformation_matrix=transformation_matrix,
                                                                          acc=acc,edge_stencil=edge_stencil )
                
                ## write to HDF5 (only data from single axis=0 index)
                for dsn, arr in data_r.items():
                    dset = self[f'{gnr}/{dsn}']
                    dset[:,xi] = arr[xi,:] ## arrays in HDF5 are transposed
                for dsn, arr in data_t.items():
                    dset = self[f'{gnt}/{dsn}']
                    dset[:,xi] = arr[xi,:]
            
            else:
                raise ValueError
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_momentum_balance_terms() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing: 1D profiles
    
    def calc_bl_edge(self, **kwargs):
        '''
        determine the boundary layer edge location
        psvel : pseudo-velocity, i.e. [-ω_z] cumulatively integrated in wall-normal direction
        y_edge : the wall-normal coordinate corresponding to the peak of the (interpolated) pseudo-velocity
        j_edge : the nearest index to y_edge
        '''
        
        verbose      = kwargs.get('verbose',True)
        #method       = kwargs.get('method','psvel')
        epsilon      = kwargs.get('epsilon',5e-4)
        acc          = kwargs.get('acc',8)
        edge_stencil = kwargs.get('edge_stencil','half')
        
        if verbose: print('\n'+'ztmd.calc_bl_edge()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if (self.x.ndim==1) and (self.y.ndim==1):
            if hasattr(self,'rectilinear'):
                if not self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if self.curvilinear:
                    raise AssertionError
        elif (self.x.ndim==2) and (self.y.ndim==2):
            if hasattr(self,'rectilinear'):
                if self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if not self.curvilinear:
                    raise AssertionError
        else:
            raise ValueError
        
        if self.requires_wall_norm_interp:
            raise NotImplementedError
        
        if (epsilon is not None):
            if verbose: even_print('epsilon','%0.1e'%(epsilon,))
        
        # ===
        
        nx = self.nx
        ny = self.ny
        
        ## copy 2D datasets into memory
        u         = np.copy( self['data/u'][()].T      )
        v         = np.copy( self['data/v'][()].T      )
        p         = np.copy( self['data/p'][()].T      )
        T         = np.copy( self['data/T'][()].T      )
        rho       = np.copy( self['data/rho'][()].T    )
        mu        = np.copy( self['data/mu'][()].T     )
        vort_z    = np.copy( self['data/vort_z'][()].T )
        nu        = np.copy( self['data/nu'][()].T     )
        M         = np.copy( self['data/M'][()].T      )
        
        if self.rectilinear:
            
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            utang  = np.copy( self['data/utang'][()].T  )
            unorm  = np.copy( self['data/unorm'][()].T  )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        # ===
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # ===
        
        psvel = np.zeros(shape=(nx,ny) , dtype=np.float64 )
        #j_max = np.zeros(shape=(nx,)   , dtype=np.int32   )
        #y_max = np.zeros(shape=(nx,)   , dtype=np.float64 )
        
        for i in range(nx):
            
            ## pseudo-velocity is a cumulative integration of (-) z-vorticity
            psvel_     = sp.integrate.cumulative_trapezoid(-1*vort_z[i,:], y_, initial=0.)
            psvel[i,:] = psvel_
            
            # ## the index of maximum
            # j_max_     = psvel_.argmax()
            # j_max[i]   = j_max_
            # y_max[i]   = y_[j_max_]
        
        if ('data/psvel' in self): del self['data/psvel']
        dset = self.create_dataset('data/psvel', data=psvel.T, chunks=None)
        if verbose: even_print('data/psvel','%s'%str(psvel.shape))
        
        ## not useful, don't write
        # if ('data_1Dx/j_max' in self): del self['data_1Dx/j_max']
        # dset = self.create_dataset('data_1Dx/j_max', data=j_max, chunks=None)
        # if verbose: even_print('data_1Dx/j_max','%s'%str(j_max.shape))
        # 
        # if ('data_1Dx/y_max' in self): del self['data_1Dx/y_max']
        # dset = self.create_dataset('data_1Dx/y_max', data=y_max, chunks=None)
        # if verbose: even_print('data_1Dx/y_max','%s'%str(y_max.shape))
        
        # ===
        
        y_edge = np.zeros(shape=(nx,) , dtype=np.float64 )
        j_edge = np.zeros(shape=(nx,) , dtype=np.int32   )
        
        y_edge_2d = np.zeros(shape=(nx,2) , dtype=np.float64 )
        y_edge_g  = np.zeros(shape=(nx,)  , dtype=np.float64 )
        
        psvel_max_per_x = np.amax(psvel, axis=1)
        ddy_psvel_normed = gradient( psvel/psvel_max_per_x[:,np.newaxis], y_/self.lchar, axis=1, d=1, acc=acc, edge_stencil=edge_stencil )
        
        if verbose: progress_bar = tqdm(total=nx, ncols=100, desc='y_edge', leave=False, file=sys.stdout)
        for i in range(nx):
            
            psvel_     = np.copy( psvel[i,:] )
            ddy_psvel_ = np.copy( ddy_psvel_normed[i,:] )
            
            ## get [y] of first intersection
            jtop = ny-1
            for j in range(ny):
                if ( ddy_psvel_[j] < epsilon):
                    jtop = j
                    break
            
            # if (i==5001):
            #     plt.close('all')
            #     fig1 = plt.figure(figsize=(3,3), dpi=300)
            #     ax1 = plt.gca()
            #     ax1.plot(  psvel_, y/self.lchar, lw=0.5 )
            #     #ax1.axhline(y=y_max[i]/self.lchar, linestyle='dashed', c='lightgray', zorder=1, lw=0.3)
            #     fig1.tight_layout(pad=0.25)
            #     fig1.tight_layout(pad=0.25)
            #     plt.show()
            
            # print(f'i={i:d}, s={stang[i]/self.lchar:0.5f}')
            
            # print(ddy_psvel_.min())
            
            # if (i==3739):
            #     plt.close('all')
            #     fig1 = plt.figure(figsize=(3,3), dpi=300)
            #     ax1 = plt.gca()
            #     #ax1.plot(  ddy_psvel_[jtop-2:jtop+1], y_[jtop-2:jtop+1]/self.lchar, lw=0.5, marker='o' )
            #     ax1.plot(  ddy_psvel_, y_/self.lchar, lw=0.5 )
            #     ax1.axvline(x=epsilon, linestyle='dashed', c='lightgray', zorder=1, lw=0.3)
            #     fig1.tight_layout(pad=0.25)
            #     fig1.tight_layout(pad=0.25)
            #     plt.show()
            
            intrp_func = sp.interpolate.interp1d(y_/self.lchar, ddy_psvel_, kind='linear', bounds_error=True)
            
            def __f_opt1(y_test, intrp_func, epsilon):
                ddy_psvel_test = intrp_func(y_test/self.lchar)
                root = np.abs( ddy_psvel_test - epsilon )
                return root
            
            sol = sp.optimize.least_squares(fun=__f_opt1,
                                            args=(intrp_func, epsilon),
                                            x0=0.5*(y_[jtop-2]+y_[jtop]),
                                            xtol=1e-14,
                                            #ftol=1e-15,
                                            method='dogbox',
                                            bounds=(y_[jtop-2], y_[jtop]))
            if not sol.success:
                raise ValueError
            
            y_edge_ = sol.x[0]
            
            # ===
            
            y_edge[i]   = y_edge_
            j_edge_     = np.abs( y_ - y_edge_ ).argmin()
            j_edge[i]   = j_edge_
            y_edge_g[i] = y_[j_edge_]
            
            ## get the [x,y] coordinates of the 'edge line' --> shape=(nx,2)
            if self.rectilinear:
                pt_edge_ = np.array([self.x[i],y_edge_], dtype=np.float64)
            elif self.curvilinear:
                p0_ = np.array([self.x[i,0],self.y[i,0]], dtype=np.float64)
                vnorm_ = np.copy( vnorm[i,0,:] ) ## unit normal vec @ wall at this x
                pt_edge_ = p0_ + np.dot( y_edge_ , vnorm_ )
            else:
                raise ValueError
            
            y_edge_2d[i,:] = pt_edge_
            
            progress_bar.update()
        progress_bar.close()
        
        if ('data_1Dx/y_edge' in self): del self['data_1Dx/y_edge']
        dset = self.create_dataset('data_1Dx/y_edge', data=y_edge, chunks=None)
        if verbose: even_print('data_1Dx/y_edge','%s'%str(y_edge.shape))
        
        if ('data_1Dx/y_edge_g' in self): del self['data_1Dx/y_edge_g']
        dset = self.create_dataset('data_1Dx/y_edge_g', data=y_edge_g, chunks=None)
        if verbose: even_print('data_1Dx/y_edge_g','%s'%str(y_edge_g.shape))
        
        if ('data_1Dx/y_edge_2d' in self): del self['data_1Dx/y_edge_2d']
        dset = self.create_dataset('data_1Dx/y_edge_2d', data=y_edge_2d, chunks=None)
        if verbose: even_print('data_1Dx/y_edge_2d','%s'%str(y_edge_2d.shape))
        
        if ('data_1Dx/j_edge' in self): del self['data_1Dx/j_edge']
        dset = self.create_dataset('data_1Dx/j_edge', data=j_edge, chunks=None)
        if verbose: even_print('data_1Dx/j_edge','%s'%str(j_edge.shape))
        
        # ===
        
        if False:
            plt.close('all')
            fig1 = plt.figure(figsize=(3*2,3), dpi=300)
            ax1 = plt.gca()
            
            if self.rectilinear:
                s_ = np.copy(x)
            elif self.curvilinear:
                s_ = np.copy(stang)
            else:
                raise ValueError
            
            #ax1.plot( s_/self.lchar, y_max/self.lchar,    label='y_max',    lw=0.5 )
            ax1.plot( s_/self.lchar, y_edge_g/self.lchar, label='y_edge_g', lw=0.5 )
            ax1.plot( s_/self.lchar, y_edge/self.lchar,   label='y_edge',   lw=0.5 )
            #ax1.set_xlabel('stang')
            #ax1.set_ylabel('snorm')
            lg = ax1.legend(loc='upper left', ncol=1, fontsize=8, facecolor=ax1.get_facecolor())
            lg.get_frame().set_linewidth(0.2)
            lg.set_zorder(21)
            fig1.tight_layout(pad=0.25)
            fig1.tight_layout(pad=0.25)
            plt.show()
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_bl_edge() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_bl_edge_quantities(self, **kwargs):
        '''
        calculate field quantity values at [y_edge]
        - skin friction coefficient: cf
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_bl_edge_quantities()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        # ===
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        ## the wall normal location of max pseudo velocity
        y_edge = np.copy( self['data_1Dx/y_edge'][()] )
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # === make a structured array
        
        names  = [ 'rho', 'u', 'v', 'w', 'T', 'p', 'vort_z', 'mu', 'nu', 'M', 'psvel' ]
        if self.curvilinear:
            names += [ 'utang', 'unorm' ]
        
        dtypes=[]
        for n in names:
            ds = self[f'data/{n}']
            dtypes.append( ds.dtype )
        
        names_edge = [ n+'_edge' for n in names ]
        
        data      = np.zeros(shape=(nx,ny), dtype={'names':names,      'formats':dtypes})
        data_edge = np.zeros(shape=(nx,),   dtype={'names':names_edge, 'formats':dtypes})
        
        # === populate structured array
        
        for scalar in data.dtype.names:
            data[scalar][:,:] = np.copy( self[f'data/{scalar}'][()].T )
        
        # === interpolate edge quantity for all vars
        
        if verbose: progress_bar = tqdm(total=nx*len(names), ncols=100, desc='edge quantities', leave=False, file=sys.stdout)
        for scalar in data.dtype.names:
            for i in range(nx):
                
                data_y_    = np.copy( data[scalar][i,:] )
                intrp_func = sp.interpolate.interp1d(y_, data_y_, kind='cubic', bounds_error=True)
                
                y_edge_    = y_edge[i]
                data_edge_ = intrp_func(y_edge_)
                
                data_edge[scalar+'_edge'][i] = data_edge_
                
                if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # === write
        
        for scalar in data_edge.dtype.names:
            if (f'data_1Dx/{scalar}' in self):
                del self[f'data_1Dx/{scalar}']
            data_ = np.copy( data_edge[scalar][:] )
            dset = self.create_dataset(f'data_1Dx/{scalar}', data=data_, chunks=None)
            if verbose: even_print(f'data_1Dx/{scalar}',str(dset.shape))
        
        # ===
        
        if False:
            plt.close('all')
            fig1 = plt.figure(figsize=(3*2,3), dpi=300)
            ax1 = plt.gca()
            ax1.plot( stang/self.lchar, data_edge['utang_edge']/self.U_inf, lw=0.5 )
            ax1.set_xlabel('stang')
            fig1.tight_layout(pad=0.25)
            fig1.tight_layout(pad=0.25)
            plt.show()
        
        # === skin friction
        
        u_tau    = np.copy( self['data_1Dx/u_tau'][()]    )
        rho_wall = np.copy( self['data_1Dx/rho_wall'][()] )
        tau_wall = np.copy( self['data_1Dx/tau_wall'][()] )
        
        u_edge   = np.copy( data_edge['u_edge']   )
        rho_edge = np.copy( data_edge['rho_edge'] )
        
        if self.curvilinear:
            utang_edge = np.copy( data_edge['utang_edge'] )
        
        if self.rectilinear:
            
            # cf_1   = np.copy( 2. * (u_tau/u_edge)**2 * (rho_wall/rho_edge) )
            # cf_2   = np.copy( 2. * tau_wall / (rho_edge*u_edge**2) )
            # np.testing.assert_allclose(cf_1, cf_2, rtol=1e-6, atol=1e-8)
            # cf = np.copy(cf_2)
            
            cf_inf = np.copy( 2. * tau_wall / ( self.rho_inf * self.U_inf**2 ) )
            cf = np.copy(cf_inf)
        
        elif self.curvilinear:
            
            # cf_1 = 2. * (u_tau/utang_edge)**2 * (rho_wall/rho_edge)
            # cf_2 = 2. * tau_wall / (rho_edge*utang_edge**2)
            # np.testing.assert_allclose(cf_1, cf_2, rtol=1e-6, atol=1e-8)
            # cf = np.copy(cf_2)
            
            cf_inf = np.copy( 2. * tau_wall / ( self.rho_inf * self.U_inf**2 ) )
            cf = np.copy(cf_inf)
        
        else:
            raise ValueError
        
        if ('data_1Dx/cf' in self): del self['data_1Dx/cf']
        self.create_dataset('data_1Dx/cf', data=cf, chunks=None)
        if verbose: even_print('data_1Dx/cf', '%s'%str(cf.shape))
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_bl_edge_quantities() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_d99(self, **kwargs):
        '''
        calculate [δ99]
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_d99()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        # ===
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # ## the coordinate/index of the location of (discrete) max pseudo-velocity
        # j_max = np.copy( self['data_1Dx/j_max'][()] )
        # y_max = np.copy( self['data_1Dx/y_max'][()] )
        
        ## the wall normal location of max pseudo velocity (interpolated, O3 cubic spline)
        y_edge   = np.copy( self['data_1Dx/y_edge'][()] )
        y_edge_g = np.copy( self['data_1Dx/y_edge_g'][()] )
        
        ## the index closest y_edge
        j_edge = np.copy( self['data_1Dx/j_edge'][()] )
        
        psvel      = np.copy( self['data/psvel'][()].T )
        psvel_edge = np.copy( self['data_1Dx/psvel_edge'][()].T )
        
        # if self.rectilinear:
        #     u = np.copy( self['data/u'][()].T )
        #     u_edge = np.copy( self['data_1Dx/u_edge'][()] )
        # elif self.curvilinear:
        #     utang = np.copy( self['data/utang'][()].T )
        #     utang_edge = np.copy( self['data_1Dx/utang_edge'][()] )
        # else:
        #     raise ValueError
        
        # ===
        
        d99     = np.zeros(shape=(nx,),  dtype=np.float64 )
        d99_2d  = np.zeros(shape=(nx,2), dtype=np.float64 )
        j99     = np.zeros(shape=(nx,),  dtype=np.int32   )
        d99g    = np.zeros(shape=(nx,),  dtype=np.float64 )
        
        if verbose: progress_bar = tqdm(total=nx, ncols=100, desc='d99', leave=False, file=sys.stdout)
        for i in range(nx):
            
            y_edge_     = y_edge[i]
            je          = j_edge[i] + 10
            psvel_      = np.copy( psvel[i,:] )
            psvel_edge_ = psvel_edge[i]
            
            # ## the local 1D wall-normal coordinate
            # if self.rectilinear:
            #     u_ = np.copy(u[i,:])
            #     u_edge_ = u_edge[i]
            # elif self.curvilinear:
            #     u_ = np.copy(utang[i,:])
            #     u_edge_ = utang_edge[i]
            # else:
            #     raise ValueError
            
            intrp_func = sp.interpolate.interp1d(y_[:je], psvel_[:je], kind='cubic', bounds_error=True)
            #intrp_func = sp.interpolate.interp1d(y_[:je], u_[:je], kind='cubic', bounds_error=True)
            
            # === get d99 = wall-normal location of 99% psvel_edge 
            
            def __f_opt(y_test, intrp_func, psvel_edge_):
                root = np.abs( 0.99*psvel_edge_ - intrp_func(y_test) )
                return root
            
            # def __f_opt(y_test, intrp_func, u_edge_):
            #     root = np.abs( 0.99*u_edge_ - intrp_func(y_test) )
            #     return root
            
            sol = sp.optimize.least_squares(fun=__f_opt,
                                            args=(intrp_func,psvel_edge_),
                                            #args=(intrp_func,u_edge_),
                                            x0=0.99*y_edge_,
                                            xtol=1e-14,
                                            #ftol=1e-14,
                                            method='dogbox',
                                            bounds=(y_.min(), y_edge_))
            if not sol.success:
                raise ValueError
            
            d99_   = sol.x[0]
            d99[i] = d99_
            
            j99_    = np.abs( y_ - d99_ ).argmin()
            j99[i]  = j99_
            d99g[i] = y_[j99_]
            
            # ===
            
            ## get the [x,y] coordinates of the 'd99 line' --> shape=(nx,2)
            if self.rectilinear:
                pt_99_ = np.array([self.x[i],d99_], dtype=np.float64)
            elif self.curvilinear:
                p0_ = np.array([self.x[i,0],self.y[i,0]], dtype=np.float64)
                vnorm_ = np.copy( vnorm[i,0,:] ) ## unit normal vec @ wall at this x
                pt_99_ = p0_ + np.dot( d99_ , vnorm_ )
            else:
                raise ValueError
            
            d99_2d[i,:] = pt_99_
            
            progress_bar.update()
        progress_bar.close()
        
        if ('data_1Dx/d99' in self): del self['data_1Dx/d99']
        dset = self.create_dataset('data_1Dx/d99', data=d99, chunks=None)
        if verbose: even_print('data_1Dx/d99','%s'%str(d99.shape))
        
        if ('data_1Dx/d99_2d' in self): del self['data_1Dx/d99_2d']
        dset = self.create_dataset('data_1Dx/d99_2d', data=d99_2d, chunks=None)
        if verbose: even_print('data_1Dx/d99_2d','%s'%str(d99_2d.shape))
        
        if ('data_1Dx/d99g' in self): del self['data_1Dx/d99g']
        dset = self.create_dataset('data_1Dx/d99g', data=d99g, chunks=None)
        if verbose: even_print('data_1Dx/d99g','%s'%str(d99g.shape))
        
        if ('data_1Dx/j99' in self): del self['data_1Dx/j99']
        dset = self.create_dataset('data_1Dx/j99', data=j99, chunks=None)
        if verbose: even_print('data_1Dx/j99','%s'%str(j99.shape))
        
        # ===
        
        if False:
            plt.close('all')
            fig1 = plt.figure(figsize=(3*2,3), dpi=300)
            ax1 = plt.gca()
            ax1.plot( stang/self.lchar, y_max/self.lchar, label='y_max', lw=0.5 )
            ax1.plot( stang/self.lchar, y_edge/self.lchar, label='y_edge', lw=0.5 )
            #ax1.plot( stang/self.lchar, d99g/self.lchar,  label='d99g',  lw=0.5 )
            ax1.plot( stang/self.lchar, d99/self.lchar,    label='d99',    lw=0.5 )
            ax1.set_xlabel('stang')
            #ax1.set_ylabel('snorm')
            lg = ax1.legend(loc='lower right', ncol=1, fontsize=8, facecolor=ax1.get_facecolor())
            lg.get_frame().set_linewidth(0.2)
            lg.set_zorder(21)
            fig1.tight_layout(pad=0.25)
            fig1.tight_layout(pad=0.25)
            plt.show()
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_d99() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_d99_quantities(self, **kwargs):
        '''
        calculate interpolated field quantity values at [δ99]
        - sc_l_out = δ99
        - sc_u_out = u99
        - sc_t_out = u99/d99
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_d99_quantities()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        # ===
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        d99 = np.copy( self['data_1Dx/d99'][()] )
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # === make a structured array
        
        names  = [ 'rho', 'u', 'v', 'w', 'T', 'p', 'vort_z', 'mu', 'nu', 'M', 'psvel' ]
        if self.curvilinear:
            names += [ 'utang', 'unorm' ]
        
        dtypes=[]
        for n in names:
            ds = self[f'data/{n}']
            dtypes.append( ds.dtype )
        
        #names_99 = [ n+'99' for n in names ]
        names_99 = [ n+'_99' if ('_' in n) else n+'99' for n in names ]
        
        data    = np.zeros(shape=(nx,ny), dtype={'names':names,    'formats':dtypes})
        data_99 = np.zeros(shape=(nx,),   dtype={'names':names_99, 'formats':dtypes})
        
        # === populate structured array
        
        for scalar in data.dtype.names:
            data[scalar][:,:] = np.copy( self[f'data/{scalar}'][()].T )
        
        # === interpolate @ d99 for all vars
        
        if verbose: progress_bar = tqdm(total=nx*len(names), ncols=100, desc='d99 quantities', leave=False, file=sys.stdout)
        for ni,scalar in enumerate(data.dtype.names):
            for i in range(nx):
                
                data_y_    = np.copy( data[scalar][i,:] )
                intrp_func = sp.interpolate.interp1d(y_, data_y_, kind='cubic', bounds_error=True)
                
                d99_    = d99[i]
                data_99_ = intrp_func(d99_)
                
                data_99[names_99[ni]][i] = data_99_
                
                if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # === write
        
        for scalar in data_99.dtype.names:
            if (f'data_1Dx/{scalar}' in self):
                del self[f'data_1Dx/{scalar}']
            data_ = np.copy( data_99[scalar][:] )
            dset = self.create_dataset(f'data_1Dx/{scalar}', data=data_, chunks=None)
            if verbose: even_print(f'data_1Dx/{scalar}',str(dset.shape))
        
        # === outer scales: length, velocity & time
        
        sc_l_out = np.copy( d99 )
        
        if self.rectilinear:
            sc_u_out = np.copy( data_99['u99'] )
            sc_t_out = np.copy( d99/data_99['u99'] )
        elif self.curvilinear:
            sc_u_out = np.copy( data_99['utang99'] )
            sc_t_out = np.copy( d99/data_99['utang99'] )
        else:
            raise ValueError
        
        np.testing.assert_allclose(sc_t_out, sc_l_out/sc_u_out, rtol=1e-14, atol=1e-14)
        
        u_tau = np.copy( self['data_1Dx/u_tau'][()] )
        sc_t_eddy = np.copy( d99/u_tau )
        
        if ('data_1Dx/sc_u_out' in self): del self['data_1Dx/sc_u_out']
        self.create_dataset('data_1Dx/sc_u_out', data=sc_u_out, chunks=None)
        if verbose: even_print('data_1Dx/sc_u_out', '%s'%str(sc_u_out.shape))
        
        if ('data_1Dx/sc_l_out' in self): del self['data_1Dx/sc_l_out']
        self.create_dataset('data_1Dx/sc_l_out', data=sc_l_out, chunks=None)
        if verbose: even_print('data_1Dx/sc_l_out', '%s'%str(sc_l_out.shape))
        
        if ('data_1Dx/sc_t_out' in self): del self['data_1Dx/sc_t_out']
        self.create_dataset('data_1Dx/sc_t_out', data=sc_t_out, chunks=None)
        if verbose: even_print('data_1Dx/sc_t_out', '%s'%str(sc_t_out.shape))
        
        if ('data_1Dx/sc_t_eddy' in self): del self['data_1Dx/sc_t_eddy']
        self.create_dataset('data_1Dx/sc_t_eddy', data=sc_t_eddy, chunks=None)
        if verbose: even_print('data_1Dx/sc_t_eddy', '%s'%str(sc_t_eddy.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_d99_quantities() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_bl_integral_quantities(self, **kwargs):
        '''
        θ, δ*, Re_θ, Re_τ
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_bl_integral_quantities()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        # ===
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # === copy 1D datasets into memory
        
        u_tau    = np.copy( self['data_1Dx/u_tau'][()]    )
        rho_wall = np.copy( self['data_1Dx/rho_wall'][()] )
        nu_wall  = np.copy( self['data_1Dx/nu_wall'][()]  )
        mu_wall  = np.copy( self['data_1Dx/mu_wall'][()]  )
        T_wall   = np.copy( self['data_1Dx/T_wall'][()]   )
        
        j_edge     = np.copy( self['data_1Dx/j_edge'][()] )
        y_edge     = np.copy( self['data_1Dx/y_edge'][()] )
        d99        = np.copy( self['data_1Dx/d99'][()]    )
        #j99        = np.copy( self['data_1Dx/j99'][()]    )
        
        if self.rectilinear:
            u_edge = np.copy( self['data_1Dx/u_edge'][()] )
        elif self.curvilinear:
            utang_edge = np.copy( self['data_1Dx/utang_edge'][()] )
        else:
            raise ValueError
        
        rho_edge = np.copy( self['data_1Dx/rho_edge'][()] )
        mu_edge  = np.copy( self['data_1Dx/mu_edge'][()]  )
        nu_edge  = np.copy( self['data_1Dx/nu_edge'][()]  )
        
        sc_l_out = np.copy( self['data_1Dx/sc_l_out'][()] )
        sc_u_out = np.copy( self['data_1Dx/sc_u_out'][()] )
        sc_t_out = np.copy( self['data_1Dx/sc_t_out'][()] )
        
        # === copy 2D datasets into memory
        
        u = np.copy( self['data/u'][()].T )
        v = np.copy( self['data/v'][()].T )
        
        vort_z = np.copy( self['data/vort_z'][()].T )
        nu     = np.copy( self['data/nu'][()].T     )
        T      = np.copy( self['data/T'][()].T      )
        rho    = np.copy( self['data/rho'][()].T    )
        
        if self.curvilinear:
            utang = np.copy( self['data/utang'][()].T )
        
        # ===
        
        theta_cmp = np.zeros(shape=(nx,), dtype=np.float64) ## momentum thickness
        theta_inc = np.zeros(shape=(nx,), dtype=np.float64)
        dstar_cmp = np.zeros(shape=(nx,), dtype=np.float64) ## displacement thickness
        dstar_inc = np.zeros(shape=(nx,), dtype=np.float64)
        
        if self.rectilinear:
            u_vd = np.zeros(shape=(nx,ny), dtype=np.float64) ## Van Driest scaled u/u_tang
        elif self.curvilinear:
            utang_vd = np.zeros(shape=(nx,ny), dtype=np.float64) ## Van Driest scaled u/u_tang
        else:
            raise ValueError
        
        if verbose: progress_bar = tqdm(total=nx, ncols=100, desc='θ,δ*,Re_θ,Re_τ', leave=False, file=sys.stdout)
        for i in range(nx):
            
            y_edge_ = y_edge[i]
            
            if self.rectilinear:
                u_      = np.copy( u[i,:] )
                u_edge_ = u_edge[i]
            elif self.curvilinear:
                u_      = np.copy( utang[i,:] )
                u_edge_ = utang_edge[i]
            else:
                raise ValueError
            
            rho_ = np.copy( rho[i,:] )
            
            # ===
            
            integrand_theta_cmp = (u_*rho_)/(u_edge_*rho_edge[i])*(1-(u_/u_edge_))
            integrand_dstar_cmp = (1-((u_*rho_)/(u_edge_*rho_edge[i])))
            
            theta_cmp_     = sp.integrate.cumulative_trapezoid(y=integrand_theta_cmp, x=y_, initial=0.)
            theta_cmp_func = sp.interpolate.interp1d(y_, theta_cmp_, kind='cubic')
            theta_cmp[i]   = theta_cmp_func(y_edge_)
            
            dstar_cmp_     = sp.integrate.cumulative_trapezoid(y=integrand_dstar_cmp, x=y_, initial=0.)
            dstar_cmp_func = sp.interpolate.interp1d(y_, dstar_cmp_, kind='cubic')
            dstar_cmp[i]   = dstar_cmp_func(y_edge_)
            
            # ===
            
            integrand_theta_inc = (u_/u_edge_)*(1-(u_/u_edge_))
            integrand_dstar_inc = 1-(u_/u_edge_)
            
            theta_inc_     = sp.integrate.cumulative_trapezoid(y=integrand_theta_inc, x=y_, initial=0.)
            theta_inc_func = sp.interpolate.interp1d(y_, theta_inc_, kind='cubic')
            theta_inc[i]   = theta_inc_func(y_edge_)
            
            dstar_inc_     = sp.integrate.cumulative_trapezoid(y=integrand_dstar_inc, x=y_, initial=0.)
            dstar_inc_func = sp.interpolate.interp1d(y_, dstar_inc_, kind='cubic')
            dstar_inc[i]   = dstar_inc_func(y_edge_)
            
            # ===
            
            integrand_u_vd  = np.sqrt(T_wall[i]/T[i,:])
            #integrand_u_vd  = np.sqrt(rho[i,:]/rho_wall[i])
            
            if self.rectilinear:
                u_vd[i,:] = sp.integrate.cumulative_trapezoid(integrand_u_vd, u[i,:], initial=0)
            elif self.curvilinear:
                utang_vd[i,:] = sp.integrate.cumulative_trapezoid(integrand_u_vd, utang[i,:], initial=0)
            else:
                raise ValueError
            
            # ===
            
            if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        if ('data_1Dx/theta_inc' in self): del self['data_1Dx/theta_inc']
        self.create_dataset('data_1Dx/theta_inc', data=theta_inc, chunks=None)
        if verbose: even_print('data_1Dx/theta_inc', '%s'%str(theta_inc.shape))
        
        if ('data_1Dx/dstar_inc' in self): del self['data_1Dx/dstar_inc']
        self.create_dataset('data_1Dx/dstar_inc', data=dstar_inc, chunks=None)
        if verbose: even_print('data_1Dx/dstar_inc', '%s'%str(dstar_inc.shape))
        
        if ('data_1Dx/theta_cmp' in self): del self['data_1Dx/theta_cmp']
        self.create_dataset('data_1Dx/theta_cmp', data=theta_cmp, chunks=None)
        if verbose: even_print('data_1Dx/theta_cmp', '%s'%str(theta_cmp.shape))
        
        if ('data_1Dx/dstar_cmp' in self): del self['data_1Dx/dstar_cmp']
        self.create_dataset('data_1Dx/dstar_cmp', data=dstar_cmp, chunks=None)
        if verbose: even_print('data_1Dx/dstar_cmp', '%s'%str(dstar_cmp.shape))
        
        if self.rectilinear:
            if ('data/u_vd' in self): del self['data/u_vd']
            self.create_dataset('data/u_vd', data=u_vd.T, chunks=None)
            if verbose: even_print('data/u_vd', '%s'%str(u_vd.shape))
        elif self.curvilinear:
            if ('data/utang_vd' in self): del self['data/utang_vd']
            self.create_dataset('data/utang_vd', data=utang_vd.T, chunks=None)
            if verbose: even_print('data/utang_vd', '%s'%str(utang_vd.shape))
        else:
            raise ValueError
        
        # ===
        
        #theta   = np.copy(theta_cmp)
        #dstar   = np.copy(dstar_cmp)
        H12     = dstar_cmp/theta_cmp
        H12_inc = dstar_inc/theta_inc
        
        if self.rectilinear:
            Re_tau        = d99*u_tau/nu_wall
            Re_theta      = theta_cmp*u_edge/nu_edge
            Re_theta_inc  = theta_inc*u_edge/nu_edge
            Re_d99        = d99*u_edge/nu_edge
        elif self.curvilinear:
            Re_tau        = d99*u_tau/nu_wall
            Re_theta      = theta_cmp*utang_edge/nu_edge
            Re_theta_inc  = theta_inc*utang_edge/nu_edge
            Re_d99        = d99*utang_edge/nu_edge
        else:
            raise ValueError
        
        if ('data_1Dx/H12' in self): del self['data_1Dx/H12']
        self.create_dataset('data_1Dx/H12', data=H12, chunks=None)
        if verbose: even_print('data_1Dx/H12', '%s'%str(H12.shape))
        
        if ('data_1Dx/H12_inc' in self): del self['data_1Dx/H12_inc']
        self.create_dataset('data_1Dx/H12_inc', data=H12_inc, chunks=None)
        if verbose: even_print('data_1Dx/H12_inc', '%s'%str(H12_inc.shape))
        
        if ('data_1Dx/Re_tau' in self): del self['data_1Dx/Re_tau']
        self.create_dataset('data_1Dx/Re_tau', data=Re_tau, chunks=None)
        if verbose: even_print('data_1Dx/Re_tau', '%s'%str(Re_tau.shape))
        
        if ('data_1Dx/Re_theta' in self): del self['data_1Dx/Re_theta']
        self.create_dataset('data_1Dx/Re_theta', data=Re_theta, chunks=None)
        if verbose: even_print('data_1Dx/Re_theta', '%s'%str(Re_theta.shape))
        
        if ('data_1Dx/Re_theta_inc' in self): del self['data_1Dx/Re_theta_inc']
        self.create_dataset('data_1Dx/Re_theta_inc', data=Re_theta_inc, chunks=None)
        if verbose: even_print('data_1Dx/Re_theta_inc', '%s'%str(Re_theta_inc.shape))
        
        if ('data_1Dx/Re_d99' in self): del self['data_1Dx/Re_d99']
        self.create_dataset('data_1Dx/Re_d99', data=Re_d99, chunks=None)
        if verbose: even_print('data_1Dx/Re_d99', '%s'%str(Re_d99.shape))
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_bl_integral_quantities() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === legacy: functions for curved TBL cases where axis=1 mesh is not identical over axis=0
    
    def get_wall_norm_mesh(self, **kwargs):
        '''
        get a new 'grid' which is extruded in the normal direction from the wall
        - this grid is good for post-processing in the wall-normal direction ONLY
        - orthogonal to the wall-normal direction may have jumps or be folded
        '''
        
        verbose = kwargs.get('verbose',True)
        
        ## the wall-normal unit tangent and normal vectors
        if ('csys/vtang' in self) and ('csys/vnorm' in self):
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            #wall_trafo_mat = np.stack((vtang,vnorm), axis=-1)
        else:
            raise AssertionError('no vnorm/vtang unit projection vector!')
        
        if ('data/wall_distance' in self):
            wall_dist = np.copy( self['data/wall_distance'][()].T )
        else:
            raise AssertionError('dset not present: data/wall_distance')
        
        xy2d_n1D          = np.zeros((self.nx,self.ny,2) , dtype=np.float64)
        wall_distance_n1D = np.zeros((self.nx,self.ny)   , dtype=np.float64)
        
        for i in range(self.nx):
            
            p0_ = np.array([self.x[i,0],self.y[i,0]], dtype=np.float64) ## wall point coordinate
            
            vnorm_ = np.copy( vnorm[i,0,:] ) ## unit normal vec @ wall at this x
            #vtang_ = np.copy( vtang[i,0,:] ) ## unit tangent vec @ wall at this x
            
            #x_  = np.copy( self.x[i,:] )
            #y_  = np.copy( self.y[i,:] )
            #dx_ = np.diff(x_,n=1)
            #dy_ = np.diff(y_,n=1)
            #ds_ = np.sqrt(dx_**2+dy_**2)
            #s_  = np.cumsum(np.concatenate(([0.,],ds_))) ## path length normal to wall @ this x
            
            s_ = np.copy( wall_dist[i,:] )
            
            wall_distance_n1D[i,:] = s_
            
            xy = p0_ + np.einsum( 'i,j->ij', s_, vnorm_ )
            
            xy2d_n1D[i,:,:] = xy
        
        if ('dims_2Dw/x' in self): del self['dims_2Dw/x']
        self.create_dataset('dims_2Dw/x', data=np.squeeze(xy2d_n1D[:,:,0]).T, chunks=None)
        
        if ('dims_2Dw/y' in self): del self['dims_2Dw/y']
        self.create_dataset('dims_2Dw/y', data=np.squeeze(xy2d_n1D[:,:,1]).T, chunks=None)
        
        if ('data_2Dw/wall_distance' in self): del self['data_2Dw/wall_distance']
        self.create_dataset('data_2Dw/wall_distance', data=wall_distance_n1D.T, chunks=None)
        
        self.attrs['requires_wall_norm_interp'] = True
        self.get_header(verbose=False)
        
        # ===
        
        if False: ## debug plot
            
            lwg = 0.12 ## line width grid
            
            xy2d1 = np.copy( np.stack((self.x,self.y), axis=-1) / self.lchar )
            xy2d2 = np.copy( xy2d_n1D / self.lchar )
            
            plt.close('all')
            mpl.style.use('dark_background')
            fig1 = plt.figure(figsize=(8,8/2), dpi=230)
            ax1 = fig1.gca()
            ax1.set_aspect('equal')
            ax1.tick_params(axis='x', which='both', direction='out')
            ax1.tick_params(axis='y', which='both', direction='out')
            ##
            # grid_ln_y = mpl.collections.LineCollection(xy2d1,                       linewidth=lwg, edgecolors='red', zorder=19)
            # grid_ln_x = mpl.collections.LineCollection(np.transpose(xy2d1,(1,0,2)), linewidth=lwg, edgecolors='red', zorder=19)
            # ax1.add_collection(grid_ln_y)
            # ax1.add_collection(grid_ln_x)
            ##
            grid_ln_y = mpl.collections.LineCollection(xy2d2,                       linewidth=lwg, edgecolors=ax1.xaxis.label.get_color(), zorder=19)
            grid_ln_x = mpl.collections.LineCollection(np.transpose(xy2d2,(1,0,2)), linewidth=lwg, edgecolors=ax1.xaxis.label.get_color(), zorder=19)
            ax1.add_collection(grid_ln_y)
            ax1.add_collection(grid_ln_x)
            ##
            ax1.set_xlabel('$x/\ell_{char}$')
            ax1.set_ylabel('$y/\ell_{char}$')
            ##
            ax1.set_xlim(xy2d1[:,:,0].min()-3,xy2d1[:,:,0].max()+3)
            ax1.set_ylim(xy2d1[:,:,1].min()-3,xy2d1[:,:,1].max()+3)
            ##
            #ax1.set_xlim(self.plot_xlim[0],self.plot_xlim[1])
            #ax1.set_ylim(self.plot_ylim[0],self.plot_ylim[1])
            ##
            ax1.xaxis.set_major_locator(mpl.ticker.MultipleLocator(20))
            ax1.xaxis.set_minor_locator(mpl.ticker.MultipleLocator(5))
            ax1.yaxis.set_major_locator(mpl.ticker.MultipleLocator(20))
            ax1.yaxis.set_minor_locator(mpl.ticker.MultipleLocator(5))
            ##
            fig1.tight_layout(pad=0.25)
            fig1.tight_layout(pad=0.25)
            ##
            #dpi_out = 2*2160/plt.gcf().get_size_inches()[1]
            #turbx.fig_trim_x(fig1, [ax1], offset_px=10, dpi=dpi_out)
            #fig1.savefig('grid.png', dpi=dpi_out)
            plt.show()
            pass
        
        return
    
    def interp_to_wall_norm_mesh(self, **kwargs):
        '''
        interpolate fields from original grid to 'wall-normal' grid
        '''
        
        verbose = kwargs.get('verbose',True)
        scalars = kwargs.get('scalars',None)
        
        if verbose: print('\n'+'turbx.ztmd.interp_to_wall_norm_mesh()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## read: wall normal grid x
        if ('dims_2Dw/x' in self):
            x_wn = np.copy( self['dims_2Dw/x'][()].T )
        else:
            raise AssertionError('dset not present: dims_2Dw/x')
        
        ## read: wall normal grid y
        if ('dims_2Dw/y' in self):
            y_wn = np.copy( self['dims_2Dw/y'][()].T )
        else:
            raise AssertionError('dset not present: dims_2Dw/y')
        
        # === get list of scalars to interpolate
        
        scalars_2d_names = list(self['data'].keys())
        if verbose: even_print('n scalars found in data/','%i'%(len(scalars_2d_names),))
        
        if (scalars is None): ## take all scalars present in file
            
            ## dont interpolate ddx[] or ddy[] gradients or 'wall_distance'
            scalars_2d_names = [ s for s in scalars_2d_names if ('ddx_' not in s)    ]
            scalars_2d_names = [ s for s in scalars_2d_names if ('ddy_' not in s)    ]
            scalars_2d_names = [ s for s in scalars_2d_names if (s!='wall_distance') ]
        
        else: ## explicit scalar list was passed
            
            if not isinstance(scalars, list):
                raise ValueError("'scalars' should be type list")
            if not isinstance(scalars[0], str):
                raise ValueError("'scalars' should contain strings")
            
            scalars_2d_names = list(self['data'].keys())
            
            ## take only scalars which actually exist
            scalars_2d_names = [ s for s in scalars if (s in scalars_2d_names) ]
        
        # === interpolate
        
        if verbose: even_print('n scalars to be interpolated','%i'%(len(scalars_2d_names),))
        
        if True: ## interpolate
            
            x2d_A = self.x
            y2d_A = self.y
            x2d_B = x_wn
            y2d_B = y_wn
            
            if verbose: progress_bar = tqdm(total=len(scalars_2d_names), ncols=100, desc='interpolate 2D', leave=False, file=sys.stdout)
            for scalar_name in scalars_2d_names:
                
                ## copy data into memory
                scalar_data = np.copy( self['data/%s'%scalar_name][()].T  )
                
                ## do interpolation
                if verbose: tqdm.write(even_print('start interpolate',scalar_name,s=True))
                t_start = timeit.default_timer()
                scalar_data_wn = interp_2d_structured(x2d_A, y2d_A, x2d_B, y2d_B, scalar_data)
                if verbose: tqdm.write(even_print('done interpolating','%s'%format_time_string((timeit.default_timer() - t_start)),s=True))
                
                ## write to HDF5
                if ('data_2Dw/%s'%scalar_name in self):
                    del self['data_2Dw/%s'%scalar_name]
                self.create_dataset('data_2Dw/%s'%scalar_name, data=scalar_data_wn.T, chunks=None)
                
                ## clear from memory
                scalar_data    = None ; del scalar_data
                scalar_data_wn = None ; del scalar_data_wn
                
                progress_bar.update()
            progress_bar.close()
        
        self.attrs['requires_wall_norm_interp'] = True
        self.get_header(verbose=False)
        
        if verbose: print('\n'+72*'-')
        if verbose: print('total time : turbx.interp_to_wall_norm_mesh() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_grid_quality_metrics_2d(self, **kwargs):
        '''
        attach grid quality measures to ZTMD
        '''
        x = np.copy( self['dims/x'][()].T )
        y = np.copy( self['dims/y'][()].T )
        
        grid_quality_dict = get_grid_quality_metrics_2d(x,y,verbose=True)
        
        if ('data_cells/skew' in self): del self['data_cells/skew']
        dset = self.create_dataset('data_cells/skew', data=grid_quality_dict['skew'].T, chunks=None)
        
        if ('data/ds1avg' in self): del self['data/ds1avg']
        dset = self.create_dataset('data/ds1avg', data=grid_quality_dict['ds1avg'].T, chunks=None)
        
        if ('data/ds2avg' in self): del self['data/ds2avg']
        dset = self.create_dataset('data/ds2avg', data=grid_quality_dict['ds2avg'].T, chunks=None)
        
        return
    
    def add_cyl_coords(self, **kwargs):
        '''
        attach [θ,r] coords, calculated from [x,y]
        '''
        cx = kwargs.get('cx',0.)
        cy = kwargs.get('cy',0.)
        
        x = np.copy( self['dims/x'][()].T )
        y = np.copy( self['dims/y'][()].T )
        
        xy2d = np.stack((x,y), axis=-1)
        
        trz = rect_to_cyl(xy2d, cx=cx, cy=cy)
        
        if ('dims/theta' in self): del self['dims/theta']
        dset = self.create_dataset('dims/theta', data=trz[:,:,0].T, chunks=None)
        if ('dims/r' in self): del self['dims/r']
        dset = self.create_dataset('dims/r', data=trz[:,:,1].T, chunks=None)
        
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from ZTMD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        makeVectors = kwargs.get('makeVectors',True) ## write vectors (e.g. velocity, vorticity) to XDMF
        makeTensors = kwargs.get('makeTensors',True) ## write 3x3 tensors (e.g. stress, strain) to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if verbose: print('\n'+'ztmd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for scalar in ['x','y','r','theta']:
            if ('dims/'+scalar in self):
                data = self['dims/'+scalar]
                dataset_precision_dict[scalar] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[scalar] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[scalar] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        ## refresh header
        self.get_header(verbose=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')
        
        # === write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                if self.curvilinear:
                    xdmf_str=f'''
                            <Topology TopologyType="3DSMesh" NumberOfElements="{self.ny:d} {self.nx:d}"/>
                            <Geometry GeometryType="X_Y_Z">
                            <DataItem Dimensions="{self.nx:d} {self.ny:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                                {fname_base}:/dims/{'x'}
                            </DataItem>
                            <DataItem Dimensions="{self.nx:d} {self.ny:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                                {fname_base}:/dims/{'y'}
                            </DataItem>
                            </Geometry>
                            '''
                else:
                    xdmf_str=f'''
                            <Topology TopologyType="3DRectMesh" NumberOfElements="1 {self.ny:d} {self.nx:d}"/>
                            <Geometry GeometryType="VxVyVz">
                            <DataItem Dimensions="{self.nx:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                                {fname_base}:/dims/{'x'}
                            </DataItem>
                            <DataItem Dimensions="{self.ny:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                                {fname_base}:/dims/{'y'}
                            </DataItem>
                            <DataItem Dimensions="1" Format="XML">
                                0.0
                            </DataItem>
                            </Geometry>
                            '''
                    
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # === .xdmf : <Grid> per 2D coordinate array
                    
                    if self.curvilinear:
                        
                        for scalar in ['x','y','r','theta']:
                            
                            dset_hf_path = 'dims/%s'%scalar
                            
                            if (dset_hf_path in self):
                                
                                ## get optional 'label' for Paraview (currently inactive)
                                #if scalar in scalar_names:
                                if False:
                                    scalar_name = scalar_names[scalar]
                                else:
                                    scalar_name = scalar
                                
                                xdmf_str=f'''
                                        <!-- ===== scalar : {scalar} ===== -->
                                        <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                        <DataItem Dimensions="{self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                            {fname_base}:/{dset_hf_path}
                                        </DataItem>
                                        </Attribute>
                                        '''
                                
                                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        
                        dset_hf_path = 'data/%s'%scalar
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        #if scalar in scalar_names:
                        if False:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        if self.curvilinear:
                            xdmf_str=f'''
                                    <!-- ===== scalar : {scalar} ===== -->
                                    <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                    <DataItem Dimensions="{self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                        {fname_base}:/{dset_hf_path}
                                    </DataItem>
                                    </Attribute>
                                    '''
                        else:
                            xdmf_str=f'''
                                    <!-- ===== scalar : {scalar} ===== -->
                                    <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                    <DataItem Dimensions="1 {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                        {fname_base}:/{dset_hf_path}
                                    </DataItem>
                                    </Attribute>
                                    '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : <Grid> per scalar (cell-centered values)
                    
                    if ('data_cells' in self):
                        scalars_cells = list(self['data_cells'].keys())
                        for scalar in scalars_cells:
                            
                            dset_hf_path = 'data_cells/%s'%scalar
                            dset = self[dset_hf_path]
                            dset_precision = dset.dtype.itemsize
                            scalar_name = scalar
                            
                            if (dset.dtype.name=='float32') or (dset.dtype.name=='float64'):
                                dset_numbertype = 'Float'
                            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                                dset_numbertype = 'Integer'
                            else:
                                raise TypeError('dtype not recognized, please update script accordingly')
                            
                            xdmf_str=f'''
                                     <!-- ===== scalar : {scalar} ===== -->
                                     <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Cell">
                                       <DataItem Dimensions="{(self.ny-1):d} {(self.nx-1):d}" NumberType="{dset_numbertype}" Precision="{dset_precision:d}" Format="HDF">
                                         {fname_base}:/{dset_hf_path}
                                       </DataItem>
                                     </Attribute>
                                     '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    xdmf_str='''
                             <!-- ===== end scalars ===== -->
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if verbose: print('--w-> %s'%fname_xdmf_base)
        return

class lpd(h5py.File):
    '''
    Lagrangian Particle Data (LPD)
    ------------------------------
    - super()'ed h5py.File class
    
    Structure
    ---------
    
    lpd.h5
    │
    ├── header/
    │   └── udef_char
    │   └── udef_real
    │
    ├── dims/ --> 1D (rectilinear coords of source volume) --> reference only!
    │   └── x
    │   └── y
    │   └── z
    │   └── t
    │
    └-─ data/
        └── <<scalar>> --> 2D [pts,time]
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, openMode = args

        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.lpd()')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## determine MPI info / hints
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(8*1024**2))) ) ## 8 [MB]
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        ## lpd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop('verbose',False)
        force   = kwargs.pop('force',False)
        
        if (openMode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with lpd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## remove file, touch, stripe
        elif (openMode == 'w') and (force is True) and os.path.isfile(self.fname):
            if (self.rank==0):
                os.remove(self.fname)
                Path(self.fname).touch()
                if shutil.which('lfs') is not None:
                    return_code = subprocess.call('lfs migrate --stripe-count 16 --stripe-size 8M %s > /dev/null 2>&1'%self.fname, shell=True)
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        ## touch, stripe
        elif (openMode == 'w') and not os.path.isfile(self.fname):
            if (self.rank==0):
                Path(self.fname).touch()
                if shutil.which('lfs') is not None:
                    return_code = subprocess.call('lfs migrate --stripe-count 16 --stripe-size 8M %s > /dev/null 2>&1'%self.fname, shell=True)
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        else:
            pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        ## call actual h5py.File.__init__()
        super(lpd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(lpd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed LPD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(lpd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of LPD class instance
        '''
        verbose = kwargs.get('verbose',True)
        
        if (self.rank!=0):
            verbose=False
        
        # === scalars, npts
        
        if ('data' in self):
            self.scalars = list(self['data'].keys()) ## string names of scalars : ['u','v','w'] ...
            npts,nt = self['data/%s'%self.scalars[0]].shape
            self.npts = npts
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self['data/%s'%scalar].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))

        # === udef
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## just make udef_real a dict with udef_char as keys
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.C_Suth      = self.udef['C_Suth']
            self.S_Suth      = self.udef['S_Suth']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            
            if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            
            # === characteristic values : derived
            
            rho_inf = self.rho_inf = self.p_inf/(self.R * self.T_inf)
            mu_inf  = self.mu_inf  = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            nu_inf  = self.nu_inf  = self.mu_inf/self.rho_inf
            a_inf   = self.a_inf   = np.sqrt(self.kappa*self.R*self.T_inf)
            U_inf   = self.U_inf   = self.Ma*self.a_inf
            cp      = self.cp      = self.R*self.kappa/(self.kappa-1.)
            cv      = self.cv      = self.cp/self.kappa                         
            r       = self.r       = self.Pr**(1/3)
            Tw      = self.Tw      = self.T_inf
            Taw     = self.Taw     = self.T_inf + self.r*self.U_inf**2/(2*self.cp)
            lchar   = self.lchar   = self.Re*self.nu_inf/self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf' , '%0.3f [kg/m³]'    % self.rho_inf )
            if verbose: even_print('mu_inf'  , '%0.6E [kg/(m·s)]' % self.mu_inf  )
            if verbose: even_print('nu_inf'  , '%0.6E [m²/s]'     % self.nu_inf  )
            if verbose: even_print('a_inf'   , '%0.6f [m/s]'      % self.a_inf   )
            if verbose: even_print('U_inf'   , '%0.6f [m/s]'      % self.U_inf   )
            if verbose: even_print('cp'      , '%0.3f [J/(kg·K)]' % self.cp      )
            if verbose: even_print('cv'      , '%0.3f [J/(kg·K)]' % self.cv      )
            if verbose: even_print('r'       , '%0.6f [-]'        % self.r       )
            if verbose: even_print('Tw'      , '%0.3f [K]'        % self.Tw      )
            if verbose: even_print('Taw'     , '%0.3f [K]'        % self.Taw     )
            if verbose: even_print('lchar'   , '%0.6E [m]'        % self.lchar   )
            if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the RGD instance
            udef_char_deriv = ['rho_inf', 'mu_inf', 'nu_inf', 'a_inf', 'U_inf', 'cp', 'cv', 'r', 'Tw', 'Taw', 'lchar']
            udef_real_deriv = [ rho_inf,   mu_inf,   nu_inf,   a_inf,   U_inf,   cp,   cv,   r,   Tw,   Taw,   lchar ]
            self.udef_deriv = dict(zip(udef_char_deriv, udef_real_deriv))
        
        else:
            pass
        
        # === time vector
        
        if ('dims/t' in self):
            self.t = np.copy(self['dims/t'][:])
            
            if ('data' in self): ## check t dim and data arrays agree
                if (nt!=self.t.size):
                    raise AssertionError('nt!=self.t.size : %i!=%i'%(nt,self.t.size))
            
            try:
                self.dt = self.t[1] - self.t[0]
            except IndexError:
                self.dt = 0.
            
            self.nt       = nt       = self.t.size
            self.duration = duration = self.t[-1] - self.t[0]
            self.ti       = ti       = np.array(range(self.nt), dtype=np.int64)
        else:
            self.t  = np.array([], dtype=np.float64)
            self.ti = np.array([], dtype=np.int64)
            self.nt = nt = 0
            self.dt = 0.
            self.duration = duration = 0.
        
        return
    
    def init_from_rgd(self, rgd_instance, **kwargs):
        '''
        initialize an LPD from an RGD (copy over header data & coordinate data)
        '''
        
        t_info = kwargs.get('t_info',True)
        
        verbose = kwargs.get('verbose',True)
        if (self.rank!=0):
            verbose=False
        
        #with rgd(fn_rgd, 'r', driver='mpio', comm=MPI.COMM_WORLD, libver='latest') as hf_ref:
        hf_ref = rgd_instance
        
        # === copy over header info if needed
        
        if all([('header/udef_real' in self),('header/udef_char' in self)]):
            raise ValueError('udef already present')
        else:
            udef         = hf_ref.udef
            udef_real    = list(udef.values())
            udef_char    = list(udef.keys())
            udef_real_h5 = np.array(udef_real, dtype=np.float64)
            udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
            
            self.create_dataset('header/udef_real', data=udef_real_h5, maxshape=np.shape(udef_real_h5), dtype=np.float64)
            self.create_dataset('header/udef_char', data=udef_char_h5, maxshape=np.shape(udef_char_h5), dtype='S128')
            self.udef      = udef
            self.udef_real = udef_real
            self.udef_char = udef_char
        
        # === copy over spatial dim info
        
        x, y, z = hf_ref.x, hf_ref.y, hf_ref.z
        nx  = self.nx  = x.size
        ny  = self.ny  = y.size
        nz  = self.nz  = z.size
        ngp = self.ngp = nx*ny*nz
        if ('dims/x' in self):
            del self['dims/x']
        if ('dims/y' in self):
            del self['dims/y']
        if ('dims/z' in self):
            del self['dims/z']
        
        self.create_dataset('dims/x', data=x)
        self.create_dataset('dims/y', data=y)
        self.create_dataset('dims/z', data=z)
        
        # === copy over temporal dim info
        
        if t_info:
            self.t  = hf_ref.t
            self.nt = self.t.size
            self.create_dataset('dims/t', data=hf_ref.t)
        else:
            t = np.array([0.], dtype=np.float64)
            if ('dims/t' in self):
                del self['dims/t']
            self.create_dataset('dims/t', data=t)
        
        self.get_header(verbose=False)
        return
    
    # ===
    
    def calc_acceleration(self,**kwargs):
        '''
        calculate velocity time derivatives (acceleration and jerk)
        '''
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'lpd.calc_acceleration()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        n_chunks = kwargs.get('n_chunks',1)
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        
        if verbose:
            even_print('fn_lpd',  self.fname)
            even_print('npts'  , '%i'%self.npts)
            even_print('nt'    , '%i'%self.nt)
            print(72*'-')
        
        ## particle list bounds this rank
        rpl_ = np.array_split(np.arange(self.npts, dtype=np.int64), self.n_ranks )
        rpl  = [[b[0],b[-1]+1] for b in rpl_ ]
        rp1, rp2 = rpl[self.rank]
        npr = rp2 - rp1
        
        ## for local rank bound, subdivide into chunks for collective reads
        cpl_ = np.array_split(np.arange(rp1, rp2, dtype=np.int64), n_chunks )
        cpl  = [[b[0],b[-1]+1] for b in cpl_ ]
        
        # === initialize (new) datasets
        
        ## shape & HDF5 chunk scheme for datasets
        shape = (self.npts, self.nt)
        chunks = h5_chunk_sizer(nxi=shape, constraint=(None,1), size_kb=chunk_kb, base=4, itemsize=4)
        
        scalars_in         = ['u','v','w','t','id']
        scalars_in_dtypes  = [self.scalars_dtypes_dict[s] for s in scalars_in]
        
        scalars_out        = ['ax','ay','az', 'jx','jy','jz']
        scalars_out_dtypes = [np.float32 for i in scalars_out]
        
        for scalar in scalars_out:
            
            if verbose:
                even_print('initializing',scalar)
            
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            dset = self.create_dataset('data/%s'%scalar, 
                                       shape=shape, 
                                       dtype=np.float32,
                                       fillvalue=np.nan,
                                       #compression='gzip', ## this causes segfaults :( :(
                                       #compression_opts=5,
                                       #shuffle=True,
                                       chunks=chunks,
                                       )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (pts,nt)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose:
            tqdm.write(72*'-')
        
        # ===
        
        if verbose: progress_bar = tqdm(total=npr, ncols=100, desc='calc accel', leave=False, file=sys.stdout)
        
        for ci in range(n_chunks):
            cp1, cp2 = cpl[ci]
            npc = cp2 - cp1
            
            ## local buffer array (for read)
            pdata_in = np.empty(shape=(npc,self.nt), dtype={'names':scalars_in, 'formats':scalars_in_dtypes})
            pdata_in[:,:] = np.nan
            
            ## local buffer array (for write)
            pdata_out = np.empty(shape=(npc,self.nt), dtype={'names':scalars_out, 'formats':scalars_out_dtypes})
            pdata_out[:,:] = np.nan
            
            ## read data
            if verbose: tqdm.write(even_print('chunk', '%i/%i'%(ci+1,n_chunks), s=True))
            #for scalar in self.scalars:
            for scalar in scalars_in:
                dset = self['data/%s'%scalar]
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi:
                    with dset.collective:
                        #pdata_in[scalar] = dset[rp1:rp2,:]
                        pdata_in[scalar] = dset[cp1:cp2,:]
                else:
                    pdata_in[scalar] = dset[cp1:cp2,:]
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                #data_gb = 4 * self.nt * self.npts / 1024**3
                data_gb = ( 4 * self.nt * self.npts / 1024**3 ) / n_chunks
                if verbose:
                    tqdm.write(even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            # === iterate over (chunk of) particle tracks
            
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            
            #for pi in range(npr): 
            for pi in range(npc): 
                
                u   = pdata_in['u'][pi,:]
                v   = pdata_in['v'][pi,:]
                w   = pdata_in['w'][pi,:]
                t   = pdata_in['t'][pi,:]
                pid = pdata_in['id'][pi,:]
                
                ii_real = np.where(~np.isnan(u))
                ii_nan  = np.where( np.isnan(u))
                n_real  = len(ii_real[0])
                n_nan   = len(ii_nan[0])
                
                if True: ## check non-NaNs have contiguous time indices (no NaN gaps) --> passes
                    aa = ii_real[0]
                    if (aa.shape[0]>1):
                        if not np.all( np.diff(aa) == np.diff(aa)[0] ):
                            #raise AssertionError('non-NaN index arr not const')
                            print('non-NaN index arr not const')
                            self.comm.Abort(1)
                
                if False: ## check non-NaN index vecs are same for every scalar --> passes
                    iiu = np.where(~np.isnan(u))
                    iiv = np.where(~np.isnan(v))
                    iiw = np.where(~np.isnan(w))
                    iit = np.where(~np.isnan(t))
                    
                    if not np.array_equal(iiu, iiv):
                        #raise AssertionError('real index vecs not same between scalars')
                        print('real index vecs not same between scalars')
                        self.comm.Abort(1)
                    if not np.array_equal(iiv, iiw):
                        print('real index vecs not same between scalars')
                        self.comm.Abort(1)
                    if not np.array_equal(iiw, iit):
                        print('real index vecs not same between scalars')
                        self.comm.Abort(1)
                
                ## take where scalar(t) vector isnt NaN
                u   = np.copy(   u[ii_real] )
                v   = np.copy(   v[ii_real] )
                w   = np.copy(   w[ii_real] )
                t   = np.copy(   t[ii_real] )
                pid = np.copy( pid[ii_real] )
                
                if False: ## check ID scalar is constant for this particle --> passes
                    if not np.all(pid == pid[0]):
                        #raise AssertionError('pt ID not same')
                        print('pt ID not same')
                        print(pid)
                        self.comm.Abort(1)
                
                if (n_real>=12): ## must have at least N timesteps with data
                    
                    if False: ## numpy O2 gradient
                        dudt   = np.gradient(u,    t, axis=0, edge_order=2)
                        dvdt   = np.gradient(v,    t, axis=0, edge_order=2)
                        dwdt   = np.gradient(w,    t, axis=0, edge_order=2)
                        d2udt2 = np.gradient(dudt, t, axis=0, edge_order=2)
                        d2vdt2 = np.gradient(dvdt, t, axis=0, edge_order=2)
                        d2wdt2 = np.gradient(dwdt, t, axis=0, edge_order=2)
                    
                    if False: ## O3 Cubic Spline
                        dudt   = sp.interpolate.CubicSpline(t,u,bc_type='natural')(t,1)
                        dvdt   = sp.interpolate.CubicSpline(t,v,bc_type='natural')(t,1)
                        dwdt   = sp.interpolate.CubicSpline(t,w,bc_type='natural')(t,1)
                        d2udt2 = sp.interpolate.CubicSpline(t,u,bc_type='natural')(t,2)
                        d2vdt2 = sp.interpolate.CubicSpline(t,v,bc_type='natural')(t,2)
                        d2wdt2 = sp.interpolate.CubicSpline(t,w,bc_type='natural')(t,2)
                    
                    if True: ## Finite Difference O6
                        
                        acc = 6 ## order of truncated term (error term) in FD formulation
                        
                        dudt   = gradient(u, t, d=1, axis=0, acc=acc, edge_stencil='full')
                        dvdt   = gradient(v, t, d=1, axis=0, acc=acc, edge_stencil='full')
                        dwdt   = gradient(w, t, d=1, axis=0, acc=acc, edge_stencil='full')
                        d2udt2 = gradient(u, t, d=2, axis=0, acc=acc, edge_stencil='full')
                        d2vdt2 = gradient(v, t, d=2, axis=0, acc=acc, edge_stencil='full')
                        d2wdt2 = gradient(w, t, d=2, axis=0, acc=acc, edge_stencil='full')
                    
                    ## write to buffer
                    pdata_out['ax'][pi,ii_real] = dudt  
                    pdata_out['ay'][pi,ii_real] = dvdt  
                    pdata_out['az'][pi,ii_real] = dwdt  
                    pdata_out['jx'][pi,ii_real] = d2udt2
                    pdata_out['jy'][pi,ii_real] = d2vdt2
                    pdata_out['jz'][pi,ii_real] = d2wdt2
                
                if verbose: progress_bar.update()
            
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            if verbose:
                tqdm.write(even_print('calc accel & jerk', '%0.2f [s]'%(t_delta,), s=True))
            
            # === write buffer out
            
            for scalar in scalars_out:
                
                dset = self['data/%s'%scalar]
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi:
                    with dset.collective:
                        dset[cp1:cp2,:] = pdata_out[scalar]
                else:
                    dset[cp1:cp2,:] = pdata_out[scalar]
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                data_gb = ( 4 * self.nt * self.npts / 1024**3 ) / n_chunks
                if verbose:
                    tqdm.write(even_print('write: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            # ===
            
            if verbose:
                tqdm.write(72*'-')
        
        if verbose: progress_bar.close()
        
        # ===
        
        if self.usingmpi: self.comm.Barrier()
        self.make_xdmf()
        
        if verbose: print('\n'+72*'-')
        if verbose: print('total time : lpd.calc_acceleration() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from LPD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        makeVectors = kwargs.get('makeVectors',True) ## write [u,v,w] and [vort_x,vort_y,vort_z] vectors to XDMF
        makeTensors = kwargs.get('makeTensors',True) ## write stress or strain tensors to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if verbose: print('\n'+'lpd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === refresh header
        self.get_header(verbose=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')
        
        # === make .xdmf/.xmf2 file
        
        if (self.rank==0):
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    xdmf_str=f'''
                             <Topology TopologyType="Polyvertex" NumberOfElements="{self.npts:d}"/>
                             <!-- === -->
                             <Geometry GeometryType="X_Y_Z">
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))

                    # ===
                    
                    xdmf_str=f'''
                              <DataItem ItemType="HyperSlab" Dimensions="{self.npts:d}" Type="HyperSlab">
                                <DataItem Dimensions="3 2" NumberType="Integer" Format="XML">
                                     0        {ti:d}
                                     1        1
                                     {self.npts:d}  1
                                </DataItem>
                                <DataItem Dimensions="{self.npts:d} {self.nt:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                                  {fname_base}:/data/x
                                </DataItem>
                              </DataItem>
                              <DataItem ItemType="HyperSlab" Dimensions="{self.npts:d}" Type="HyperSlab">
                                <DataItem Dimensions="3 2" NumberType="Integer" Format="XML">
                                     0        {ti:d}
                                     1        1
                                     {self.npts:d}  1
                                </DataItem>
                                <DataItem Dimensions="{self.npts:d} {self.nt:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                                  {fname_base}:/data/y
                                </DataItem>
                              </DataItem>
                              <DataItem ItemType="HyperSlab" Dimensions="{self.npts:d}" Type="HyperSlab">
                                <DataItem Dimensions="3 2" NumberType="Integer" Format="XML">
                                     0        {ti:d}
                                     1        1
                                     {self.npts:d}  1
                                </DataItem>
                                <DataItem Dimensions="{self.npts:d} {self.nt:d}" NumberType="{dataset_numbertype_dict['z']}" Precision="{dataset_precision_dict['z']:d}" Format="HDF">
                                  {fname_base}:/data/z
                                </DataItem>
                              </DataItem>
                            </Geometry>
                            <!-- === -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        
                        dset_hf_path = 'data/%s'%scalar
                        
                        xdmf_str=f'''
                                 <!-- ===== scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar}" AttributeType="Scalar" Center="Node">
                                       <DataItem ItemType="HyperSlab" Dimensions="{self.npts:d}" Type="HyperSlab">
                                         <DataItem Dimensions="3 2" NumberType="Integer" Format="XML">
                                              0        {ti:d}
                                              1        1
                                              {self.npts:d}  1
                                         </DataItem>
                                         <DataItem Dimensions="{self.npts:d} {self.nt:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                           {fname_base}:{dset_hf_path}
                                         </DataItem>
                                       </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : end Grid : </Grid>
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if verbose: print('--w-> %s'%fname_xdmf_base)
        return

class spd(h5py.File):
    '''
    Surface Polydata (SPD)
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.spd()')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## spd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 32    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8     )
        perms          = kwargs.pop('perms'          , '640' )
        
        if not isinstance(stripe_count, int):
            raise ValueError
        if not isinstance(stripe_size_mb, int):
            raise ValueError
        if not isinstance(perms, str):
            raise ValueError
        if not len(perms)==3:
            raise ValueError
        if not re.fullmatch(r'\d{3}',perms):
            raise ValueError
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using romio --> causes crash
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        
        ## set ROMIO hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(16*1024**2))) ) ## 16 [MB]
                ##
                #mpi_info.Set('romio_no_indep_rw' , 'true' ) ## deferred open + only collective I/O 
                #mpi_info.Set('cb_nodes' , str(int(round(1*self.n_ranks))) )
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        ## | rdcc_nbytes:
        ## | ------------
        ## | Integer setting the total size of the raw data chunk cache for this dataset in bytes.
        ## | In most cases increasing this number will improve performance, as long as you have 
        ## | enough free memory. The default size is 1 MB
        
        ## --> gets passed to H5Pset_chunk_cache
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        ## | rdcc_nslots:
        ## | ------------
        ## | Integer defining the number of chunk slots in the raw data chunk cache for this dataset.
        
        ## if ('rdcc_nslots' not in kwargs):
        ##     kwargs['rdcc_nslots'] = 521
        
        ## spd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with spd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(0.1)
                Path(self.fname).touch() ## touch a new file
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        self.mod_avail_tqdm = ('tqdm' in sys.modules)
        
        ## call actual h5py.File.__init__()
        super(spd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(spd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed SPD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(spd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of SPD class instance
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if (self.rank!=0):
            verbose=False
        
        # === udef (header vector dset based)
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## just make udef_real a dict with udef_char as keys
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.C_Suth      = self.udef['C_Suth']
            self.S_Suth      = self.udef['S_Suth']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            
            # === characteristic values : derived
            
            rho_inf   = self.rho_inf   = self.p_inf/(self.R * self.T_inf)
            mu_inf    = self.mu_inf    = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            nu_inf    = self.nu_inf    = self.mu_inf/self.rho_inf
            a_inf     = self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            U_inf     = self.U_inf     = self.Ma*self.a_inf
            cp        = self.cp        = self.R*self.kappa/(self.kappa-1.)
            cv        = self.cv        = self.cp/self.kappa
            recov_fac = self.recov_fac = self.Pr**(1/3)
            Tw        = self.Tw        = self.T_inf
            Taw       = self.Taw       = self.T_inf + self.r*self.U_inf**2/(2*self.cp)
            lchar     = self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Tw'              , '%0.3f [K]'        % self.Tw        )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            #if verbose: print(72*'-'+'\n')
            #if verbose: print(72*'-')
            
            # === write the 'derived' udef variables to a dict attribute of the ZTMD instance
            udef_char_deriv = ['rho_inf', 'mu_inf', 'nu_inf', 'a_inf', 'U_inf', 'cp', 'cv', 'recov_fac', 'Tw', 'Taw', 'lchar']
            udef_real_deriv = [ rho_inf,   mu_inf,   nu_inf,   a_inf,   U_inf,   cp,   cv,   recov_fac,   Tw,   Taw,   lchar ]
            self.udef_deriv = dict(zip(udef_char_deriv, udef_real_deriv))
        
        else:
            #print("dset 'header' not in SPD")
            pass
        
        # === udef (attr based)
        
        header_attr_str_list = ['Ma','Re','Pr','kappa','R','p_inf','T_inf','C_Suth','S_Suth','mu_Suth_ref','T_Suth_ref']
        if all([ attr_str in self.attrs.keys() for attr_str in header_attr_str_list ]):
            header_attr_based = True
        else:
            header_attr_based = False
        
        if header_attr_based:
            
            ## set all attributes
            for attr_str in header_attr_str_list:
                setattr( self, attr_str, self.attrs[attr_str] )
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            
            # === characteristic values : derived
            
            mu_inf_1 = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            mu_inf_2 = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2)*(self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth)
            mu_inf_3 = self.C_Suth*self.T_inf**(3/2)/(self.T_inf+self.S_Suth)
            
            rho_inf   = self.rho_inf   = self.p_inf/(self.R*self.T_inf)
            mu_inf    = self.mu_inf    = mu_inf_3
            nu_inf    = self.nu_inf    = self.mu_inf/self.rho_inf
            a_inf     = self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            U_inf     = self.U_inf     = self.Ma*self.a_inf
            cp        = self.cp        = self.R*self.kappa/(self.kappa-1.)
            cv        = self.cv        = self.cp/self.kappa
            recov_fac = self.recov_fac = self.Pr**(1/3)
            Tw        = self.Tw        = self.T_inf
            Taw       = self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            lchar     = self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Tw'              , '%0.3f [K]'        % self.Tw        )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            #if verbose: print(72*'-'+'\n')
            if verbose: print(72*'-')
            
            # === write the 'derived' udef variables to a dict attribute of the ZTMD instance
            udef_char_deriv = ['rho_inf', 'mu_inf', 'nu_inf', 'a_inf', 'U_inf', 'cp', 'cv', 'recov_fac', 'Tw', 'Taw', 'lchar']
            udef_real_deriv = [ rho_inf,   mu_inf,   nu_inf,   a_inf,   U_inf,   cp,   cv,   recov_fac,   Tw,   Taw,   lchar ]
            self.udef_deriv = dict(zip(udef_char_deriv, udef_real_deriv))
        
        #if ('duration_avg' in self.attrs.keys()):
        #    self.duration_avg = self.attrs['duration_avg']
        #if ('nx' in self.attrs.keys()):
        #    self.nx = self.attrs['nx']
        #if ('ny' in self.attrs.keys()):
        #    self.ny = self.attrs['ny']
        if ('p_inf' in self.attrs.keys()):
            self.p_inf = self.attrs['p_inf']
        if ('lchar' in self.attrs.keys()):
            self.lchar = self.attrs['lchar']
        if ('U_inf' in self.attrs.keys()):
            self.U_inf = self.attrs['U_inf']
        if ('Re' in self.attrs.keys()):
            self.Re = self.attrs['Re']
        
        if ('T_inf' in self.attrs.keys()):
            self.T_inf = self.attrs['T_inf']
        if ('rho_inf' in self.attrs.keys()):
            self.rho_inf = self.attrs['rho_inf']
        
        if ('dims/xyz' in self):
            self.xyz = np.copy( self['dims/xyz'][()] )
        if ('dims/stang' in self):
            self.stang = np.copy( self['dims/stang'][()] )
        if ('dims/snorm' in self):
            self.snorm = np.copy( self['dims/snorm'][()] )
        if ('dims/crv_R' in self):
            self.crv_R = np.copy( self['dims/crv_R'][()] )
        
        if ('n_quads' in self.attrs.keys()):
            self.n_quads = self.attrs['n_quads']
        if ('n_pts' in self.attrs.keys()):
            self.n_pts = self.attrs['n_pts']
        if ('ni' in self.attrs.keys()):
            self.ni = self.attrs['ni']
        if ('nj' in self.attrs.keys()):
            self.nj = self.attrs['nj']
        
        if ('nt' in self.attrs.keys()):
            self.nt = self.attrs['nt']
        
        if ('dims/t' in self):
            self.t = t = np.copy( self['dims/t'][()] )
        if hasattr(self,'t'):
            if (self.t.ndim!=1):
                raise ValueError('self.t.ndim!=1')
            nt = self.t.shape[0]
            
            if hasattr(self,'nt'):
                if not isinstance(self.nt, (int,np.int32,np.int64)):
                    raise TypeError('self.nt is not type int')
                if (self.nt != nt):
                    raise ValueError('self.nt != nt')
            else:
                #self.attrs['nt'] = nt
                self.nt = nt
        
        ## check n_quads / n_pts is consistent with xyz
        ## if xyz exists and attrs n_quads/n_pts do not exist, set them
        if hasattr(self,'xyz'):
            if (self.xyz.ndim!=3):
                raise ValueError('self.xyz.ndim!=3')
            ni,nj,three = self.xyz.shape
            
            if hasattr(self,'ni'):
                if not isinstance(self.ni, (int,np.int32,np.int64)):
                    raise TypeError('self.ni is not type int')
                if (self.ni != ni):
                    raise ValueError('self.ni != ni')
            else:
                #self.attrs['ni'] = ni
                self.ni = ni
            
            if hasattr(self,'nj'):
                if not isinstance(self.nj, (int,np.int32,np.int64)):
                    raise TypeError('self.nj is not type int')
                if (self.nj != nj):
                    raise ValueError('self.nj != nj')
            else:
                #self.attrs['nj'] = nj
                self.nj = nj
            
            if hasattr(self,'n_quads'):
                if not isinstance(self.n_quads, (int,np.int32,np.int64)):
                    raise TypeError('self.n_quads is not type int')
                if (self.n_quads != (ni-1)*(nj-1)):
                    raise ValueError('self.n_quads != (ni-1)*(nj-1)')
            else:
                #self.attrs['n_quads'] = (ni-1)*(nj-1)
                self.n_quads = (ni-1)*(nj-1)
            
            if hasattr(self,'n_pts'):
                if not isinstance(self.n_pts, (int,np.int32,np.int64)):
                    raise TypeError('self.n_pts is not type int')
                if (self.n_pts != ni*nj):
                    raise ValueError('self.n_pts != ni*nj')
            else:
                #self.attrs['n_pts'] = ni*nj
                self.n_pts = ni*nj
        
        if any([hasattr(self,'ni'), hasattr(self,'nj'), hasattr(self,'n_quads'), hasattr(self,'n_pts') ]):
            if verbose and hasattr(self,'nt'):      even_print('nt', '%i'%self.nt)
            if verbose and hasattr(self,'ni'):      even_print('ni', '%i'%self.ni)
            if verbose and hasattr(self,'nj'):      even_print('nj', '%i'%self.nj)
            if verbose and hasattr(self,'n_quads'): even_print('n_quads', '%i'%self.n_quads)
            if verbose and hasattr(self,'n_pts'):   even_print('n_pts', '%i'%self.n_pts)
            #if verbose: print(72*'-')
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys())
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self['data/%s'%scalar].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    def make_unstruct(self,**kwargs):
        '''
        convert structured datasets (data/<scalar>) to unstructured (data_unstruct/<scalar>)
        /dims_unstruct/quads
        /dims_unstruct/pts
        /data_unstruct/<scalar>
        --> currently only parallelized over [t]
        --> parallelization over [i] and [j] is probably a bit more tricky due to reshaping
        '''
        
        verbose  = kwargs.get( 'verbose'  , True )
        indexing = kwargs.get( 'xy'       , 'xy' ) ## 'xy', 'ij'
        chunk_kb = kwargs.get( 'chunk_kb' , 8*1024 ) ## 8 [MB]
        
        ##chunk_constraint = kwargs.get('chunk_constraint',(None,None))
        chunk_base       = kwargs.get('chunk_base',2)
        
        ri = kwargs.get('ri',1) ## should be =1
        rj = kwargs.get('rj',1) ## should be =1
        rt = kwargs.get('rt',1)
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        if 'dims/xyz' not in self:
            raise ValueError('dims/xyz not in file')
        
        if verbose: print('\n'+'spd.make_unstruct()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if (ri*rj*rt != self.n_ranks):
            raise AssertionError('ri*rj*rt != n_ranks')
        if (ri!=1):
            raise AssertionError('ri!=1')
        if (rj!=1):
            raise AssertionError('rj!=1')
        #if (rt>self.nt):
        #    raise AssertionError('rt>self.nt')
        
        #if verbose: even_print('ri','%i'%ri)
        #if verbose: even_print('rj','%i'%rj)
        if verbose: even_print('rt','%i'%rt)
        #if verbose: print(72*'-')
        
        # ===
        
        rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        ## if rt>nt, assign null range
        try:
            rt1,rt2 = rtl[self.rank]
        except IndexError:
            rt1,rt2 = 0,0
        ntr = rt2 - rt1
        
        # ===
        
        ## every rank gets grid
        xyz = np.copy( self['dims/xyz'][()] )
        if verbose:
            even_print('dims/xyz',str(xyz.shape))
        
        if (xyz.ndim!=3):
            raise ValueError('xyz.ndim!=3')
        
        ni,nj,three = xyz.shape
        
        n_quads = (ni-1)*(nj-1)
        n_pts   = ni*nj
        nt      = self.nt
        
        if verbose:
            even_print('ni'      , '%i'%(ni,)      )
            even_print('nj'      , '%i'%(nj,)      )
            even_print('n_quads' , '%i'%(n_quads,) )
            even_print('n_pts'   , '%i'%(n_pts,)   )
            even_print('nt'      , '%i'%(nt,)      )
        
        # ===
        
        xi, yi = np.meshgrid(np.arange(ni,dtype=np.int64), np.arange(nj,dtype=np.int64), indexing=indexing)
        
        inds_list = np.stack((xi,yi), axis=2)
        inds_list = np.reshape(inds_list, (ni*nj,2), order='C')
        inds_list = np.ravel_multi_index((inds_list[:,0],inds_list[:,1]), (ni,nj), order='F')
        inds_list = np.reshape(inds_list, (ni,nj), order='C')
        
        if verbose: progress_bar = tqdm(total=(ni-1)*(nj-1), ncols=100, desc='quads', leave=False, file=sys.stdout)
        
        ## quad index array
        quads = np.zeros(((ni-1),(nj-1),4), dtype=np.int64)
        for i in range(ni-1):
            for j in range(nj-1):
                
                ## Counter-Clockwise (CCW)
                quads[i,j,0] = inds_list[i,   j  ]
                quads[i,j,1] = inds_list[i+1, j  ]
                quads[i,j,2] = inds_list[i+1, j+1]
                quads[i,j,3] = inds_list[i,   j+1]
                
                ## Clockwise (CW)
                #quads[i,j,0] = inds_list[i,   j  ]
                #quads[i,j,1] = inds_list[i,   j+1]
                #quads[i,j,2] = inds_list[i+1, j+1]
                #quads[i,j,3] = inds_list[i+1, j  ]
                
                if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # ===
        
        if (indexing=='xy'):
            order='C'
        elif (indexing=='ij'):
            order = 'F'
        else:
            raise ValueError
        
        # ===
        
        ## flatten quad index vector
        quads = np.reshape(quads, ((ni-1)*(nj-1),4), order=order)
        
        dsn = 'dims_unstruct/quads'
        if (dsn in self):
            del self[dsn]
        
        shape = quads.shape
        dtype = quads.dtype
        itemsize = quads.dtype.itemsize
        chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None), size_kb=chunk_kb, base=chunk_base, itemsize=itemsize)
        ds = self.create_dataset(dsn,
                                 shape=shape,
                                 chunks=chunks,
                                 dtype=dtype )
        
        chunk_kb_ = np.prod(ds.chunks)*itemsize / 1024. ## actual
        if verbose:
            even_print('chunk shape (n_quads,4)','%s'%str(ds.chunks))
            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        if self.usingmpi:
            with ds.collective:
                ds[:,:] = quads
        else:
            ds[:,:] = quads
        
        # ===
        
        ## flatten point coordinate vector
        pts = np.reshape(xyz, (ni*nj,3), order=order)
        
        dsn = 'dims_unstruct/pts'
        if (dsn in self):
            del self[dsn]
        
        shape = pts.shape
        dtype = pts.dtype
        itemsize = pts.dtype.itemsize
        chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None), size_kb=chunk_kb, base=chunk_base, itemsize=itemsize)
        ds = self.create_dataset(dsn,
                                 shape=shape,
                                 chunks=chunks,
                                 dtype=dtype )
        
        chunk_kb_ = np.prod(ds.chunks)*itemsize / 1024. ## actual
        if verbose:
            even_print('chunk shape (n_pts,3)','%s'%str(ds.chunks))
            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        if self.usingmpi:
            with ds.collective:
                ds[:,:] = pts
        else:
            ds[:,:] = pts
        
        # ===
        
        ## initialize all data_unstruct/<scalar> datasets
        for scalar in self.scalars:
            
            dsn = f'data/{scalar}'
            dtype = self[dsn].dtype
            itemsize = pts.dtype.itemsize
            
            ## initialize data_unstruct/<scalar> datasets
            shape = (ni*nj,nt)
            chunks = h5_chunk_sizer(nxi=shape, constraint=(None,1), size_kb=chunk_kb, base=chunk_base, itemsize=itemsize)
            
            dsn = f'data_unstruct/{scalar}'
            if (dsn in self):
                del self[dsn]
            
            ds = self.create_dataset(dsn,
                                     shape=shape,
                                     chunks=chunks,
                                     dtype=dtype,
                                     )
            
            # chunk_kb_ = np.prod(ds.chunks)*itemsize / 1024. ## actual
            # if verbose:
            #     even_print('chunk shape (n_pts,3)','%s'%str(ds.chunks))
            #     even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        # ===
        
        ## copy scalar data, reshape, write
        if verbose: progress_bar = tqdm(total=self.n_scalars*ntr, ncols=100, desc='reshape', leave=False, file=sys.stdout)
        for scalar in self.scalars:
            
            dsn = f'data_unstruct/{scalar}'
            ds = self[dsn]
            #if verbose: even_print(dsn,str(ds.shape))
            
            #for ti in range(nt):
            for ti in range(rt1,rt2):
                
                ## get structured scalar data
                data = np.copy( self[f'data/{scalar}'][:,:,ti] )
                #_,_,one = data.shape
                
                ## reshape / flatten scalar data
                #data = np.reshape(data, (ni*nj,nt), order=order)
                data = np.reshape(data, (ni*nj,), order=order)
                
                ## write reshaped scalar data
                ds[:,ti] = data
                
                if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: print('total time : spd.make_unstruct() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    @staticmethod
    def copy(fn_spd_src, fn_spd_tgt, **kwargs):
        '''
        copy header info, selected scalars, and [i,j,t] range to new SPD file
        - currently copies complete [i,j] range
        - if [i,j] range clipping were to be implemented, taking data_unstruct would be difficult
        --> this currently does NOT work in serial mode
        '''
        
        #comm    = MPI.COMM_WORLD
        rank    = MPI.COMM_WORLD.Get_rank()
        n_ranks = MPI.COMM_WORLD.Get_size()
        
        if (rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'spd.copy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx       = kwargs.get('rx',None)
        ry       = kwargs.get('ry',None)
        rz       = kwargs.get('rz',None)
        
        ri       = kwargs.get('ri',1)
        rj       = kwargs.get('rj',1)
        
        rt       = kwargs.get('rt',1)
        force    = kwargs.get('force',False) ## overwrite or raise error if exists
        
        ti_min   = kwargs.get('ti_min',None)
        ti_max   = kwargs.get('ti_max',None)
        scalars  = kwargs.get('scalars',None)
        
        i_min  = kwargs.get( 'i_min'  , None )
        i_max  = kwargs.get( 'i_max'  , None )
        j_min  = kwargs.get( 'j_min'  , None )
        j_max  = kwargs.get( 'j_max'  , None )
        ti_min = kwargs.get( 'ti_min' , None )
        ti_max = kwargs.get( 'ti_max' , None )
        
        ct = kwargs.get('ct',1) ## 'chunks' in time
        
        chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(None,None,1)) ## the 'constraint' parameter for sizing h5 chunks (i,j,t)
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 32 ) ## for initializing SPD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8  )
        
        xi_step = kwargs.get('xi_step',1)
        yi_step = kwargs.get('yi_step',1)
        zi_step = kwargs.get('zi_step',1)
        
        if (rx is not None):
            raise ValueError('rx not a valid option for spd.copy(). accepted are: ri,rj')
        if (ry is not None):
            raise ValueError('ry not a valid option for spd.copy(). accepted are: ri,rj')
        if (rz is not None):
            raise ValueError('rz not a valid option for spd.copy(). accepted are: ri,rj')
        
        if (i_min is not None):
            raise NotImplementedError('i/j_min/max not yet supported')
        if (i_max is not None):
            raise NotImplementedError('i/j_min/max not yet supported')
        if (j_min is not None):
            raise NotImplementedError('i/j_min/max not yet supported')
        if (j_max is not None):
            raise NotImplementedError('i/j_min/max not yet supported')
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (ri*rj!=n_ranks):
            raise AssertionError('ri*rj!=n_ranks')
        if not os.path.isfile(fn_spd_src):
            raise FileNotFoundError('%s not found!'%fn_spd_src)
        if os.path.isfile(fn_spd_tgt) and not force:
            raise FileExistsError('%s already exists. delete it or use \'force=True\' kwarg'%fn_spd_tgt)
        
        # ===
        
        with spd(fn_spd_src, 'r', comm=MPI.COMM_WORLD, driver='mpio') as hf_src:
            with spd(fn_spd_tgt, 'w',
                     force=force,
                     comm=MPI.COMM_WORLD,
                     driver='mpio',
                     stripe_count=stripe_count,
                     stripe_size_mb=stripe_size_mb) as hf_tgt:
                
                ni      = hf_src.ni
                nj      = hf_src.nj
                n_quads = hf_src.n_quads
                n_pts   = hf_src.n_pts
                nt      = hf_src.nt
                
                ## report info from source file
                fsize = os.path.getsize(hf_src.fname)/1024**3
                if verbose: even_print(os.path.basename(hf_src.fname),'%0.1f [GB]'%fsize)
                if verbose: even_print('ni','%i'%hf_src.ni)
                if verbose: even_print('nj','%i'%hf_src.nj)
                if verbose: even_print('nt','%i'%hf_src.nt)
                if verbose: even_print('n_quads','%i'%hf_src.n_quads)
                if verbose: even_print('n_pts','%i'%hf_src.n_pts)
                if verbose: print(72*'-')
                
                # ===
                
                ## get OUTPUT times
                t_  = np.copy(hf_src['dims/t'][()])
                ti_ = np.arange(t_.shape[0], dtype=np.int32)
                if (ti_min is None):
                    ti_min = ti_.min()
                if (ti_max is None):
                    ti_max = ti_.max()
                ti  = np.copy(ti_[ti_min:ti_max+1])
                if (ti.shape[0]==0):
                    raise ValueError('ti_min/ti_max combo yields no times')
                ti1 = ti.min()
                ti2 = ti.max()+1
                t   = np.copy(t_[ti1:ti2])
                nt  = t.shape[0]
                
                if (ti_min<0):
                    if verbose: even_print('ti_min', f'{ti_min:d} / {ti1:d}')
                else:
                    if verbose: even_print('ti_min', f'{ti_min:d}')
                
                if (ti_max<0):
                    if verbose: even_print('ti_max', f'{ti_max:d} / {ti2:d}')
                else:
                    if verbose: even_print('ti_max', f'{ti_max:d}')
                
                if verbose: even_print('t range', f'{ti.shape[0]:d}/{ti_.shape[0]:d}')
                
                # ===
                
                ## time chunk ranges
                if (ct>nt):
                    raise ValueError('ct>nt')
                
                tfi = np.arange(ti1,ti2,dtype=np.int64)
                ctl_ = np.array_split(tfi,ct)
                ctl = [[b[0],b[-1]+1] for b in ctl_ ]
                
                if verbose: print(72*'-')
                
                # ===
                
                ## copy over attributes
                for key,val in hf_src.attrs.items():
                    hf_tgt.attrs[key] = val
                
                # === get rank distribution over (i,j) dims
                
                comm2d = hf_src.comm.Create_cart(dims=[ri,rj], periods=[False,False], reorder=False)
                t2d = comm2d.Get_coords(rank)
                
                ril_ = np.array_split(np.arange(hf_src.ni,dtype=np.int64),ri)
                rjl_ = np.array_split(np.arange(hf_src.nj,dtype=np.int64),rj)
                
                ril = [[b[0],b[-1]+1] for b in ril_ ]
                rjl = [[b[0],b[-1]+1] for b in rjl_ ]
                
                ri1, ri2 = ril[t2d[0]]; nir = ri2 - ri1
                rj1, rj2 = rjl[t2d[1]]; njr = rj2 - rj1
                
                # === copy over non-attribute metadata
                
                ## 'dims/xyz' : 3D polydata grid coordinates : shape=(ni,nj,3)
                dsn = f'dims/xyz'
                dset = hf_src[dsn]
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                with dset.collective:
                    xyz = np.copy( dset[ri1:ri2,rj1:rj2,:] )
                shape  = (ni,nj,3)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,3), size_kb=chunk_kb, base=4, itemsize=float_bytes)
                data_gb = float_bytes * ni * nj / 1024**3
                if verbose:
                    even_print(f'initializing {dsn}','%0.1f [GB]'%(data_gb,))
                dset = hf_tgt.create_dataset(dsn, dtype=xyz.dtype, shape=shape, chunks=chunks)
                chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                if verbose:
                    even_print('chunk shape (i,j,3)',str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                with dset.collective:
                    dset[ri1:ri2,rj1:rj2,:] = xyz
                
                if verbose: print(72*'-')
                
                ## copy over [t]
                dsn = f'dims/t'
                ds = hf_tgt.create_dataset(dsn, chunks=None, data=t)
                if verbose: even_print(dsn,str(ds.shape))
                hf_tgt.attrs['nt'] = t.shape[0]
                
                ## copy over additional [dims/<>] dsets
                for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                    if (dsn in hf_src):
                        data = np.copy(hf_src[dsn][()])
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,str(ds.shape))
                    else:
                        if verbose: even_print(dsn,'not found')
                
                ## copy over additional [csys/<>] dsets
                for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                    if (dsn in hf_src):
                        data = np.copy(hf_src[dsn][()])
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,str(ds.shape))
                    else:
                        if verbose: even_print(dsn,'not found')
                
                if verbose: print(72*'-')
                hf_tgt.get_header(verbose=verbose)
                if verbose: print(72*'-')
                
                # === initialize datasets in target file
                
                for scalar in hf_src.scalars:
                    
                    dsn = f'data/{scalar}'
                    ds = hf_src[dsn]
                    dtype = ds.dtype
                    float_bytes = dtype.itemsize
                    
                    data_gb = ni * nj * nt * float_bytes / 1024**3
                    shape   = (ni,nj,nt)
                    chunks  = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                    
                    if verbose:
                        even_print(f'initializing data/{scalar}','%0.1f [GB]'%(data_gb,))
                    if (dsn in hf_tgt):
                        del hf_tgt[dsn]
                    dset = hf_tgt.create_dataset(dsn,
                                                 shape=shape,
                                                 dtype=dtype,
                                                 chunks=chunks )
                    
                    chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                    if verbose:
                        even_print('chunk shape (i,j,t)','%s'%str(dset.chunks))
                        even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if verbose: print(72*'-')
                
                # === main loop
                
                data_gb_read  = 0.
                data_gb_write = 0.
                t_read  = 0.
                t_write = 0.
                
                if verbose:
                    progress_bar = tqdm(total=len(ctl)*hf_src.n_scalars, ncols=100, desc='copy', leave=False, file=sys.stdout)
                
                for scalar in hf_src.scalars:
                    dset_src = hf_src[f'data/{scalar}']
                    dset_tgt = hf_tgt[f'data/{scalar}']
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    for ctl_ in ctl:
                        ct1, ct2 = ctl_
                        ntc = ct2 - ct1
                        
                        ## read
                        hf_src.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_src.collective:
                            data = np.copy( dset_src[ri1:ri2,rj1:rj2,ct1:ct2] )
                        hf_src.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * ni * nj * ntc / 1024**3
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        if verbose:
                            tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        ## write
                        hf_tgt.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_tgt.collective:
                            dset_tgt[ri1:ri2,rj1:rj2,:] = data
                        hf_tgt.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * ni * nj * ntc / 1024**3
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            tqdm.write(even_print(f'write: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        if verbose: progress_bar.update()
                
                if verbose: progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: print('total time : spd.copy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from SPD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        #makeVectors = kwargs.get('makeVectors',True) ## write vectors (e.g. velocity, vorticity) to XDMF
        #makeTensors = kwargs.get('makeTensors',True) ## write 3x3 tensors (e.g. stress, strain) to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if 'dims_unstruct/quads' not in self:
            raise ValueError('dims_unstruct/quads not in file')
        if 'dims_unstruct/pts' not in self:
            raise ValueError('dims_unstruct/pts not in file')
        
        ## this should be added to spd.get_header()
        dsn = 'dims_unstruct/quads'
        n_quads,four = self[dsn].shape
        dsn = 'dims_unstruct/pts'
        n_pts,three = self[dsn].shape
        self.n_quads = n_quads
        self.n_pts   = n_pts
        
        if verbose: print('\n'+'spd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for dsn_unstruct in ['pts','quads']:
            if (f'dims_unstruct/{dsn_unstruct}' in self):
                data = self[f'dims_unstruct/{dsn_unstruct}']
                dataset_precision_dict[dsn_unstruct] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[dsn_unstruct] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[dsn_unstruct] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        ## refresh header
        self.get_header(verbose=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')

        # === write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                xdmf_str=f'''
                         <Topology TopologyType="Quadrilateral" NumberOfElements="{n_quads:d}">
                             <DataItem Dimensions="{n_quads:d} 4" NumberType="{dataset_numbertype_dict['quads']}" Precision="{dataset_precision_dict['quads']}" Format="HDF">
                                 {self.fname_base}:/dims_unstruct/quads
                             </DataItem>
                         </Topology>
                         <Geometry GeometryType="XYZ">
                             <DataItem Dimensions="{n_pts:d} 3" NumberType="{dataset_numbertype_dict['pts']}" Precision="{dataset_precision_dict['pts']}" Format="HDF">
                                 {self.fname_base}:/dims_unstruct/pts
                             </DataItem>
                         </Geometry>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # === .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        
                        dset_hf_path = f'data_unstruct/{scalar}'
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        #if scalar in scalar_names:
                        if False:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        xdmf_str=f'''
                                 <!-- ===== scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem ItemType="HyperSlab" Dimensions="{self.n_pts:d}" Type="HyperSlab">
                                     <DataItem Dimensions="3 2" NumberType="Integer" Format="XML">
                                       {0:<9d} {ti:<9d}
                                       {1:<9d} {1:<9d}
                                       {self.n_pts:<9d} {1:<9d}
                                     </DataItem>
                                     <DataItem Dimensions="{self.n_pts:d} {self.nt:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                       {fname_base}:/{dset_hf_path}
                                     </DataItem>
                                   </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if verbose: print('--w-> %s'%fname_xdmf_base)
        
        return

# meta HDF5 & h5py
# ======================================================================

def h5_chunk_sizer(nxi, **kwargs):
    '''
    solve for HDF5 dataset chunk size
    -----
    nxi --> an iterable (tuple,list,etc.) containing the target shape of the full HDF5 dataset
    constraint --> an iterable with arguments <int> , None , 'full'/-1
    -----
    constraint = (1,None,None,None) ## [t] convention
    constraint = (None,-1,1,-1) ## [y] convention
    '''
    
    size_kb    = kwargs.get('size_kb'    , 2*1024 ) ## target chunk size in [KB] --> default = 2 [MB]
    itemsize   = kwargs.get('itemsize'   , 4      ) ## dtype.itemsize --> default single precision i.e. 4 [B]
    constraint = kwargs.get('constraint' , None   ) ## iterable of nxi constraints --> int,None,'full'/-1
    base       = kwargs.get('base'       , 2      ) ## axis chunk size = ceil[size/(<int>*base)] where <int> is incremented
    
    ## check inputs
    if not hasattr(nxi, '__iter__'):
        raise AssertionError('\'nxi\' must be an iterable')
    if constraint is None:
        constraint = [ None for i in range(len(nxi)) ]
    if not hasattr(constraint, '__iter__'):
        raise AssertionError('\'constraint\' must be an iterable')
    if not (len(nxi)==len(constraint)):
        raise ValueError('nxi and constraint must have same size')
    if not isinstance(base, int):
        raise TypeError('\'base\' must be type int')
    
    if True: ## increment divisor on largest axis, with divisor=<int>*base
        
        nxi_ = [ n for n in nxi ] ## copy the input shape
        div = [ 1 for i in range(len(nxi)) ] ## divisor vector, initialize with int ones
        i_flexible = [ i for i in range(len(nxi)) if constraint[i] is None ] ## list of axes indices which are 'flexible'
        
        while True:
            
            #div_last = div ## no! pointer, not a copy
            div_last = [ div[i] for i in range(len(nxi)) ]
            
            chunks = []
            for i in range(len(nxi)):
                if (constraint[i] is None): ## axis is 'flexible', so divide by divisor
                    cas = max( int(round(nxi_[i]/div[i])) , 1 ) ## chunk axis shape is solved for
                elif (constraint[i] == 'full') or (constraint[i] == -1):
                    cas = nxi_[i] ## chunk axis shape is == dset axis shape
                else:
                    
                    ## this axis is neither 'flexible' nor 'full', so
                    ## assert that the constraint is a positive integer
                    if not isinstance(constraint[i], int):
                        raise ValueError('constraint[%i]=%s is not type int'%(i,str(constraint[i])))
                    if not (constraint[i]>0):
                        raise ValueError('constraint[%i]=%i is not >0'%(i,constraint[i]))
                    
                    cas = constraint[i] ## chunk axis shape is just the constraint
                
                chunks.append(cas)
            
            ## there are no flexible axes --> exit loop
            if len(i_flexible)==0:
                break
            
            ## the current size of a chunk, given 
            chunk_size_kb = np.prod(chunks)*itemsize / 1024.
            # print('chunk size %0.1f [KB]'%chunk_size_kb)
            
            if ( chunk_size_kb <= size_kb ): ## if chunk size is < target, then break
                break
            else: ## otherwise, increase the divisor of the greatest 'flexible' axis
                
                ## get index of (flexible) axis with greatest size
                aa = [ i for i, j in enumerate(chunks) if (constraint[i] is None) ]
                bb = [ j for i, j in enumerate(chunks) if (constraint[i] is None) ]
                i_gt = aa[np.argmax(bb)]
                
                ## get new divisor
                if (div[i_gt]==1) and (base!=1):
                    div[i_gt] = 0
                div[i_gt] += base
            
            # print(f'div = {str(tuple(div))}')
            # print(f'div_last = {str(tuple(div_last))}')
            # print(f'chunks = {str(tuple(chunks))}')
            # print('---')
            
            ## check if in infinite loop (divisor not being updated)
            if (div_last is not None) and (div == div_last):
                raise ValueError('invalid parameters for h5_chunk_sizer() : constraint=%s, size_kb=%i, base=%i'%(str(constraint),size_kb,base))
    
    return tuple(chunks)

def h5_visititems_print_attrs(name, obj):
    '''
    callable for input to h5py.Group.visititems() to print names & attributes
    '''
    n_slashes = name.count('/')
    shift = n_slashes*2*' '
    item_name = name.split('/')[-1]
    
    if isinstance(obj,h5py._hl.dataset.Dataset):
        print(shift + item_name + ' --> shape=%s, dtype=%s'%( str(obj.shape), str(obj.dtype) ) )
    else:
        print(shift + item_name)
    
    ## print attributes
    for key, val in obj.attrs.items():
        try:
            print(shift + 2*' ' + f'{key} = {str(val)} --> dtype={str(val.dtype)}')
        except:
            print(shift + 2*' ' + f'{key} = {str(val)} --> type={str(type(val).__name__)}')

class h5_visit_container:
    '''
    callable for input to h5py.Group.visit() which stores dataset/group names
    '''
    def __init__(self):
        self.names = []
    def __call__(self, name):
        if (name not in self.names):
            self.names.append(name)

# data container interface class for EAS3 (legacy NS3D format)
# ======================================================================

class eas3:
    '''
    Interface class for EAS3 files (legacy binary NS3D output format)
    '''
    
    def __init__(self, fname, **kwargs):
        '''
        initialize class instance
        '''
        self.fname   = fname
        self.verbose = kwargs.get('verbose',True)
        
        if isinstance(fname, str):
            self.f = open(fname,'rb')
        elif isinstance(fname, io.BytesIO):
            self.f = fname
        else:
            raise TypeError('fname should be type str or io.BytesIO')
        
        self.udef    = self.get_header()
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #print('opening from enter() --> used with statement')
        return self
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        self.f.close()
        
        if exception_type is not None:
            print('\nsafely closed EAS3 due to exception')
            print(72*'-')
            print('exception type : '+str(exception_type))
        if exception_value is not None:
            print('exception_value : '+str(exception_value))
        if exception_traceback is not None:
            print('exception_traceback : '+str(exception_traceback))
        if exception_type is not None:
            print(72*'-')
    
    def close(self):
        '''
        close() passthrough to HDF5
        '''
        self.f.close()
    
    def get_header(self, **kwargs):
        
        ATTRLEN     = kwargs.get('ATTRLEN',10)
        UDEFLEN     = kwargs.get('UDEFLEN',20)
        ChangeGmode = kwargs.get('ChangeGmode',1)
        
        ## Definitions
        EAS2=1; EAS3=2
        IEEES=1; IEEED=2; IEEEQ=3
        EAS3_NO_ATTR=1; EAS3_ALL_ATTR=2
        EAS3_NO_G=1; EAS3_X0DX_G=2; EAS3_UDEF_G=3; EAS3_ALL_G=4; EAS3_FULL_G=5
        EAS3_NO_UDEF=1; EAS3_ALL_UDEF=2; EAS3_INT_UDEF=3
        
        self.IEEES = IEEES
        self.IEEED = IEEED
        self.IEEEQ = IEEEQ
        
        ## Identifier: 20 byte character
        identifier = self.f.read(20).strip()
        
        ## File type: 8 byte integer
        file_type = struct.unpack('!q',self.f.read(8))[0]
        
        ## Accuracy: 8 byte integer
        accuracy = struct.unpack('!q',self.f.read(8))[0]
        self.accuracy = accuracy
        
        if (self.accuracy == self.IEEES):
            self.dtype = np.float32
        elif (self.accuracy == self.IEEED):
            self.dtype = np.float64
        elif (self.accuracy == self.IEEEQ):
            self.dtype = np.float128
        else:
            raise ValueError('precision not identifiable')
        
        ## Array sizes: each 8 byte integer
        nzs   = struct.unpack('!q',self.f.read(8))[0]
        npar  = struct.unpack('!q',self.f.read(8))[0]
        ndim1 = struct.unpack('!q',self.f.read(8))[0]
        ndim2 = struct.unpack('!q',self.f.read(8))[0]
        ndim3 = struct.unpack('!q',self.f.read(8))[0]
        
        self.ndim1 = ndim1
        self.ndim2 = ndim2
        self.ndim3 = ndim3
        self.npar  = npar
        self.nzs   = nzs
        
        ## Attribute mode: 8 byte integer
        attribute_mode = struct.unpack('!q',self.f.read(8))[0]
        
        ## Geometry mode: each 8 byte integer
        gmode_time  = struct.unpack('!q',self.f.read(8))[0]
        gmode_param = struct.unpack('!q',self.f.read(8))[0]
        gmode_dim1  = struct.unpack('!q',self.f.read(8))[0]  
        gmode_dim2  = struct.unpack('!q',self.f.read(8))[0]
        gmode_dim3  = struct.unpack('!q',self.f.read(8))[0]
        
        ## Array sizes for geometry data: each 8 byte integer
        size_time  = struct.unpack('!q',self.f.read(8))[0]
        size_param = struct.unpack('!q',self.f.read(8))[0]
        size_dim1  = struct.unpack('!q',self.f.read(8))[0]
        size_dim2  = struct.unpack('!q',self.f.read(8))[0]
        size_dim3  = struct.unpack('!q',self.f.read(8))[0]
        
        ## Specification of user defined data: 8 byte integer
        udef = struct.unpack('!q',self.f.read(8))[0]
        
        ## Array sizes for used defined data: each 8 byte integer
        udef_char_size = struct.unpack('!q',self.f.read(8))[0]
        udef_int_size  = struct.unpack('!q',self.f.read(8))[0]
        udef_real_size = struct.unpack('!q',self.f.read(8))[0]
        
        ## Time step array: nzs x 8 byte
        time_step = np.zeros(nzs,int)
        for it in range(nzs):
            time_step[it] = struct.unpack('!q',self.f.read(8))[0]
        
        if attribute_mode==EAS3_ALL_ATTR:
            ## Time step attributes
            attr_time = [ self.f.read(ATTRLEN).decode('UTF-8').strip() ]
            for it in range(1,nzs):
                attr_time.append( self.f.read(ATTRLEN).decode('UTF-8').strip() )
            ## Parameter attributes
            attr_param = [ self.f.read(ATTRLEN).decode('UTF-8').strip() ]
            for it in range(1,npar):
                attr_param.append( self.f.read(ATTRLEN).decode('UTF-8').strip() )
            
            # Spatial attributes
            attr_dim1 = self.f.read(ATTRLEN).decode('UTF-8').strip()
            attr_dim2 = self.f.read(ATTRLEN).decode('UTF-8').strip()
            attr_dim3 = self.f.read(ATTRLEN).decode('UTF-8').strip()
        
        ## If geometry mode > EAS3_NO_G for time
        if gmode_time == EAS3_X0DX_G:
            time_data = np.zeros(2)
            for it in range(2):
                time_data[it] = struct.unpack('!d',self.f.read(8))[0]
        elif gmode_time == EAS3_ALL_G:
            time_data = np.zeros(size_time)
            for it in range(size_time):
                time_data[it] = struct.unpack('!d',self.f.read(8))[0]
        else: time_data = np.zeros(1)

        ## If geometry mode > EAS3_NO_G for parameters
        if gmode_param > EAS3_NO_G:
            param = np.zeros(size_param)
            for it in range(size_param):
                param[it] = struct.unpack('!d',self.f.read(8))[0]

        ## If geometry mode > EAS3_NO_G for dimensions 1 to 3
        dim1_data = np.zeros(size_dim1)
        if gmode_dim1 > EAS3_NO_G:
            for it in range(size_dim1):
                dim1_data[it] = struct.unpack('!d',self.f.read(8))[0]
            if abs(dim1_data[0]) < 1e-18: dim1_data[0] = 0.   
        dim2_data = np.zeros(size_dim2)
        if gmode_dim2 > EAS3_NO_G:
            for it in range(size_dim2):
                dim2_data[it] = struct.unpack('!d',self.f.read(8))[0]
            if abs(dim2_data[0]) < 1e-18: dim2_data[0] = 0.
        dim3_data = np.zeros(size_dim3)
        if gmode_dim3 > EAS3_NO_G:
            for it in range(size_dim3):
                dim3_data[it] = struct.unpack('!d',self.f.read(8))[0]
        else: dim3_data = 0.
        
        ## If user-defined data is chosen 
        if udef==EAS3_ALL_UDEF:
            udef_char = []
            for it in range(udef_char_size):
                udef_char.append(self.f.read(UDEFLEN).decode('UTF-8').strip())
            udef_int = np.zeros(udef_int_size,int)
            for it in range(udef_int_size):
                udef_int[it] = struct.unpack('!q',self.f.read(8))[0]
            udef_real = np.zeros(udef_real_size)
            for it in range(udef_real_size):
                udef_real[it] = struct.unpack('!d',self.f.read(8))[0]
        
        ## Option: convert gmode=EAS3_X0DX_G to gmode=EAS3_ALL_G
        if ChangeGmode==1:
            if gmode_dim1==EAS3_X0DX_G:
                dim1_data = np.linspace(dim1_data[0],dim1_data[0]+dim1_data[1]*(ndim1-1), ndim1)
                gmode_dim1=EAS3_ALL_G
            if gmode_dim2==EAS3_X0DX_G:
                dim2_data = np.linspace(dim2_data[0],dim2_data[0]+dim2_data[1]*(ndim2-1), ndim2)
                gmode_dim2=EAS3_ALL_G
            if gmode_dim3==EAS3_X0DX_G:
                dim3_data = np.linspace(dim3_data[0],dim3_data[0]+dim3_data[1]*(ndim3-1), ndim3)
                gmode_dim3=EAS3_ALL_G
            if gmode_time==EAS3_X0DX_G:
                time_data = np.linspace(time_data[0],time_data[0]+time_data[1]*(nzs  -1), nzs  )
                gmode_time=EAS3_ALL_G
        
        # ===
        
        self.attr_param = attr_param
        self.scalars    = attr_param
        self.t          = time_data
        self.nt         = self.t.size
        
        if   (attr_dim1=='x'):
            self.x = dim1_data
        elif (attr_dim1=='y'):
            self.y = dim1_data
        elif (attr_dim1=='z'):
            self.z = dim1_data
        else:
            raise ValueError('attr_dim1 = %s not identifiable as any x,y,z'%attr_dim1)
        
        if   (attr_dim2=='x'):
            self.x = dim2_data
        elif (attr_dim2=='y'):
            self.y = dim2_data
        elif (attr_dim2=='z'):
            self.z = dim2_data
        else:
            raise ValueError('attr_dim2 = %s not identifiable as any x,y,z'%attr_dim2)
        
        if   (attr_dim3=='x'):
            self.x = dim3_data
        elif (attr_dim3=='y'):
            self.y = dim3_data
        elif (attr_dim3=='z'):
            self.z = dim3_data
        else:
            raise ValueError('attr_dim3 = %s not identifiable as any x,y,z'%attr_dim3)
        
        # === transpose order to [xyz]
        
        if all([(attr_dim1=='x'),(attr_dim2=='y'),(attr_dim3=='z')]):
            self.axes_transpose_xyz = (0,1,2)
        elif all([(attr_dim1=='y'),(attr_dim2=='x'),(attr_dim3=='z')]):
            self.axes_transpose_xyz = (1,0,2)
        elif all([(attr_dim1=='z'),(attr_dim2=='y'),(attr_dim3=='x')]):
            self.axes_transpose_xyz = (2,1,0)
        elif all([(attr_dim1=='x'),(attr_dim2=='z'),(attr_dim3=='y')]):
            self.axes_transpose_xyz = (0,2,1)
        elif all([(attr_dim1=='y'),(attr_dim2=='z'),(attr_dim3=='x')]):
            self.axes_transpose_xyz = (2,0,1)
        elif all([(attr_dim1=='z'),(attr_dim2=='x'),(attr_dim3=='y')]):
            self.axes_transpose_xyz = (1,2,0)
        else:
            raise ValueError('could not figure out transpose axes')
        
        # ===
        
        self.nx  = self.x.size
        self.ny  = self.y.size
        self.nz  = self.z.size
        self.ngp = self.nx*self.ny*self.nz
        
        if self.verbose: print(72*'-')
        if self.verbose: even_print('nx', '%i'%self.nx )
        if self.verbose: even_print('ny', '%i'%self.ny )
        if self.verbose: even_print('nz', '%i'%self.nz )
        if self.verbose: even_print('ngp', '%i'%self.ngp )
        if self.verbose: print(72*'-')
        
        if self.verbose: even_print('x_min', '%0.2f'%self.x.min())
        if self.verbose: even_print('x_max', '%0.2f'%self.x.max())
        if self.verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (self.x[1]-self.x[0]), (self.x[-1]-self.x[-2]) ))
        if self.verbose: even_print('y_min', '%0.2f'%self.y.min())
        if self.verbose: even_print('y_max', '%0.2f'%self.y.max())
        if self.verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (self.y[1]-self.y[0]), (self.y[-1]-self.y[-2]) ))
        if self.verbose: even_print('z_min', '%0.2f'%self.z.min())
        if self.verbose: even_print('z_max', '%0.2f'%self.z.max())        
        if self.verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (self.z[1]-self.z[0]), (self.z[-1]-self.z[-2]) ))
        if self.verbose: print(72*'-'+'\n')
        
        # ===
        
        udef_dict = {}
        for i in range(len(udef_char)):
            if (udef_char[i]!=''):
                if (udef_int[i]!=0):
                    udef_dict[udef_char[i]] = int(udef_int[i])
                elif (udef_real[i]!=0.):
                    udef_dict[udef_char[i]] = float(udef_real[i])
                else:
                    udef_dict[udef_char[i]] = 0.
        
        if self.verbose:
            print('udef from EAS3\n' + 72*'-')
            for key in udef_dict:
                if isinstance(udef_dict[key],float):
                    even_print(key, '%0.8f'%udef_dict[key])
                elif isinstance(udef_dict[key],int):
                    even_print(key, '%i'%udef_dict[key])
                else:
                    print(type(udef_dict[key]))
                    raise TypeError('udef dict item not float or int')
            print(72*'-'+'\n')
        
        self.Ma    = udef_dict['Ma']
        self.Re    = udef_dict['Re']
        self.Pr    = udef_dict['Pr']
        self.kappa = udef_dict['kappa']
        
        if ('T_unend' in udef_dict):
            self.T_inf = udef_dict['T_unend']
        elif ('T_inf' in udef_dict):
            self.T_inf = udef_dict['T_inf']
        elif ('Tinf' in udef_dict):
            self.T_inf = udef_dict['Tinf']
        else:
            if self.verbose: print('WARNING! No match in udef for any T_unend, T_inf, Tinf')
            if self.verbose: print('--> setting T_inf = (273.15+15) [K]')
            self.T_inf = 273.15 + 15
        
        if ('R' in udef_dict):
            self.R = udef_dict['R']
        ## elif ('cv' in udef_dict):
        ##     print('WARNING! No match in udef for R, but cv given')
        ##     self.cv = udef_dict['cv']
        ##     self.R  = self.cv / (5/2)
        ##     print('--> assuming air, taking R = cv/(5/2) = %0.3f [J/(kg·K)]'%self.R)
        else:
            #if self.verbose: print('WARNING! No match in udef for R and no cv given')
            if self.verbose: print('WARNING! No match in udef for R')
            if self.verbose: print('--> assuming air, setting R = 287.055 [J/(kg·K)]')
            self.R = 287.055
        
        if ('p_unend' in udef_dict):
            self.p_inf = udef_dict['p_unend']
        elif ('p_inf' in udef_dict):
            self.p_inf = udef_dict['p_inf']
        elif ('pinf' in udef_dict):
            self.p_inf = udef_dict['pinf']
        else:
            if self.verbose: print('WARNING! No match in udef for any p_unend, p_inf, pinf')
            if self.verbose: print('--> setting p_inf = 101325 [Pa]')
            self.p_inf = 101325.

        self.rho_inf     = self.p_inf/(self.R*self.T_inf) ## mass density [kg/m³]
        
        self.S_Suth      = 110.4    ## [K] --> Sutherland temperature
        self.mu_Suth_ref = 1.716e-5 ## [kg/(m·s)] --> μ of air at T_Suth_ref = 273.15 [K]
        self.T_Suth_ref  = 273.15   ## [K]
        self.C_Suth      = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## [kg/(m·s·√K)]
        #mu_inf      = self.C_Suth*self.T_inf**(3/2)/(self.T_inf+self.S_Suth)
        self.mu_inf      = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2)*(self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth) ## [Pa s] | [N s m^-2]
        
        self.nu_inf = self.mu_inf / self.rho_inf ## kinematic viscosity [m²/s] --> momentum diffusivity
        
        # ===
        
        if self.verbose: print(72*'-')
        if self.verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
        if self.verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
        if self.verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
        if self.verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
        if self.verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
        if self.verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
        if self.verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
        if self.verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
        if self.verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
        if self.verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
        if self.verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
        if self.verbose: print(72*'-')
        
        self.a_inf = np.sqrt(self.kappa*self.R*self.T_inf)           
        self.U_inf = self.Ma*self.a_inf
        self.cp    = self.R*self.kappa/(self.kappa-1.)
        self.cv    = self.cp/self.kappa                         
        self.r     = self.Pr**(1/3)                        
        self.Tw    = self.T_inf                            
        self.Taw   = self.T_inf + self.r*self.U_inf**2/(2*self.cp)        
        self.lchar = self.Re*self.nu_inf/self.U_inf
        
        if self.verbose: even_print('rho_inf' , '%0.3f [kg/m³]'    % self.rho_inf )
        if self.verbose: even_print('mu_inf'  , '%0.6E [kg/(m·s)]' % self.mu_inf  )
        if self.verbose: even_print('nu_inf'  , '%0.6E [m²/s]'     % self.nu_inf  )
        if self.verbose: even_print('a_inf'   , '%0.6f [m/s]'      % self.a_inf   )
        if self.verbose: even_print('U_inf'   , '%0.6f [m/s]'      % self.U_inf   )
        if self.verbose: even_print('cp'      , '%0.3f [J/(kg·K)]' % self.cp      )
        if self.verbose: even_print('cv'      , '%0.3f [J/(kg·K)]' % self.cv      )
        if self.verbose: even_print('r'       , '%0.6f [-]'        % self.r       )
        if self.verbose: even_print('Tw'      , '%0.3f [K]'        % self.Tw      )
        if self.verbose: even_print('Taw'     , '%0.3f [K]'        % self.Taw     )
        if self.verbose: even_print('lchar'   , '%0.6E [m]'        % self.lchar   )
        if self.verbose: print(72*'-'+'\n')
        
        # ===
        
        udef_char = [    'Ma',     'Re',     'Pr',     'kappa',    'R',    'p_inf',    'T_inf',    'C_Suth',    'mu_Suth_ref',    'T_Suth_ref' ]
        udef_real = [self.Ma , self.Re , self.Pr , self.kappa, self.R, self.p_inf, self.T_inf, self.C_Suth, self.mu_Suth_ref, self.T_Suth_ref  ]
        udef = dict(zip(udef_char, udef_real))
        
        return udef

# 1D curve fitting
# ======================================================================

class curve_fitter(object):
    '''
    creates a curve fitter instance which is callable afterward
    - includes text for LaTeX and matplotlib
    '''
    def __init__(self, curveType, x, y):
        
        self.curveType = curveType
        
        if (curveType=='linear'):
            '''
            linear function : y=a+b·x
            a straight line on a lin-lin plot
            '''
            def curve(x, a, b):
                return a + b*x
        
        elif (curveType=='power'):
            '''
            power law y=a·x^b
            a straight line on a log-log plot
            '''
            def curve(x, a, b):
                return a*np.power(x,b)
        
        elif (curveType=='power_plus_const'):
            '''
            power law (plus a constant) y = a + b·(x^c)
            no longer a straight line on log-log but allows non-zero y-intercept
            '''
            def curve(x, a, b, c):
                return a + b*np.power(x,c)
        
        elif (curveType=='power_asymp'):
            '''
            power asymptotic
            '''
            def curve(x, a, b, c, d):
                return a/(b + c*np.power(x,d))
        
        elif (curveType=='exp'):
            '''
            exponential curve y = a + b·exp(c*x)
            '''
            def curve(x, a, b, c):
                return a + b*np.exp(c*x)
        
        elif (curveType=='log'):
            '''
            a straight line on a semi-log (lin-log) plot 
            '''
            def curve(x, a, b):
                return a + b*np.log(x)
        
        else:
            raise ValueError('curveType not recognized : %s'%str(curveType))
        
        self.__curve = curve ## private copy of curve() method
        self.popt, self.pcov = sp.optimize.curve_fit(self.__curve, x, y, maxfev=int(5e5), method='trf')
        
        # ===
        
        if (curveType=='linear'):
            a, b = self.popt
            self.txt = '%0.12e + %0.12e * x'%(a,b)
            self.latex = r'$%0.5f + %0.5f{\cdot}x$'%(a,b)
        
        elif (curveType=='power'):
            a, b = self.popt
            self.txt = '%0.12e * x**%0.12e'%(a,b)
            self.latex = r'$%0.5f{\cdot}x^{%0.5f}$'%(a,b)
        
        elif (curveType=='power_plus_const'):
            a, b, c = self.popt
            self.txt = '%0.12e + %0.12e * x**%0.12e'%(a,b,c)
            self.latex = r'$%0.5f + %0.5f{\cdot}x^{%0.5f}$'%(a,b,c)
        
        elif (curveType=='power_asymp'):
            a, b, c, d = self.popt
            self.txt = '%0.12e / (%0.12e + %0.12e * x**%0.12e)'%(a,b,c,d)
            self.latex = r'$%0.5f / (%0.5f + %0.5f {\cdot} x^{%0.5f})$'%(a,b,c,d)
        
        elif (curveType=='exp'):
            a, b, c = self.popt
            self.txt = '%0.12e + %0.12e * np.exp(%0.12e * x)'%(a,b,c)
            self.latex = r'$%0.5f + %0.5f{\cdot}\text{exp}(%0.5f{\cdot}x)$'%(a,b,c)
        
        elif (curveType=='log'):
            a, b = self.popt
            self.txt = '%0.12e + %0.12e * np.log(x)'%(a,b)
            self.latex = r'$%0.6f + %0.6f{\cdot}\text{ln}(x)$'%(a,b)
        
        else:
            raise NotImplementedError('curveType \'%s\' not recognized'%str(curveType))
    
    def __call__(self, xn):
        return self.__curve(xn, *self.popt)

# boundary layer & aerodynamics
# ======================================================================

def Blasius_solution(eta):
    '''
    f·f′′ + 2·f′′′ = 0  ==>  f′′′ = -(1/2)·f·f′′
    BCs: f(0)=0, f′(0)=0, f′(∞)=1
    -----
    for solve_ivp(): d[f′′(η)]/dη = F(f(η), f′(η), f′′(η))
    y=[f,f′,f′′], y′=[ y[1], y[2], (-1/2)·y[0]·y[2] ]
    '''
    
    #Blasius = lambda t,y: [y[1],y[2],-0.5*y[0]*y[2]]
    
    def Blasius(t,y):
        return np.array([y[1], y[2], -0.5*y[0]*y[2]])
    
    if False: ## calculate c0
        import warnings
        warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) 
        def eq_root(c0,eta):
            sol = sp.integrate.solve_ivp( fun=Blasius, t_span=[0,eta[-1]], y0=[0,0,c0], t_eval=eta, method='RK45' )
            fp  = np.copy( sol.y[1] ) ## f′
            res = 1. - fp[-1] ## residual for BC: f′(∞)=0
            return res
        eta = np.linspace(0,1e3,int(1e3))
        sol = sp.optimize.fsolve(eq_root, x0=0.332, args=(eta,), xtol=1e-9)
        c0 = sol[0]
        #print('c0 = %0.14f'%c0)
    else:
        c0 = 0.33200221890262
    
    sol = sp.integrate.solve_ivp( fun=Blasius, 
                                  t_span=[0,eta[-1]], 
                                  y0=[ 0, 0, c0 ], 
                                  t_eval=eta, 
                                  method='RK45' )
    
    f, fp, fpp = sol.y ## f′=u/U
    
    ## i_99 = np.abs(fp-0.99).argmin()
    ## eta_99 = eta[i_99]
    ## print('η(f′=%0.14f) = %0.14f'%(fp[i_99],eta_99))
    ## ## η(f′=0.0.99000000002790)=4.91131834911318
    
    return f, fp, fpp

class freestream_parameters(object):
    '''
    given freestream Mach Number (M_inf) and a reference Reynolds Number (Re), calculate 
    freestream parameters and characteristic scales for dry air (21% O2 / 78% N2) and standard conditions
    '''
    def __init__(self, Re=3000., M_inf=None, compressible=True):
        
        self.Re = Re
        self.M_inf = M_inf
        
        self.compressible = compressible
        
        if (M_inf is None):
            raise ValueError('M_inf not provided --> even if incompressible, provide for preliminary calculation of U_inf')
        
        # ===
        
        T_inf   = 273.15 + 15 ## [K]
        p_inf   = 101325.     ## [Pa]
        
        ## dry air (21% O2 / 78% N2)
        M_molar = 28.9647e-3       ## [kg/mol]
        R_molar = 8.31446261815324 ## [J/(K·mol)]
        
        R     = R_molar / M_molar ## specific / individual gas constant [J/(kg·K)]
        cp    = (7/2)*R           ## isobaric specific heat (ideal gas) [J/(kg·K)]
        cv    = (5/2)*R           ## isochoric specific heat (ideal gas) [J/(kg·K)]
        kappa = cp/cv
        
        rho_inf = p_inf/(R*T_inf) ## mass density [kg/m³]
        
        ## Sutherland's Law : dynamic viscosity : μ(T)
        S_Suth      = 110.4    ## [K] --> Sutherland temperature
        mu_Suth_ref = 1.716e-5 ## [kg/(m·s)] --> μ of air at T_Suth_ref = 273.15 [K]
        T_Suth_ref  = 273.15   ## [K]
        C_Suth      = mu_Suth_ref/(T_Suth_ref**(3/2))*(T_Suth_ref + S_Suth) ## [kg/(m·s·√K)]
        #mu_inf      = C_Suth*T_inf**(3/2)/(T_inf+S_Suth)
        mu_inf      = mu_Suth_ref*(T_inf/T_Suth_ref)**(3/2)*(T_Suth_ref+S_Suth)/(T_inf+S_Suth) ## [Pa·s] | [N·s/m²]
        
        nu_inf = mu_inf / rho_inf ## kinematic viscosity [m²/s] --> momentum diffusivity
        
        ## Sutherland's Law : thermal conductivity : k(T)
        k_Suth_ref = 0.0241 ## [W/(m·K)]
        Sk_Suth    = 194.0  ## [K]
        k_inf      = k_Suth_ref*(T_inf/T_Suth_ref)**(3/2)*(T_Suth_ref+Sk_Suth)/(T_inf+Sk_Suth) ## [W/(m·K)]
        
        alpha     = k_inf/(rho_inf*cp) ## thermal diffusivity [m²/s]
        Pr        = nu_inf/alpha       ## [-] ==cp·mu/k
        recov_fac = pow(Pr,1/3)        ## recovery factor
        
        if False: ## simple definition (set rho_inf, Pr explicitly)
            
            T_inf   = 273.15 + 15  ## temperature [K]
            rho_inf = 1.225        ## mass density [kg/m³]
            
            Pr        = 0.71
            recov_fac = pow(Pr,1/3) ## recovery factor
            
            R       = 287.058 ## specific / individual gas constant [J/(kg·K)]
            cp      = (7/2)*R ## isobaric specific heat (ideal gas) [J kg^-1 K^-1]
            cv      = (5/2)*R ## isochoric specific heat (ideal gas) [J kg^-1 K^-1]
            kappa   = cp/cv
            p_inf   = rho_inf*R*T_inf
            
            S_Suth      = 110.4    ## [K] --> Sutherland temperature
            mu_Suth_ref = 1.716e-5 ## [kg/(m·s)] --> μ of air at T_Suth_ref = 273.15 [K]
            T_Suth_ref  = 273.15   ## [K]
            C_Suth      = mu_Suth_ref/(T_Suth_ref**(3/2))*(T_Suth_ref + S_Suth) ## [kg/(m·s·√K)]
            
            mu_inf = mu_Suth_ref*(T_inf/T_Suth_ref)**(3/2)*(T_Suth_ref+S_Suth)/(T_inf+S_Suth)
            nu_inf = mu_inf/rho_inf
            k_inf  = cp*mu_inf/Pr ## thermal conductivity [W/(m·K)]
        
        # ===
        
        a_inf = np.sqrt(kappa*R*T_inf) ## speed of sound [m/s]
        U_inf = a_inf * M_inf          ## velocity freestream [m/s]
        
        lchar = Re * nu_inf / U_inf
        tchar = lchar / U_inf
        uchar = lchar / tchar
        if not np.isclose(uchar, U_inf, rtol=1e-14):
            raise AssertionError('U_inf!=uchar')
        
        if self.compressible:
            
            ## isentropic total quantities (compressible)
            T_tot   = T_inf   * (1 + (kappa-1)/2 * M_inf**2 ) 
            p_tot   = p_inf   * (1 + (kappa-1)/2 * M_inf**2)**(kappa/(kappa-1))
            rho_tot = rho_inf * (1 + (kappa-1)/2 * M_inf**2)**(1/(kappa-1))
        
        else:
            
            self.M_inf = 0
            a_inf = np.inf
            
            T_tot   = T_inf   + U_inf**2 / (2 * cp)
            p_tot   = p_inf   + 0.5*rho_inf*U_inf**2 ## Bernoulli
            rho_tot = rho_inf
        
        HTfac = 1.0 ## adiabatic
        Taw = T_inf + (recov_fac*U_inf**2)/(2*cp)
        Tw  = T_inf + (recov_fac*U_inf**2)/(2*cp)*HTfac
        
        # === attach to self
        
        self.T_inf       = T_inf
        self.p_inf       = p_inf
        self.M_molar     = M_molar
        self.R_molar     = R_molar
        self.R           = R
        self.cp          = cp
        self.cv          = cv
        self.kappa       = kappa
        self.rho_inf     = rho_inf
        self.S_Suth      = S_Suth
        self.mu_Suth_ref = mu_Suth_ref
        self.T_Suth_ref  = T_Suth_ref
        self.C_Suth      = C_Suth
        self.mu_inf      = mu_inf
        self.nu_inf      = nu_inf
        self.k_Suth_ref  = k_Suth_ref
        self.Sk_Suth     = Sk_Suth
        self.k_inf       = k_inf
        self.alpha       = alpha
        self.Pr          = Pr
        self.recov_fac   = recov_fac
        self.a_inf       = a_inf
        self.U_inf       = U_inf
        
        self.lchar       = lchar
        self.tchar       = tchar
        self.uchar       = uchar
        
        self.T_tot       = T_tot
        self.p_tot       = p_tot
        self.rho_tot     = rho_tot
        self.HTfac       = HTfac
        self.Taw         = Taw
        self.Tw          = Tw
    
    def set_Re(self, Re):
        '''
        set reference Reynolds Number, which determines lchar & tchar
        '''
        self.Re    = Re
        self.lchar = Re * self.nu_inf / self.U_inf
        self.tchar = self.lchar / self.U_inf
        self.uchar = self.lchar / self.tchar
        if not np.isclose(self.uchar, self.U_inf, rtol=1e-14):
            raise AssertionError('U_inf!=uchar')
        return
    
    def print(self,):
        
        even_print('R_molar'        , '%0.6f [J/(K·mol)]'   % self.R_molar    )
        even_print('M_molar'        , '%0.5e [kg/mol]'      % self.M_molar    )
        even_print('R'              , '%0.3f [J/(kg·K)]'    % self.R          )
        even_print('cp'             , '%0.3f [J/(kg·K)]'    % self.cp         )
        even_print('cv'             , '%0.3f [J/(kg·K)]'    % self.cv         )
        even_print('kappa'          , '%0.3f [-]'           % self.kappa      )
        ##
        print(72*'-')
        even_print('compressible'   , '%s'                  % self.compressible )
        even_print('M_inf'          , '%0.4f [-]'           % self.M_inf      )
        even_print('Re'             , '%0.4f [-]'           % self.Re         )
        ##
        even_print('T_inf'          , '%0.4f [K]'           % self.T_inf      )
        even_print('T_tot'          , '%0.4f [K]'           % self.T_tot      )
        even_print('Tw'             , '%0.4f [K]'           % self.Tw         )
        even_print('Taw'            , '%0.4f [K]'           % self.Taw        )
        even_print('p_inf'          , '%0.1f [Pa]'          % self.p_inf      )
        even_print('p_tot'          , '%0.1f [Pa]'          % self.p_tot      )
        even_print('rho_inf'        , '%0.4f [kg/m³]'       % self.rho_inf    )
        even_print('rho_tot'        , '%0.4f [kg/m³]'       % self.rho_tot    )
        ##
        even_print('T_Suth_ref'     , '%0.2f [K]'           % self.T_Suth_ref )
        even_print('C_Suth'         , '%0.3e [kg/(m·s·√K)]' % self.C_Suth     )
        even_print('S_Suth'         , '%0.2f [K]'           % self.S_Suth     )
        ##
        even_print('mu_inf'         , '%0.5e [kg/(m·s)]'    % self.mu_inf     )
        even_print('nu_inf'         , '%0.5e [m²/s]'        % self.nu_inf     )
        even_print('k_inf'          , '%0.5e [W/(m·K)]'     % self.k_inf      )
        even_print('alpha'          , '%0.5e [m²/s]'        % self.alpha      )
        ##
        even_print('Pr'             , '%0.5f [-]'           % self.Pr         )
        even_print('recovery factor', '%0.5f [-]'           % self.recov_fac  )
        ##
        even_print('a_inf'          , '%0.5f [m/s]'         % self.a_inf      )
        even_print('U_inf'          , '%0.5f [m/s]'         % self.U_inf      )
        even_print('uchar'          , '%0.5f [m/s]'         % self.uchar      )
        even_print('lchar'          , '%0.5e [m]'           % self.lchar      )
        even_print('tchar'          , '%0.5e [s]'           % self.tchar      )
        print(72*'-')
        
        return

def calc_bl_edge_1d(y, psvel, **kwargs):
    '''
    determine the boundary layer edge location of 1D profile
    '''
    verbose      = kwargs.get('verbose',True)
    #method       = kwargs.get('method','psvel')
    epsilon      = kwargs.get('epsilon',5e-4)
    acc          = kwargs.get('acc',8)
    edge_stencil = kwargs.get('edge_stencil','half')
    
    ynorm = kwargs.get('ynorm',1.) ## normalization factor... usually lchar
    
    if (y.ndim!=1):
        raise ValueError
    if (psvel.ndim!=1):
        raise ValueError
    if (psvel.shape[0]!=y.shape[0]):
        raise ValueError
    
    ny = y.shape[0]
    psvel_max = psvel.max()
    
    ddy_psvel_normed = gradient( psvel/psvel_max, y/ynorm, axis=0, d=1, acc=acc, edge_stencil=edge_stencil )
    #ddy_psvel_normed = gradient( psvel, y/ynorm, axis=0, d=1, acc=acc, edge_stencil=edge_stencil )
    
    ## debug plot
    # plt.close('all')
    # fig1 = plt.figure(figsize=(3,3), dpi=300)
    # ax1 = plt.gca()
    # ax1.plot(  ddy_psvel_normed, y/ynorm, lw=0.5 )
    # ax1.axvline(x=epsilon, linestyle='dashed', c='lightgray', zorder=1, lw=0.3)
    # fig1.tight_layout(pad=0.25)
    # fig1.tight_layout(pad=0.25)
    # plt.show()
    
    ## get [y] of first intersection
    jtop = ny-1
    for j in range(ny):
        if ( ddy_psvel_normed[j] < epsilon):
            jtop = j
            break
    
    intrp_func = sp.interpolate.interp1d(y/ynorm, ddy_psvel_normed, kind='linear', bounds_error=True)
    
    def __f_opt1(y_test, intrp_func, epsilon, ynorm):
        ddy_psvel_test = intrp_func(y_test/ynorm)
        root = np.abs( ddy_psvel_test - epsilon )
        return root
    
    sol = sp.optimize.least_squares(fun=__f_opt1,
                                    args=(intrp_func, epsilon, ynorm),
                                    x0=0.5*(y[jtop-2]+y[jtop]),
                                    xtol=1e-14,
                                    #ftol=1e-15,
                                    method='dogbox',
                                    bounds=(y[jtop-2], y[jtop]))
    if not sol.success:
        raise ValueError
    
    y_edge = sol.x[0]
    
    return y_edge

def calc_d99_1d(y, psvel, y_edge, psvel_edge, **kwargs):
    '''
    calculate [δ99] of 1D profile
    '''
    
    if (y.ndim!=1):
        raise ValueError
    if (psvel.ndim!=1):
        raise ValueError
    if (psvel.shape[0]!=y.shape[0]):
        raise ValueError
    if not isinstance(y_edge, (float,)):
        raise ValueError
    if not isinstance(psvel_edge, (float,)):
        raise ValueError
    
    if (y_edge>y.max()):
        raise ValueError
    if (y_edge<y.min()):
        raise ValueError
    
    ny = y.shape[0]
    
    j_edge = np.abs(y-y_edge).argmin()
    y_edge_g = y[j_edge]
    
    je = j_edge + 2
    #je = min( j_edge + 2, ny+1 )
    
    intrp_func = sp.interpolate.interp1d(y[:je], psvel[:je], kind='cubic', bounds_error=True)
    
    ## check that [psvel_edge] is correct
    psvel_edge_ = intrp_func(y_edge)
    np.testing.assert_allclose(psvel_edge, psvel_edge_, rtol=1e-6)
    
    def __f_opt(y_test, intrp_func, psvel_edge):
        root = np.abs( 0.99*psvel_edge - intrp_func(y_test) )
        return root
    
    sol = sp.optimize.least_squares(fun=__f_opt,
                                    args=(intrp_func,psvel_edge),
                                    #args=(intrp_func,u_edge_),
                                    x0=0.99*y_edge,
                                    xtol=1e-14,
                                    #ftol=1e-14,
                                    method='dogbox',
                                    bounds=(y.min(), y_edge))
    if not sol.success:
        raise ValueError
    
    d99 = sol.x[0]
    
    return d99

def calc_bl_integral_quantities_1d( y, u, rho, u_tau, d99, y_edge, rho_edge, nu_edge, u_edge, nu_wall, **kwargs):
    '''
    for 1D profile get [θ, δ*, Re_θ, Re_τ]
    '''
    
    if (y.ndim!=1):
        raise ValueError
    
    if (u.ndim!=1):
        raise ValueError
    if (u.shape[0]!=y.shape[0]):
        raise ValueError
    if (rho.ndim!=1):
        raise ValueError
    if (rho.shape[0]!=y.shape[0]):
        raise ValueError
    
    if not isinstance(u_tau, (float,)):
        raise ValueError
    if not isinstance(d99, (float,)):
        raise ValueError
    if not isinstance(y_edge, (float,)):
        raise ValueError
    if not isinstance(rho_edge, (float,)):
        raise ValueError
    if not isinstance(nu_edge, (float,)):
        raise ValueError
    if not isinstance(u_edge, (float,)):
        raise ValueError
    if not isinstance(nu_wall, (float,)):
        raise ValueError
    
    if (y_edge>y.max()):
        raise ValueError
    if (y_edge<y.min()):
        raise ValueError
    
    ny = y.shape[0]
    
    rho_edge_ = sp.interpolate.interp1d(y, rho, kind='cubic', bounds_error=True)(y_edge)
    np.testing.assert_allclose(rho_edge, rho_edge_, rtol=1e-5)
    
    u_edge_ = sp.interpolate.interp1d(y, u, kind='cubic', bounds_error=True)(y_edge)
    np.testing.assert_allclose(u_edge, u_edge_, rtol=1e-5)
    
    # ===
    
    integrand_theta_cmp = (u*rho)/(u_edge*rho_edge)*(1-(u/u_edge))
    integrand_dstar_cmp = (1-((u*rho)/(u_edge*rho_edge)))
    
    theta_cmp_     = sp.integrate.cumulative_trapezoid(y=integrand_theta_cmp, x=y, initial=0.)
    theta_cmp_func = sp.interpolate.interp1d(y, theta_cmp_, kind='cubic')
    theta_cmp      = theta_cmp_func(y_edge)
    
    dstar_cmp_     = sp.integrate.cumulative_trapezoid(y=integrand_dstar_cmp, x=y, initial=0.)
    dstar_cmp_func = sp.interpolate.interp1d(y, dstar_cmp_, kind='cubic')
    dstar_cmp      = dstar_cmp_func(y_edge)
    
    integrand_theta_inc = (u/u_edge)*(1-(u/u_edge))
    integrand_dstar_inc = 1-(u/u_edge)
    
    theta_inc_     = sp.integrate.cumulative_trapezoid(y=integrand_theta_inc, x=y, initial=0.)
    theta_inc_func = sp.interpolate.interp1d(y, theta_inc_, kind='cubic')
    theta_inc      = theta_inc_func(y_edge)
    
    dstar_inc_     = sp.integrate.cumulative_trapezoid(y=integrand_dstar_inc, x=y, initial=0.)
    dstar_inc_func = sp.interpolate.interp1d(y, dstar_inc_, kind='cubic')
    dstar_inc      = dstar_inc_func(y_edge)
    
    # ===
    
    H12     = dstar_cmp/theta_cmp
    H12_inc = dstar_inc/theta_inc
    
    Re_tau       = d99*u_tau/nu_wall
    Re_theta     = theta_cmp*u_edge/nu_edge
    Re_theta_inc = theta_inc*u_edge/nu_edge
    Re_d99       = d99*u_edge/nu_edge
    
    dd = { 'theta_cmp':theta_cmp,
           'dstar_cmp':dstar_cmp,
           'theta_inc':theta_inc,
           'dstar_inc':dstar_inc,
           'H12':H12,
           'H12_inc':H12_inc,
           'Re_tau':Re_tau,
           'Re_theta':Re_theta,
           'Re_theta_inc':Re_theta_inc,
           'Re_d99':Re_d99,
         }
    
    return dd

# numerical & grid
# ======================================================================

def interp_2d_structured(x2d_A, y2d_A, x2d_B, y2d_B, data_A, **kwargs):
    '''
    interpolate 2D array 'data_A' from grid A onto grid B, yielding 'data_B'
    --> based on sp.interpolate.griddata()
    --> default 'cubic' interpolation, where NaNs occur, fill with 'nearest'
    '''
    
    method = kwargs.get('method','cubic')
    
    if not isinstance(x2d_A, np.ndarray):
        raise ValueError('x2d_A should be a numpy array')
    if not isinstance(y2d_A, np.ndarray):
        raise ValueError('y2d_A should be a numpy array')
    if not isinstance(x2d_B, np.ndarray):
        raise ValueError('x2d_B should be a numpy array')
    if not isinstance(y2d_B, np.ndarray):
        raise ValueError('y2d_B should be a numpy array')
    if not isinstance(data_A, np.ndarray):
        raise ValueError('data_A should be a numpy array')
    
    # < need a lot of checks still >
    
    if not any([(method=='linear'),(method=='cubic')]):
        raise ValueError("'method' should be one of 'cubic' or 'linear'")
    
    nxA,nyA = data_A.shape
    nxB,nyB = x2d_B.shape
    
    ## interp2d() --> gets OverflowError for big meshes
    # interpolant = sp.interpolate.interp2d(x2d_A.flatten(),
    #                                       y2d_A.flatten(),
    #                                       data_A.flatten(),
    #                                       kind='linear',
    #                                       copy=True,
    #                                       bounds_error=False,
    #                                       fill_value=np.nan)
    # utang_wn = interpolant( x2d_B.flatten(), 
    #                         y2d_B.flatten() )
    
    B_nearest = sp.interpolate.griddata( points=(x2d_A.flatten(), y2d_A.flatten()),
                                         values=data_A.flatten(),
                                         xi=(x2d_B.flatten(), y2d_B.flatten()),
                                         method='nearest',
                                         fill_value=np.nan )
    B_nearest = np.reshape(B_nearest, (nxB,nyB), order='C')
    
    B_cubic = sp.interpolate.griddata( points=(x2d_A.flatten(), y2d_A.flatten()),
                                       values=data_A.flatten(),
                                       xi=(x2d_B.flatten(), y2d_B.flatten()),
                                       method=method,
                                       fill_value=np.nan )
    B_cubic = np.reshape(B_cubic, (nxB,nyB), order='C')
    
    #nan_indices = np.nonzero(np.isnan(B_cubic))
    #n_nans = np.count_nonzero(np.isnan(B_cubic))
    
    data_B = np.where( np.isnan(B_cubic), B_nearest, B_cubic)
    
    if np.isnan(data_B).any():
        raise AssertionError('interpolated scalar field has NaNs')
    
    return data_B

def interp_1d(x1d_A, x1d_B, data_A, axis=0):
    '''
    interpolate 1D array 'data_A' from grid A onto grid B, yielding 'data_B'
    - data_A can be any shape, as long as size of 'axis' dim is == size of x1d_A
    - based on sp.interpolate.interp1d()
    - default 'cubic' interpolation, where NaNs occur, fill with 'nearest'
    '''
    
    if not isinstance(x1d_A, np.ndarray):
        raise ValueError('x1d_A should be a numpy array')
    if not isinstance(x1d_B, np.ndarray):
        raise ValueError('x1d_B should be a numpy array')
    if not isinstance(data_A, np.ndarray):
        raise ValueError('data_A should be a numpy array')
    if (x1d_A.ndim!=1):
        raise ValueError('x1d_A.ndim!=1')
    if (x1d_B.ndim!=1):
        raise ValueError('x1d_B.ndim!=1')
    #if (data_A.ndim!=1):
    #    raise ValueError('data_A.ndim!=1')
    #if ( x1d_A.shape[0] != data_A.shape[0] ):
    #    raise ValueError('x1d_A.shape[0] != data_A.shape[0]')
    
    ## < could check monotonicity >
    
    intrp_func_nearest = sp.interpolate.interp1d(x1d_A, data_A, axis=axis, kind='nearest', bounds_error=False, fill_value='extrapolate')
    intrp_func_cubic   = sp.interpolate.interp1d(x1d_A, data_A, axis=axis, kind='cubic',   bounds_error=False, fill_value=np.nan)
    
    B_nearest = intrp_func_nearest(x1d_B)
    B_cubic   = intrp_func_cubic(x1d_B)
    
    #n_nans_cubic   = np.count_nonzero(np.isnan(B_cubic))
    #n_nans_nearest = np.count_nonzero(np.isnan(B_nearest))
    
    data_B = np.where( np.isnan(B_cubic), B_nearest, B_cubic)
    
    if np.isnan(data_B).any():
        raise AssertionError('interpolated scalar field has NaNs')
    
    return data_B

def fd_coeff_calculator(stencil, d=1, x=None, dx=None):
    '''
    Calculate Finite Difference Coefficients for Arbitrary Stencil
    -----
    stencil : indices of stencil pts e.g. np.array([-2,-1,0,1,2])
    d       : derivative order
    x       : locations of grid points corresponding to stencil indices
    dx      : spacing of grid points in the case of uniform grid
    -----
    https://en.wikipedia.org/wiki/Finite_difference_coefficient
    https://web.media.mit.edu/~crtaylor/calculator.html
    -----
    Fornberg B. (1988) Generation of Finite Difference Formulas on
    Arbitrarily Spaced Grids, Mathematics of Computation 51, no. 184 : 699-706.
    http://www.ams.org/journals/mcom/1988-51-184/S0025-5718-1988-0935077-0/S0025-5718-1988-0935077-0.pdf
    '''
    
    stencil = np.asanyarray(stencil)
    
    if not isinstance(stencil, np.ndarray):
        raise ValueError('stencil should be of type np.ndarray')
    if (stencil.ndim!=1):
        raise ValueError('stencil should be 1D')
    if (stencil.shape[0]<2):
        raise ValueError('stencil size should be >=2')
    if (0 not in stencil):
        raise ValueError('stencil does not contain 0')
    if not np.issubdtype(stencil.dtype, np.integer):
        raise ValueError('stencil.dtype not a subdtype of np.integer')
    
    if not isinstance(d, int):
        raise ValueError('d (derivative order) should be of type int')
    if not (d>0):
        raise ValueError('d (derivative order) should be >0')
    
    if (dx is None) and (x is None):
        raise ValueError('one of args \'dx\' or \'x\' should be defined')
    if (dx is not None) and (x is not None):
        raise ValueError('only one of args \'dx\' or \'x\' should be defined')
    if (dx is not None):
        if not isinstance(dx, float):
            raise ValueError('dx should be of type float')
    
    if (x is not None):
        if not isinstance(x, np.ndarray):
            raise ValueError('x should be of type np.ndarray')
        if (x.shape[0] != stencil.shape[0]):
            raise ValueError('x, stencil should have same shape')
        if (not np.all(np.diff(x) > 0.)) and (not np.all(np.diff(x) < 0.)):
            raise AssertionError('x is not monotonically increasing/decreasing')
    
    ## overwrite stencil (int index) to be coordinate array (delta from 0 position)
    
    i0 = np.where(stencil==0)[0][0]
    
    if (x is not None):
        stencil = x - x[i0] 
    
    if (dx is not None):
        stencil = dx * stencil.astype(np.float64)
    
    nn = stencil.shape[0]
    
    dvec = np.zeros( (nn,) , dtype=np.float64 )
    #dvec = np.zeros( (nn,) , dtype=np.longdouble )
    dfac=1
    for i in range(d):
        dfac *= (i+1)
    dvec[d] = dfac
    
    ## increase precision
    #stencil = np.copy(stencil).astype(np.longdouble)
    
    stencil_abs_max         = np.abs(stencil).max()
    stencil_abs_min_nonzero = np.abs(stencil[[ i for i in range(stencil.size) if i!=i0 ]]).min()
    
    '''
    scale/normalize the coordinate stencil (to avoid ill-conditioning)
    - if coordinates are already small/large, the Vandermonde matrix becomes
       HIGHLY ill-conditioned due to row exponents
    - coordinates are normalized here so that smallest absolute non-zero delta coord. is =1
    - RHS vector (dvec) gets normalized too
    - FD coefficients are (theoretically) unaffected
    '''
    normalize_stencil = True
    
    if normalize_stencil:
        stencil /= stencil_abs_min_nonzero
    
    mat = np.zeros( (nn,nn) , dtype=np.float64)
    #mat = np.zeros( (nn,nn) , dtype=np.longdouble)
    for i in range(nn):
        mat[i,:] = np.power( stencil , i )
    
    ## condition_number = np.linalg.cond(mat, p=-2)
    
    # mat_inv = np.linalg.inv( mat )
    # coeffv  = np.dot( mat_inv , dvec )
    
    if normalize_stencil:
        for i in range(nn):
            dvec[i] /= np.power( stencil_abs_min_nonzero , i )
    
    #coeffv = np.linalg.solve(mat, dvec)
    coeffv = sp.linalg.solve(mat, dvec)
    
    return coeffv

def assemble_1d_fd_coeff_vector_central(x=None, dx=None, edge_stencil='full', acc=4, d=1):
    '''
    assemble FD coefficients for 1D coordinate vector using central discretization
    -----
    d : derivative order
    x : 1D coordinate array
    ...
    '''
    
    ## checks
    
    uniform_grid = False ## tmp
    nx = x.shape[0]
    
    fdc_vec = [] ## vector of finite difference coefficient information to be returned
    
    ## for the d'th derivative with accuracy=acc, the following formula gives the n pts of the (central) stencil
    stencil_npts = 2*int(np.floor((d+1)/2)) - 1 + acc
    
    if not isinstance(stencil_npts, int):
        raise ValueError('stencil_npts must be of type \'int\'')
    if (stencil_npts<3):
        raise ValueError('stencil_npts should be >=3')
    if ((stencil_npts-1)%2 != 0):
        raise ValueError('(stencil_npts-1) should be divisible by 2 (for central stencil)')
    if (stencil_npts > nx):
        raise ValueError('stencil_npts > nx')
    
    if all([ (edge_stencil!='half') , (edge_stencil!='full') ]):
        raise ValueError('edge_stencil=%s not valid. options are: \'full\', \'half\''%str(edge_stencil))
    
    # ===
    
    n_full_central_stencils = nx - stencil_npts + 1
    
    # if ( n_full_central_stencils < 5 ) and not no_warn:
    #     print('\nWARNING\n'+72*'-')
    #     print('n pts with full central stencils = %i (<5)'%n_full_central_stencils)
    #     #print('nx//3=%i'%(nx//3))
    #     print('--> consider reducing acc arg (accuracy order)')
    #     print(72*'-'+'\n')
    
    stencil_width = stencil_npts-1
    sw2           = stencil_width//2
    
    ## left side
    for i in range(0,sw2):
        
        if (edge_stencil=='half'):
            stencil_L = np.arange(-i,sw2+1)
        elif (edge_stencil=='full'):
            stencil_L = np.arange(-i,stencil_width+1-i)
        else:
            raise ValueError('edge_stencil options are: \'full\', \'half\'')
        
        i_range = np.arange( 0 , stencil_L.shape[0] )
        
        if uniform_grid:
            fdc = fd_coeff_calculator( stencil_L , d=d , dx=x )
        else:
            fdc = fd_coeff_calculator( stencil_L , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil_L ] )
    
    ## inner pts
    stencil = np.arange(stencil_npts) - sw2
    if uniform_grid:
        fdc_inner = fd_coeff_calculator( stencil , d=d , dx=x )
    for i in range(sw2,nx-sw2):
        
        i_range  = np.arange(i-sw2,i+sw2+1)
        
        if uniform_grid:
            fdc = fdc_inner
        else:
            fdc = fd_coeff_calculator( stencil , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil ] )
    
    ## right side
    for i in range(nx-sw2,nx):
        
        if (edge_stencil=='half'):
            stencil_R = np.arange(-sw2,nx-i)
        elif (edge_stencil=='full'):
            stencil_R = np.arange(-stencil_width+(nx-i-1),nx-i)
        else:
            raise ValueError('edge_stencil options are: \'full\', \'half\'')
        
        i_range  = np.arange( nx-stencil_R.shape[0] , nx )
        
        if uniform_grid:
            fdc = fd_coeff_calculator( stencil_R , d=d , dx=x )
        else:
            fdc = fd_coeff_calculator( stencil_R , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil_R ] )
    
    
    return fdc_vec

def assemble_1d_fd_coeff_vector_custom(x, stencil_base, d=1):
    '''
    assemble FD coefficients for 1D coordinate vector using a custom stencil index
    -----
    stencil_base : stencil index array, e.g. np.array([-1,0,1,2,3],dtype=np.int32)
    d            : derivative order
    x            : 1D coordinate array
    '''
    
    ## checks
    if not isinstance(x, np.ndarray):
        raise ValueError("x must be type np.ndarray")
    if not (x.ndim==1):
        raise ValueError("x must be 1D")
    if not isinstance(stencil_base, np.ndarray):
        raise ValueError("stencil_base must be type np.ndarray")
    if not (stencil_base.ndim==1):
        raise ValueError("stencil_base must be 1D")
    if not np.issubdtype(stencil_base.dtype, np.integer):
        raise ValueError('stencil_base.dtype not a subdtype of np.integer')
    if (0 not in stencil_base):
        raise ValueError('stencil_base does not contain 0')
    if not np.all(np.diff(stencil_base)==1):
        raise ValueError("stencil_base must have np.all(np.diff(stencil_base)==1")
    
    nx = x.shape[0]            ## size of coord vector
    ss = stencil_base.shape[0] ## size of stencil
    
    if (ss>nx):
        raise ValueError('stencil is larger than the coordinate vector')
    
    i0  = np.where(stencil_base==0)[0][0] ## location of 0 in stencil_base
    niL = i0                              ## n stencil indices to the left (L) of 0
    niR = ss-i0-1                         ## n stencil indices to the right (R) of 0
    
    fdc_vec = [] ## vector of finite difference coefficient information to be returned
    for i in range(nx):
        
        iR = nx-i-1 ## decrementer --> [nx-1...0] --> 0 at last loop index
        
        shift=0
        isL=False
        isR=False
        if (niL>i): ## left edge
            shift = niL-i
            isL=True
        if (niR>iR): ## right edge
            shift = iR-niR
            isR=True
        
        if (isL and isR):
            raise ValueError('something went wrong in assemble_1d_fd_coeff_vector_custom()')
        
        stencil_ = np.copy(stencil_base + shift)
        
        ## the indices corresponding to coordinate vector [x]
        i_range = np.arange(i+stencil_.min(),i+stencil_.max()+1)
        
        ## get actual coefficients
        fdc = fd_coeff_calculator( stencil_, d=d, x=x[i_range] )
        
        ## add all info to output list
        fdc_vec.append( [ fdc , i_range , stencil_ ] )
    
    return fdc_vec

def gradient(u, x=None, d=1, axis=0, acc=6, edge_stencil='full', return_coeffs=False, no_warn=True):
    '''
    Numerical Gradient Approximation Using Finite Differences
    -----
    - calculates stencil given arbitrary accuracy & derivative order
    - handles non-uniform grids
    - accuracy order is only mathematically valid for:
       - uniform coordinate array
       - inner points which have full central stencil
    - handles N-D numpy arrays (gradient performed over axis denoted by axis arg)
    -----
    u    : input array to perform differentiation upon
    x    : coordinate vector (np.ndarray) OR dx (float) in the case of a uniform grid
    d    : derivative order
    axis : axis along which to perform gradient
    acc  : accuracy order (only fully valid for inner points with central stencil on uniform grid)
    -----
    edge_stencil  : type of edge stencil to use ('half','full')
    return_coeffs : if True, then return stencil & coefficient information
    -----
    # stencil_npts : number of index pts in (central) stencil
    #     --> no longer an input
    #     --> using 'acc' (accuracy order) instead and calculating npts from formula
    #     - stencil_npts=3 : stencil=[      -1,0,+1      ]
    #     - stencil_npts=5 : stencil=[   -2,-1,0,+1,+2   ]
    #     - stencil_npts=7 : stencil=[-3,-2,-1,0,+1,+2,+3]
    #     - edges are filled out with appropriate clipping of central stencil
    -----
    turbx.gradient( u , x , d=1 , acc=2 , edge_stencil='half' , axis=0 )
    ...reproduces...
    np.gradient(u, x, edge_order=1, axis=0)
    
    turbx.gradient( u , x , d=1 , acc=2 , edge_stencil='full' , axis=0 )
    ...reproduces...
    np.gradient(u, x, edge_order=2, axis=0)
    -----
    https://en.wikipedia.org/wiki/Finite_difference_coefficient
    https://web.media.mit.edu/~crtaylor/calculator.html
    -----
    Fornberg B. (1988) Generation of Finite Difference Formulas on
    Arbitrarily Spaced Grids, Mathematics of Computation 51, no. 184 : 699-706.
    http://www.ams.org/journals/mcom/1988-51-184/S0025-5718-1988-0935077-0/S0025-5718-1988-0935077-0.pdf
    '''
    
    u = np.asanyarray(u)
    nd = u.ndim
    
    # print('contiguous   : %s'%str( u.data.contiguous   ) )
    # print('C-contiguous : %s'%str( u.data.c_contiguous ) )
    # print('F-contiguous : %s'%str( u.data.f_contiguous ) )
    
    if (nd==0):
        raise ValueError('turbx.gradient() requires input that is at least 1D')
    
    axes = tuple(range(nd))
    
    if not isinstance(axis, int):
        raise ValueError('axis should be of type int')
    if (axis not in axes):
        raise ValueError('axis=%i is not valid for array with u.ndim=%s'%(axis,str(u.ndim)))
    
    nx = u.shape[axis] ## size of axis over which gradient will be performed
    
    if (nx<3):
        raise ValueError('nx<3')
    
    if (x is not None):
        if isinstance(x, float):
            if (x<=0.):
                raise ValueError('if x is a float it should be >0.')
        elif isinstance(x, int):
            x = float(x)
        elif isinstance(x, np.ndarray):
            if (x.ndim!=1):
                raise ValueError('x should be 1D if it is of type np.ndarray')
            if (x.shape[0]!=nx):
                raise ValueError('size of x does not match data axis specified')
            if (not np.all(np.diff(x) > 0.)) and (not np.all(np.diff(x) < 0.)):
                    raise AssertionError('x is not monotonically increasing/decreasing')
            
            ## optimization: check if x is actually uniformly spaced, in which case x=Δx
            dx0 = x[1]-x[0]
            if np.all(np.isclose(np.diff(x), dx0, rtol=1e-12)): 
                #print('turbx.gradient() : x arr with x.shape=%s seems like it is actually uniformly spaced. applying x=%0.8e'%(str(x.shape),dx0))
                x = dx0
        
        else:
            raise ValueError('x should be a 1D np.ndarray or float')
    else:
        x = 1. ## if x not provided, assume uniform unit coordinate vector
    
    if isinstance(x, float):
        uniform_grid = True
    elif isinstance(x, np.ndarray):
        uniform_grid = False
    else:
        raise ValueError('turbx.gradient() : this should never happen... check!')
    
    if not isinstance(d, int):
        raise ValueError('d (derivative order) should be of type int')
    if not (d>0):
        raise ValueError('d (derivative order) should be >0')
    
    if not isinstance(acc, int):
        raise ValueError('acc (accuracy order) should be of type int')
    if not (acc>=2):
        raise ValueError('acc (accuracy order) should be >=2')
    if (acc%2!=0):
        raise ValueError('acc (accuracy order) should be an integer multiple of 2')
    
    ## for the d'th derivative with accuracy=acc, the following formula gives the n pts of the (central) stencil
    stencil_npts = 2*int(np.floor((d+1)/2)) - 1 + acc
    
    if not isinstance(stencil_npts, int):
        raise ValueError('stencil_npts must be of type \'int\'')
    if (stencil_npts<3):
        raise ValueError('stencil_npts should be >=3')
    if ((stencil_npts-1)%2 != 0):
        raise ValueError('(stencil_npts-1) should be divisible by 2 (for central stencil)')
    if (stencil_npts > nx):
        raise ValueError('stencil_npts > nx')
    
    if all([ (edge_stencil!='half') , (edge_stencil!='full') ]):
        raise ValueError('edge_stencil=%s not valid. options are: \'full\', \'half\''%str(edge_stencil))
    
    # ===
    
    n_full_central_stencils = nx - stencil_npts + 1
    
    if ( n_full_central_stencils < 5 ) and not no_warn:
        print('\nWARNING\n'+72*'-')
        print('n pts with full central stencils = %i (<5)'%n_full_central_stencils)
        #print('nx//3=%i'%(nx//3))
        print('--> consider reducing acc arg (accuracy order)')
        print(72*'-'+'\n')
    
    stencil_width = stencil_npts-1
    sw2           = stencil_width//2
    
    # === build up stencil & coefficients vector
    
    fdc_vec = [] ## vector of finite difference coefficient information
    
    ## left side
    for i in range(0,sw2):
        
        ## full
        stencil_L = np.arange(-i,stencil_width+1-i)
        
        ## half
        if (edge_stencil=='half') and not all([(acc==2),(d>=2)]):
            stencil_L = np.arange(-i,sw2+1)
        
        i_range = np.arange( 0 , stencil_L.shape[0] )
        
        if uniform_grid:
            fdc = fd_coeff_calculator( stencil_L , d=d , dx=x )
        else:
            fdc = fd_coeff_calculator( stencil_L , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil_L ] )
    
    ## inner pts
    stencil = np.arange(stencil_npts) - sw2
    if uniform_grid:
        fdc_inner = fd_coeff_calculator( stencil , d=d , dx=x )
    for i in range(sw2,nx-sw2):
        
        i_range  = np.arange(i-sw2,i+sw2+1)
        
        if uniform_grid:
            fdc = fdc_inner
        else:
            fdc = fd_coeff_calculator( stencil , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil ] )
    
    ## right side
    for i in range(nx-sw2,nx):
        
        ## full
        stencil_R = np.arange(-stencil_width+(nx-i-1),nx-i)
        
        ## half
        if (edge_stencil=='half') and not all([(acc==2),(d>=2)]):
            stencil_R = np.arange(-sw2,nx-i)
        
        i_range  = np.arange( nx-stencil_R.shape[0] , nx )
        
        if uniform_grid:
            fdc = fd_coeff_calculator( stencil_R , d=d , dx=x )
        else:
            fdc = fd_coeff_calculator( stencil_R , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil_R ] )
    
    # === debug
    
    # print('i_range')
    # for fdc_vec_ in fdc_vec:
    #     print(fdc_vec_[1])
    # print('')
    # print('stencil')
    # for fdc_vec_ in fdc_vec:
    #     print(fdc_vec_[2])
    
    # === evaluate gradient
    
    if (nd==1): ## 1D
        
        u_ddx = np.zeros_like(u)
        for i in range(len(fdc_vec)):
            fdc, i_range, stencil = fdc_vec[i]
            u_ddx[i] = np.dot( fdc , u[i_range] )
    
    else: ## N-D
        
        ## the order in memory of the incoming data
        if u.data.contiguous:
            if u.data.c_contiguous:
                order='C'
            elif u.data.f_contiguous:
                order='F'
            else:
                raise ValueError
        else:
            order='C'
        
        ## if the array is C-ordered, use axis=0 as the axis to shift to
        ## if the array is F-ordered, use axis=-1 as the axis to shift to
        if order=='C':
            shift_pos=0 ## 0th axis
        elif order=='F':
            shift_pos=nd-1 ## last axis
        
        ## shift gradient axis
        u = np.swapaxes(u, axis, shift_pos)
        shape_new = u.shape
        if (shift_pos==0):
            size_all_but_ax = np.prod(np.array(shape_new)[1:])
        elif (shift_pos==nd-1):
            size_all_but_ax = np.prod(np.array(shape_new)[:-1])
        else:
            raise ValueError
        
        ## reshape N-D to 2D
        ## --> shift_pos=0    : gradient axis is 0, all other axes are flattened on axis=1)
        ## --> shift_pos=last : ...
        if (shift_pos==0):
            u = np.reshape(u, (nx, size_all_but_ax), order=order)
        elif (shift_pos==nd-1):
            u = np.reshape(u, (size_all_but_ax, nx), order=order)
        else:
            raise ValueError
        
        u_ddx = np.zeros_like(u,order=order)
        for i in range(nx):
            fdc, i_range, stencil = fdc_vec[i]
            ia=min(i_range)
            ib=max(i_range)+1
            if (shift_pos==0):
                u_ddx[i,:] = np.einsum('ij,i->j', u[ia:ib,:], fdc)
            else:
                u_ddx[:,i] = np.einsum('ji,i->j', u[:,ia:ib], fdc)
        
        ## reshape 2D back to original N-D
        u_ddx = np.reshape(u_ddx, shape_new, order=order)
        
        ## shift gradient axis back to original position
        u_ddx = np.swapaxes(u_ddx, shift_pos, axis)
    
    ## the original data array should have been de-referenced during this func
    u = None; del u
    
    if return_coeffs:
        return u_ddx, fdc_vec
    else:
        return u_ddx

def get_metric_tensor_3d(x3d, y3d, z3d, acc=2, edge_stencil='full', **kwargs):
    '''
    compute the grid metric tensor (inverse of grid Jacobian) for a 3D grid
    -----
    Computational Fluid Mechanics and Heat Transfer (2012) Pletcher, Tannehill, Anderson
    p.266-270, 335-337, 652
    '''
    
    verbose = kwargs.get('verbose',False)
    no_warn = kwargs.get('no_warn',False)
    
    if not isinstance(x3d, np.ndarray):
        raise ValueError('x3d should be of type np.ndarray')
    if not isinstance(y3d, np.ndarray):
        raise ValueError('y3d should be of type np.ndarray')
    if not isinstance(z3d, np.ndarray):
        raise ValueError('z3d should be of type np.ndarray')
    
    if (x3d.ndim!=3):
        raise ValueError('x3d should have ndim=3 (xyz)')
    if (y3d.ndim!=3):
        raise ValueError('y3d should have ndim=3 (xyz)')
    if (z3d.ndim!=3):
        raise ValueError('z3d should have ndim=3 (xyz)')
    
    if not (x3d.shape==y3d.shape):
        raise ValueError('x3d.shape!=y3d.shape')
    if not (y3d.shape==z3d.shape):
        raise ValueError('y3d.shape!=z3d.shape')
    
    nx,ny,nz = x3d.shape
    
    ## the 'computational' grid (unit Cartesian)
    ## --> [x_comp,y_comp,z_comp ]= [ξ,η,ζ] = [q1,q2,q3]
    #x_comp = np.arange(nx, dtype=np.float64)
    #y_comp = np.arange(ny, dtype=np.float64)
    #z_comp = np.arange(nz, dtype=np.float64)
    x_comp = 1.
    y_comp = 1.
    z_comp = 1.
    
    # === get Jacobian :: ∂(x,y,z)/∂(q1,q2,q3)
    
    t_start = timeit.default_timer()
    
    dxdx = gradient(x3d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydx = gradient(y3d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dzdx = gradient(z3d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    ##
    dxdy = gradient(x3d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydy = gradient(y3d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dzdy = gradient(z3d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    ##
    dxdz = gradient(x3d, z_comp, axis=2, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydz = gradient(y3d, z_comp, axis=2, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dzdz = gradient(z3d, z_comp, axis=2, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    
    J = np.stack((np.stack((dxdx, dydx, dzdx), axis=3),
                  np.stack((dxdy, dydy, dzdy), axis=3),
                  np.stack((dxdz, dydz, dzdz), axis=3)), axis=4)
    
    t_delta = timeit.default_timer() - t_start
    if verbose: tqdm.write( even_print('get J','%0.3f [s]'%(t_delta,), s=True) )
    
    # === get metric tensor M = J^-1 = ∂(q1,q2,q3)/∂(x,y,z) = ∂(ξ,η,ζ)/∂(x,y,z)
    
    if False: ## method 1
        
        t_start = timeit.default_timer()
        
        M = np.linalg.inv(J)
        
        # M_bak = np.copy(M)
        # for i in range(nx):
        #     for j in range(ny):
        #         for k in range(nz):
        #             M[i,j,k,:,:] = sp.linalg.inv( J[i,j,k,:,:] )
        # np.testing.assert_allclose(M_bak, M, atol=1e-12, rtol=1e-12)
        # print('check passed')
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get M','%0.3f [s]'%(t_delta,), s=True) )
    
    if True: ## method 2
        
        if ('M' in locals()):
            M_bak = np.copy(M)
            M = None; del M
        
        t_start = timeit.default_timer()
        
        a = J[:,:,:,0,0]
        b = J[:,:,:,0,1]
        c = J[:,:,:,0,2]
        d = J[:,:,:,1,0]
        e = J[:,:,:,1,1]
        f = J[:,:,:,1,2]
        g = J[:,:,:,2,0]
        h = J[:,:,:,2,1]
        i = J[:,:,:,2,2]
        
        # a = J[:,:,:,0,0]
        # b = J[:,:,:,1,0]
        # c = J[:,:,:,2,0]
        # d = J[:,:,:,0,1]
        # e = J[:,:,:,1,1]
        # f = J[:,:,:,2,1]
        # g = J[:,:,:,0,2]
        # h = J[:,:,:,1,2]
        # i = J[:,:,:,2,2]
        
        I = ( + a*e*i
              + b*f*g
              + c*d*h
              - c*e*g
              - b*d*i
              - a*f*h )
        
        M = np.zeros((nx,ny,nz,3,3), dtype=np.float64)
        M[:,:,:,0,0] = +( dydy * dzdz - dydz * dzdy ) / I ## ξ_x
        M[:,:,:,0,1] = -( dxdy * dzdz - dxdz * dzdy ) / I ## ξ_y
        M[:,:,:,0,2] = +( dxdy * dydz - dxdz * dydy ) / I ## ξ_z
        M[:,:,:,1,0] = -( dydx * dzdz - dydz * dzdx ) / I ## η_x
        M[:,:,:,1,1] = +( dxdx * dzdz - dxdz * dzdx ) / I ## η_y
        M[:,:,:,1,2] = -( dxdx * dydz - dxdz * dydx ) / I ## η_z
        M[:,:,:,2,0] = +( dydx * dzdy - dydy * dzdx ) / I ## ζ_x
        M[:,:,:,2,1] = -( dxdx * dzdy - dxdy * dzdx ) / I ## ζ_y
        M[:,:,:,2,2] = +( dxdx * dydy - dxdy * dydx ) / I ## ζ_z
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get M','%0.3f [s]'%(t_delta,), s=True) )
        
        if ('M_bak' in locals()):
            np.testing.assert_allclose(M[:,:,:,0,0], M_bak[:,:,:,0,0], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_x')
            np.testing.assert_allclose(M[:,:,:,0,1], M_bak[:,:,:,0,1], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_y')
            np.testing.assert_allclose(M[:,:,:,0,2], M_bak[:,:,:,0,2], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_z')
            np.testing.assert_allclose(M[:,:,:,1,0], M_bak[:,:,:,1,0], atol=1e-14, rtol=1e-14)
            print('check passed: η_x')
            np.testing.assert_allclose(M[:,:,:,1,1], M_bak[:,:,:,1,1], atol=1e-14, rtol=1e-14)
            print('check passed: η_y')
            np.testing.assert_allclose(M[:,:,:,1,2], M_bak[:,:,:,1,2], atol=1e-14, rtol=1e-14)
            print('check passed: η_z')
            np.testing.assert_allclose(M[:,:,:,2,0], M_bak[:,:,:,2,0], atol=1e-14, rtol=1e-14)
            print('check passed: ζ_x')
            np.testing.assert_allclose(M[:,:,:,2,1], M_bak[:,:,:,2,1], atol=1e-14, rtol=1e-14)
            print('check passed: ζ_y')
            np.testing.assert_allclose(M[:,:,:,2,2], M_bak[:,:,:,2,2], atol=1e-14, rtol=1e-14)
            print('check passed: ζ_z')
            np.testing.assert_allclose(M, M_bak, atol=1e-14, rtol=1e-14)
            print('check passed: M')
    
    return M

def get_metric_tensor_2d(x2d, y2d, acc=2, edge_stencil='full', **kwargs):
    '''
    compute the grid metric tensor (inverse of grid Jacobian) for a 2D grid
    -----
    Computational Fluid Mechanics and Heat Transfer (2012) Pletcher, Tannehill, Anderson
    p.266-270, 335-337, 652
    '''
    
    verbose = kwargs.get('verbose',False)
    no_warn = kwargs.get('no_warn',False)
    
    if not isinstance(x2d, np.ndarray):
        raise ValueError('x2d should be of type np.ndarray')
    if not isinstance(y2d, np.ndarray):
        raise ValueError('y2d should be of type np.ndarray')
    
    if (x2d.ndim!=2):
        raise ValueError('x2d should have ndim=2 (xy)')
    if (y2d.ndim!=2):
        raise ValueError('y2d should have ndim=2 (xy)')
    
    if not (x2d.shape==y2d.shape):
        raise ValueError('x2d.shape!=y2d.shape')
    
    nx,ny = x2d.shape
    
    ## the 'computational' grid (unit Cartesian)
    ## --> [x_comp,y_comp]= [ξ,η] = [q1,q2]
    #x_comp = np.arange(nx, dtype=np.float64)
    #y_comp = np.arange(ny, dtype=np.float64)
    x_comp = 1.
    y_comp = 1.
    
    # === get Jacobian :: ∂(x,y)/∂(q1,q2)
    
    t_start = timeit.default_timer()
    
    dxdx = gradient(x2d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydx = gradient(y2d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dxdy = gradient(x2d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydy = gradient(y2d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    
    J = np.stack((np.stack((dxdx, dydx), axis=2),
                  np.stack((dxdy, dydy), axis=2)), axis=3)
    
    t_delta = timeit.default_timer() - t_start
    if verbose: tqdm.write( even_print('get J','%0.3f [s]'%(t_delta,), s=True) )
    
    # === get metric tensor M = J^-1 = ∂(q1,q2)/∂(x,y) = ∂(ξ,η)/∂(x,y)
    
    if False: ## method 1
        
        t_start = timeit.default_timer()
        
        M = np.linalg.inv(J)
        
        # M_bak = np.copy(M)
        # M = np.zeros((nx,ny,2,2),dtype=np.float64)
        # for i in range(nx):
        #     for j in range(ny):
        #         M[i,j,:,:] = sp.linalg.inv( J[i,j,:,:] )
        # np.testing.assert_allclose(M_bak, M, atol=1e-12, rtol=1e-12)
        # print('check passed')
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get M','%0.3f [s]'%(t_delta,), s=True) )
    
    if True: ## method 2
        
        if ('M' in locals()):
            M_bak = np.copy(M)
            M = None; del M
        
        t_start = timeit.default_timer()
        
        ## Jacobian determinant
        I = dxdx*dydy - dydx*dxdy
        
        # I_bak = np.copy(I)
        # I = None; del I
        # I = np.linalg.det(J)
        # np.testing.assert_allclose(I, I_bak, atol=1e-14, rtol=1e-14)
        # print('check passed')
        
        M = np.zeros((nx,ny,2,2), dtype=np.float64)
        M[:,:,0,0] = +dydy / I ## ξ_x
        M[:,:,0,1] = -dxdy / I ## ξ_y
        M[:,:,1,0] = -dydx / I ## η_x
        M[:,:,1,1] = +dxdx / I ## η_y
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get M','%0.3f [s]'%(t_delta,), s=True) )
        
        if ('M_bak' in locals()):
            np.testing.assert_allclose(M[:,:,0,0], M_bak[:,:,0,0], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_x')
            np.testing.assert_allclose(M[:,:,0,1], M_bak[:,:,0,1], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_y')
            np.testing.assert_allclose(M[:,:,1,0], M_bak[:,:,1,0], atol=1e-14, rtol=1e-14)
            print('check passed: η_x')
            np.testing.assert_allclose(M[:,:,1,1], M_bak[:,:,1,1], atol=1e-14, rtol=1e-14)
            print('check passed: η_y')
            np.testing.assert_allclose(M, M_bak, atol=1e-14, rtol=1e-14)
            print('check passed: M')
    
    return M

def get_grid_quality_metrics_2d(x2d, y2d, **kwargs):
    '''
    get 2d grid quality metrics
    -----
    - skew
    - avg diagonal length (not yet implemented)
    -----
    ## https://coreform.com/cubit_help/mesh_generation/mesh_quality_assessment/quadrilateral_metrics.htm
    '''
    
    verbose = kwargs.get('verbose',True)
    
    if not isinstance(x2d, np.ndarray):
        raise ValueError('x2d should be of type np.ndarray')
    if not isinstance(y2d, np.ndarray):
        raise ValueError('y2d should be of type np.ndarray')
    
    if (x2d.ndim!=2):
        raise ValueError('x2d should have ndim=2 (xy)')
    if (y2d.ndim!=2):
        raise ValueError('y2d should have ndim=2 (xy)')
    
    if not (x2d.shape==y2d.shape):
        raise ValueError('x2d.shape!=y2d.shape')
    
    nx,ny = x2d.shape
    xy2d = np.stack((x2d,y2d), axis=-1)
    
    '''
    there are very likely ways to vectorize the functions below
    '''
    
    ds1avg = np.zeros((nx,ny), dtype=np.float64)
    ds2avg = np.zeros((nx,ny), dtype=np.float64)
    
    if verbose: progress_bar = tqdm(total=(nx-1)*(ny-1), ncols=100, desc='grid ds', leave=False, file=sys.stdout)
    for i in range(nx):
        for j in range(ny):
            
            ## W/E ds
            if (i==0):
                dsW = None
                dsE = sp.linalg.norm( xy2d[i+1,j,:] - xy2d[i,  j,:], ord=2)
                ds1avg[i,j] = dsE
            elif (i==nx-1):
                dsW = sp.linalg.norm( xy2d[i,  j,:] - xy2d[i-1,j,:], ord=2)
                dsE = None
                ds1avg[i,j] = dsW
            else:
                dsW = sp.linalg.norm( xy2d[i,  j,:] - xy2d[i-1,j,:], ord=2)
                dsE = sp.linalg.norm( xy2d[i+1,j,:] - xy2d[i,  j,:], ord=2)
                ds1avg[i,j] = 0.5*(dsW+dsE)
            
            ## S/N ds
            if (j==0):
                dsS = None
                dsN = sp.linalg.norm( xy2d[i,j+1,:] - xy2d[i,  j,:], ord=2)
                ds2avg[i,j] = dsN
            elif (j==ny-1):
                dsS = sp.linalg.norm( xy2d[i,  j,:] - xy2d[i,j-1,:], ord=2)
                dsN = None
                ds2avg[i,j] = dsS
            else:
                dsS = sp.linalg.norm( xy2d[i,  j,:] - xy2d[i,j-1,:], ord=2)
                dsN = sp.linalg.norm( xy2d[i,j+1,:] - xy2d[i,  j,:], ord=2)
                ds2avg[i,j] = 0.5*(dsS+dsN)
            
            if verbose: progress_bar.update()
    if verbose: progress_bar.close()
    
    grid_inner_angle_cosines = np.zeros((nx-1,ny-1,4), dtype=np.float64)
    if verbose: progress_bar = tqdm(total=(nx-1)*(ny-1), ncols=100, desc='grid skew angle', leave=False, file=sys.stdout)
    for i in range(nx-1):
        for j in range(ny-1):
            ## (SW-->NW)::(SW-->SE)
            v1 = xy2d[i,j+1,:] - xy2d[i,j,:]
            v2 = xy2d[i+1,j,:] - xy2d[i,j,:]
            v1mag = sp.linalg.norm(v1, ord=2)
            v2mag = sp.linalg.norm(v2, ord=2)
            grid_inner_angle_cosines[i,j,0] = np.abs(np.dot(v1,v2)/(v1mag*v2mag))
            ## (SE-->SW)::(SE-->NE)
            v1 = xy2d[i,j,:]     - xy2d[i+1,j,:]
            v2 = xy2d[i+1,j+1,:] - xy2d[i+1,j,:]
            v1mag = sp.linalg.norm(v1, ord=2)
            v2mag = sp.linalg.norm(v2, ord=2)
            grid_inner_angle_cosines[i,j,1] = np.abs(np.dot(v1,v2)/(v1mag*v2mag))
            ## (NE-->NW)::(NE-->SE)
            v1 = xy2d[i,j+1,:] - xy2d[i+1,j+1,:]
            v2 = xy2d[i+1,j,:] - xy2d[i+1,j+1,:]
            v1mag = sp.linalg.norm(v1, ord=2)
            v2mag = sp.linalg.norm(v2, ord=2)
            grid_inner_angle_cosines[i,j,2] = np.abs(np.dot(v1,v2)/(v1mag*v2mag))
            ## (NW-->NE)::(NW-->SW)
            v1 = xy2d[i+1,j+1,:] - xy2d[i,j+1,:]
            v2 = xy2d[i,j,:]     - xy2d[i,j+1,:]
            v1mag = sp.linalg.norm(v1, ord=2)
            v2mag = sp.linalg.norm(v2, ord=2)
            grid_inner_angle_cosines[i,j,3] = np.abs(np.dot(v1,v2)/(v1mag*v2mag))
            ##
            if verbose: progress_bar.update()
    if verbose: progress_bar.close()
    
    skew = np.max(grid_inner_angle_cosines, axis=-1)
    
    grid_quality_dict = { 'skew':skew, 'ds1avg':ds1avg, 'ds2avg':ds2avg }
    return grid_quality_dict

def smoothstep(N, a=None, b=None, order=3, mode='index'):
    '''
    a smoothed step function that is differentiable (order) times
    - actually starts/ends at identically 0/1 --> non-asymptotic (unlike e.g. sigmoid)
    - based on Hermite polynomials
    -----
    index : N should be an int... returns funcs on array of size N (independent of grid)
    coord_vec : N should be a numpy array describing coordinates (a,b) are bounds of step in coord vec
    '''
    if (mode=='index'):
        if (b is None):
            b=N-1
        if (a is None):
            a=0
        if not all([isinstance(N,int),isinstance(a,int),isinstance(b,int)]):
            raise ValueError('N,a,b must all be type int for mode=\'index\'')
        x = np.arange(0,N,1,dtype=np.float64)
        a = x[a]
        b = x[b]
    elif (mode=='coord_vec'):
        if not (isinstance(N,np.ndarray)):
            raise ValueError('N should be a numpy array.')
        x = np.copy(N)
        if (b is None):
            b=x[-1]
        if (a is None):
            a=x[0]
    else:
        raise ValueError('mode=\'%s\' not a valid input. options are: \'index\' \'coord_vec\' '%str(mode))
    
    x = np.clip((x-a)/(b-a),0,1)
    f = np.zeros(x.size, dtype=np.float64)
    for n in range(0,order+1):
        f += sp.special.comb(order+n,n) * sp.special.comb(2*order+1, order-n) * (-x) ** n
    f *= x**(order+1)
    f_inv = 1. - f
    return f, f_inv

def stretch_1d_cluster_ends(x, max_growth_rate=1.02, max_ratio=3, inverse=False, n_uniform=2):
    '''
    stretch 1D grid, cluster at ends
    '''
    
    x = np.copy(x)
    
    ## backup for check later
    xmin = x.min()
    xmax = x.max()
    
    nx = x.shape[0]
    
    if (nx<10):
        raise ValueError
    
    ## scaling func range [0,1]
    f_scale = np.zeros(nx-1,dtype=np.float64)
    f_scale[n_uniform:-n_uniform]  = signal.windows.parzen(nx-1-2*n_uniform)
    #f_scale[n_uniform:-n_uniform]  = signal.windows.hamming(nx-1-2*n_uniform)
    #f_scale[n_uniform:-n_uniform]  = signal.windows.hann(nx-1-2*n_uniform)
    f_scale -= f_scale.min()
    f_scale /= f_scale.max()
    
    ## coarser at ends (rather than finer at ends)
    if inverse:
        f_scale = 1. - f_scale
    
    # === solve for max dx ratio based on max allowable growth rate
    
    def __f_opt_max_growth_rate(dx_ratio_max, x,max_growth_rate_tgt):
        '''
        given a max growth rate (e.g. 1.02), find the max dx_ratio
        '''
        x = np.copy(x)
        nx = x.shape[0]
        
        dx_fac = 1 + f_scale*(dx_ratio_max-1) ## scaling func range [1,dx_ratio_max]
        
        dx_  = np.diff(x,n=1)
        dx_ *= dx_fac
        x_   = np.cumsum(np.concatenate(([0.],dx_)))
        
        growth_rate = np.zeros((nx-3,), dtype=np.float64)
        for i in range(1,nx-2):
            dxL=dx_[i]
            dxR=dx_[i+1]
            growth_rate[i-1] = max( dxL/dxR , dxR/dxL )
        max_growth_rate = growth_rate.max()
        
        root = np.abs( max_growth_rate - max_growth_rate_tgt )
        return root
    
    sol = sp.optimize.least_squares(fun=__f_opt_max_growth_rate,
                                    x0=2,
                                    xtol=1e-14,
                                    ftol=1e-14,
                                    method='trf',
                                    max_nfev=int(1e4),
                                    args=(x,max_growth_rate,),
                                    bounds=(1.,999999.))
    if not sol.success:
        raise ValueError
    
    dx_ratio_max = sol.x
    
    ## if the ratio is > the input 'max_ratio', take 'max_ratio' instead
    if (dx_ratio_max>max_ratio):
        dx_ratio_max = max_ratio
        assert_exact_growth_rate = False
        assert_max_growth_rate = True
    else:
        assert_exact_growth_rate = True
        assert_max_growth_rate = False
    
    # ===
    
    dx_fac = 1 + f_scale*(dx_ratio_max-1) ## scaling func range [1,dx_ratio_max]
    
    dx_  = np.diff(x,n=1)
    dx_ *= dx_fac
    x_   = np.cumsum(np.concatenate(([0.],dx_)))
    scf  = xmax / x_.max() ## scale factor
    
    dx = np.diff(x,n=1)
    x  = np.cumsum(np.concatenate(([0.],dx*dx_fac*scf)))
    dx = np.diff(x,n=1)
    
    ## recalculate growth rate, do checks
    aa = dx[1:]/dx[:-1]
    bb = dx[:-1]/dx[1:]
    cc = np.maximum(aa,bb)
    max_growth_rate_ = cc.max()
    if assert_exact_growth_rate:
        np.testing.assert_allclose( max_growth_rate_, max_growth_rate, rtol=1e-6, atol=1e-6 )
    if assert_max_growth_rate:
        if ( max_growth_rate > max_growth_rate ):
            raise AssertionError
    
    ## assert min/max stayed the same
    np.testing.assert_allclose(x.max(), xmax, rtol=1e-14, atol=1e-14)
    np.testing.assert_allclose(x.min(), xmin, rtol=1e-14, atol=1e-14)
    
    # ===
    
    if False: ## plot : debug : grid dx_fac
        plt.close('all')
        fig1 = plt.figure(figsize=(6,6/2), dpi=200)
        ax1 = plt.gca()
        ax1.tick_params(axis='x', which='both', direction='out')
        ax1.tick_params(axis='y', which='both', direction='out')
        ##
        #ln1, = ax1.plot(np.arange(dx.shape[0]), dx, c=red, linestyle='None', marker='o', markersize=1.0, zorder=20)
        ln1, = ax1.plot(np.arange(dx.shape[0]), dx_fac*scf, c='blue', linestyle='None', marker='o', markersize=1.0, zorder=20)
        ##
        fig1.tight_layout(pad=0.25)
        fig1.tight_layout(pad=0.25)
        dpi_out = 2160/plt.gcf().get_size_inches()[1]
        #fig1.savefig('dx_fac.png', dpi=dpi_out)
        plt.show()
    
    if False: ## plot : debug : grid dx_fac
        plt.close('all')
        fig1 = plt.figure(figsize=(6,6/2), dpi=200)
        ax1 = plt.gca()
        ax1.tick_params(axis='x', which='both', direction='out')
        ax1.tick_params(axis='y', which='both', direction='out')
        ##
        ln1, = ax1.plot(np.arange(dx.shape[0]), dx, c='blue', linestyle='None', marker='o', markersize=1.0, zorder=20)
        ##
        fig1.tight_layout(pad=0.25)
        fig1.tight_layout(pad=0.25)
        dpi_out = 2160/plt.gcf().get_size_inches()[1]
        #fig1.savefig('dx.png', dpi=dpi_out)
        plt.show()
    
    return x

def time_integrate_2d(u,v, x2d,y2d, xy_pts, dt,nt, uchar=1.,lchar=1., p=None, bounds_check=True):
    '''
    integrate [x,y] paths for a single [u,v] field
    '''
    
    ## check
    ## ...
    
    xy_pts = np.copy(xy_pts)
    
    t = dt*np.arange(nt,dtype=np.float64)
    
    bwdfwd = True
    if bwdfwd:
        
        nt_nom = nt
        nt = 1 + 2*(nt-1)
        tL = np.copy( np.flip(-t[1:]) )
        tR = np.copy( t[1:] )
        t  = np.concatenate( [tL,np.array([0.],dtype=np.float64),tR] , casting='no' )
        
        ti_fwd = np.arange(nt_nom-1,nt)
        ti_bwd = np.flip(np.arange(0,nt_nom))
    
    ## construct polygon object using matplotlib
    nx,ny = x2d.shape
    
    if bounds_check:
        n_edge_pts = 2*nx + 2*(ny-2)
        n_edge_faces = n_edge_pts
        poly = np.zeros((n_edge_faces,2),dtype=np.float64)
        ii=-1
        for j in range(ny): ## W
            ii+=1
            poly[ii,0] = x2d[0,j]
            poly[ii,1] = y2d[0,j]
        for i in range(1,nx-1): ## N
            ii+=1
            poly[ii,0] = x2d[i,-1]
            poly[ii,1] = y2d[i,-1]
        for j in range(ny): ## E
            ii+=1
            poly[ii,0] = x2d[-1,-(j+1)]
            poly[ii,1] = y2d[-1,-(j+1)]
        for i in range(1,nx-1): ## S
            ii+=1
            poly[ii,0] = x2d[-(i+1),0]
            poly[ii,1] = y2d[-(i+1),0]
    
    ## a polygon/path object
    if bounds_check:
        poly_obj = mpl.path.Path(poly,closed=False)
    
    ## interpolant callables
    
    f_u = sp.interpolate.CloughTocher2DInterpolator(points=(x2d.ravel(), y2d.ravel()),
                                                            values=u.ravel(),
                                                            fill_value=np.nan )
    
    f_v = sp.interpolate.CloughTocher2DInterpolator(points=(x2d.ravel(), y2d.ravel()),
                                                            values=v.ravel(),
                                                            fill_value=np.nan )
    
    ## f_u = sp.interpolate.LinearNDInterpolator(points=(x2d.ravel(), y2d.ravel()),
    ##                                                         values=u.ravel(),
    ##                                                         fill_value=np.nan )
    ## 
    ## f_v = sp.interpolate.LinearNDInterpolator(points=(x2d.ravel(), y2d.ravel()),
    ##                                                         values=v.ravel(),
    ##                                                         fill_value=np.nan )
    
    n_particles = xy_pts.shape[0]
    pnum = np.arange(n_particles, dtype=np.int64)
    
    scalars = [ 'x','y',
                'u','v', #'p',
                't','id',
              ]
    
    scalars_dtype = [ np.float64, np.float64, 
                      np.float64, np.float64, #np.float64,
                      np.float64, np.int64,
                    ]
    
    data = np.zeros(shape=(n_particles,nt), dtype={'names':scalars, 'formats':scalars_dtype})
    
    ## fill in IDs
    for ti in range(nt):
        data['id'][:,ti] = pnum
    
    xy_pts_orig = np.copy(xy_pts)
    
    ## convect (forwards)
    for ti in ti_fwd:
        tt = t[ti]
        
        ## store position
        data['x'][:,ti] = np.copy( xy_pts[:,0] )
        data['y'][:,ti] = np.copy( xy_pts[:,1] )
        
        ## store time
        data['t'][:,ti] = tt
        
        ## interpolate velocity
        u_pts = f_u( xy_pts[:,0], xy_pts[:,1] )
        v_pts = f_v( xy_pts[:,0], xy_pts[:,1] )
        
        data['u'][:,ti] = np.copy( u_pts )
        data['v'][:,ti] = np.copy( v_pts )
        
        ## convect (Euler 1D)
        xy_pts[:,0] += u_pts*dt
        xy_pts[:,1] += v_pts*dt
        
        ## test if contained after convect
        if bounds_check:
            contained = np.zeros((n_particles,), dtype=np.int32)
            for pi in range(n_particles):
                in_poly = poly_obj.contains_point(xy_pts[pi,:])
                if in_poly:
                    contained[pi] = 1
            ii_nan = np.where(contained==0)
            xy_pts[ii_nan,:] = np.nan
    
    ## reset position before bwd interpolation
    xy_pts = np.copy(xy_pts_orig)
    
    ## convect (backwards)
    for ti in ti_bwd:
        tt = t[ti]
        
        ## store position
        data['x'][:,ti] = np.copy( xy_pts[:,0] )
        data['y'][:,ti] = np.copy( xy_pts[:,1] )
        
        ## store time
        data['t'][:,ti] = tt
        
        ## interpolate velocity
        u_pts = f_u( xy_pts[:,0], xy_pts[:,1] )
        v_pts = f_v( xy_pts[:,0], xy_pts[:,1] )
        
        data['u'][:,ti] = np.copy( u_pts )
        data['v'][:,ti] = np.copy( v_pts )
        
        ## convect (backwards)
        xy_pts[:,0] -= u_pts*dt
        xy_pts[:,1] -= v_pts*dt
        
        ## test if contained after convect
        if bounds_check:
            contained = np.zeros((n_particles,), dtype=np.int32)
            for pi in range(n_particles):
                in_poly = poly_obj.contains_point(xy_pts[pi,:])
                if in_poly:
                    contained[pi] = 1
            ii_nan = np.where(contained==0)
            xy_pts[ii_nan,:] = np.nan
    
    return data

# csys
# ======================================================================

def rect_to_cyl(xyz,**kwargs):
    '''
    convert [x,y,<z>] to [θ,r,<z>]
    '''
    
    cx = kwargs.get('cx',0.)
    cy = kwargs.get('cy',0.)
    
    trz = np.zeros_like(xyz)
    
    if (xyz.ndim==1) and (xyz.shape[-1]==2): ## a single point, shape=(2,)
        xx = xyz[0]-cx
        yy = xyz[1]-cy
        trz[0] = np.arctan2(yy,xx)
        trz[1] = np.sqrt(xx**2 + yy**2)
    
    elif (xyz.ndim==1) and (xyz.shape[-1]==3): ## a single point, shape=(3,)
        xx = xyz[0]-cx
        yy = xyz[1]-cy
        trz[0] = np.arctan2(yy,xx)
        trz[1] = np.sqrt(xx**2 + yy**2)
        trz[2] = xyz[2]
    
    elif (xyz.ndim==2) and (xyz.shape[-1]==2): ## a 1D vector of 2D points, shape=(N,2)
        xx = xyz[:,0]-cx
        yy = xyz[:,1]-cy
        trz[:,0] = np.arctan2(yy,xx)
        trz[:,1] = np.sqrt(xx**2 + yy**2)
    
    elif (xyz.ndim==2) and (xyz.shape[-1]==3): ## a 1D vector of 3D points, shape=(N,3)
        xx = xyz[:,0]-cx
        yy = xyz[:,1]-cy
        trz[:,0] = np.arctan2(yy,xx)
        trz[:,1] = np.sqrt(xx**2 + yy**2)
        trz[:,2] = xyz[:,2]
    
    elif (xyz.ndim==3) and (xyz.shape[-1]==2): ## 2D, shape=(nx,ny,2)
        xx    = xyz[:,:,0] - cx
        yy    = xyz[:,:,1] - cy
        trz[:,:,0] = np.arctan2(yy,xx)
        trz[:,:,1] = np.sqrt(xx**2 + yy**2)
    
    elif (xyz.ndim==4) and (xyz.shape[-1]==3): ## 3D, shape=(nx,ny,nz,3)
        xx    = xyz[:,:,:,0] - cx
        yy    = xyz[:,:,:,1] - cy
        trz[:,:,:,0] = np.arctan2(yy,xx)
        trz[:,:,:,1] = np.sqrt(xx**2 + yy**2)
        trz[:,:,:,2] = xyz[:,:,:,2]
    
    else:
        raise ValueError('this input is not supported')
    
    return trz

def cyl_to_rect(trz,**kwargs):
    '''
    convert [θ,r,<z>] to [x,y,<z>]
    '''
    
    cx = kwargs.get('cx',0.)
    cy = kwargs.get('cy',0.)
    
    xyz = np.zeros_like(trz)
    
    if (trz.ndim==1) and (trz.shape[-1]==2): ## a single point, shape=(2,)
        tt = trz[0]
        rr = trz[1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[0] = xx + cx
        xyz[1] = yy + cy
    
    elif (trz.ndim==1) and (trz.shape[-1]==3): ## a single point, shape=(3,)
        tt = trz[0]
        rr = trz[1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[0] = xx + cx
        xyz[1] = yy + cy
        xyz[2] = trz[2]
    
    elif (trz.ndim==2) and (trz.shape[-1]==2): ## a 1D vector of 2D points, shape=(N,2)
        tt = trz[:,0]
        rr = trz[:,1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[:,0] = xx + cx
        xyz[:,1] = yy + cy
    
    elif (trz.ndim==2) and (trz.shape[-1]==3): ## a 1D vector of 3D points, shape=(N,3)
        tt = trz[:,0]
        rr = trz[:,1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[:,0] = xx + cx
        xyz[:,1] = yy + cy
        xyz[:,2] = trz[:,2]
    
    elif (trz.ndim==3) and (trz.shape[-1]==2): ## 2D, shape=(nx,ny,2)
        tt = trz[:,:,0]
        rr = trz[:,:,1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[:,:,0] = xx + cx
        xyz[:,:,1] = yy + cy
    
    elif (trz.ndim==4) and (trz.shape[-1]==3): ## 3D, shape=(nx,ny,nz,3)
        tt = trz[:,:,:,0]
        rr = trz[:,:,:,1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[:,:,:,0] = xx + cx
        xyz[:,:,:,1] = yy + cy
        xyz[:,:,:,2] = trz[:,:,:,2]
    
    else:
        raise ValueError('this input is not supported')
    
    return xyz

def rotate_2d(xy,theta,**kwargs):
    '''
    rotate a cartesian [x,y] point coordinate array around a center point [cx,cy]
    '''
    
    cx = kwargs.get('cx',0.)
    cy = kwargs.get('cy',0.)
    
    xy_rotated = np.zeros_like(xy)
    
    xy_ = np.copy(xy) ## prevent any pointer disasters
    
    translation_mat_1 = np.array([[ 1, 0, -cx ],
                                  [ 0, 1, -cy ],
                                  [ 0, 0,  1  ]], dtype=np.float64)
    
    rotation_mat = np.array([[ np.cos(theta), -np.sin(theta), 0 ],
                             [ np.sin(theta),  np.cos(theta), 0 ],
                             [ 0,              0,             1 ]], dtype=np.float64)
    
    translation_mat_2 = np.array([[ 1, 0, +cx ],
                                  [ 0, 1, +cy ],
                                  [ 0, 0,  1  ]], dtype=np.float64)
    
    transform_mat = np.einsum( 'ij,jk,kl->il' , translation_mat_2 , rotation_mat , translation_mat_1 )
    
    if (xy_.ndim==1) and (xy_.shape[-1]==2): ## a single point (2,)
        xy_ = np.concatenate((xy_,[1])) ## pad a 1
        xy_rotated = np.einsum( 'ij,j->i' , transform_mat, xy_ )
        xy_rotated = xy_rotated[:2]
    
    elif (xy_.ndim==2) and (xy_.shape[-1]==2): ## a 1D vector of points (N,2)
        raise NotImplementedError
    elif (xy_.ndim==3) and (xy_.shape[-1]==2): ## a 2D field of points (nx,ny,2)
        raise NotImplementedError
    else:
        raise ValueError('this input is not supported')
    
    return xy_rotated

# post-processing : vector & tensor ops
# ======================================================================

def get_grad(a,b,c, x,y,z, **kwargs):
    '''
    get the 3D gradient tensor (∂Ai/∂xj) from vector A=[a,b,c]
    -----
    - a,b,c are 3D arrays
    - x,y,z are 1D arrays (coord vectors)
    '''
    
    do_stack = kwargs.get('do_stack',True)
    hiOrder  = kwargs.get('hiOrder',True)
    verbose  = kwargs.get('verbose',False)
    
    print('get_grad() has been deprecated --> needs to be updated with turbx.gradient()')
    sys.exit(1)
    
    nx = x.size; ny = y.size; nz = z.size
    
    dtype = a.dtype
    
    # === gradients with O3 Spline + natural BCs
    if hiOrder:
        '''
        this could be parallelized with multiprocessing + async threads
        '''
        dadx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dady = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dadz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdy = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdy = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        
        if verbose: progress_bar = tqdm(total=((nx*ny)+(ny*nz)+(nz*nx)), ncols=100, desc='get_grad()', leave=False, file=sys.stdout)
        
        for i in range(nx):
            for j in range(ny):
                dadz[i,j,:] = sp.interpolate.CubicSpline(z,a[i,j,:],bc_type='natural')(z,1)
                dbdz[i,j,:] = sp.interpolate.CubicSpline(z,b[i,j,:],bc_type='natural')(z,1)
                dcdz[i,j,:] = sp.interpolate.CubicSpline(z,c[i,j,:],bc_type='natural')(z,1)
                if verbose: progress_bar.update()
        for j in range(ny):
            for k in range(nz):
                dadx[:,j,k] = sp.interpolate.CubicSpline(x,a[:,j,k],bc_type='natural')(x,1)
                dbdx[:,j,k] = sp.interpolate.CubicSpline(x,b[:,j,k],bc_type='natural')(x,1)
                dcdx[:,j,k] = sp.interpolate.CubicSpline(x,c[:,j,k],bc_type='natural')(x,1)
                if verbose: progress_bar.update()
        for k in range(nz):
            for i in range(nx):
                dady[i,:,k] = sp.interpolate.CubicSpline(y,a[i,:,k],bc_type='natural')(y,1)
                dbdy[i,:,k] = sp.interpolate.CubicSpline(y,b[i,:,k],bc_type='natural')(y,1)
                dcdy[i,:,k] = sp.interpolate.CubicSpline(y,c[i,:,k],bc_type='natural')(y,1)
                if verbose: progress_bar.update()
        
        if verbose: progress_bar.close()
    
    else: ## numpy.gradient() --> 1st (or 2nd?) order
        if verbose: progress_bar = tqdm(total=9, ncols=100, desc='get_grad()', leave=False, file=sys.stdout)
        
        dadx = np.gradient(a, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        dady = np.gradient(a, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        dadz = np.gradient(a, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        dbdx = np.gradient(b, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        dbdy = np.gradient(b, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        dbdz = np.gradient(b, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        dcdx = np.gradient(c, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        dcdy = np.gradient(c, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        dcdz = np.gradient(c, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        
        if verbose: progress_bar.close()
    
    if do_stack:
        dAdx_ij = np.stack((np.stack((dadx, dady, dadz), axis=3),
                            np.stack((dbdx, dbdy, dbdz), axis=3),
                            np.stack((dcdx, dcdy, dcdz), axis=3)), axis=4)
        return dAdx_ij
    else:
        dAdx_ij = dict(zip(['dadx', 'dady', 'dadz', 'dbdx', 'dbdy', 'dbdz', 'dcdx', 'dcdy', 'dcdz'], 
                           [ dadx,   dady,   dadz,   dbdx,   dbdy,   dbdz,   dcdx,   dcdy,   dcdz]))
        return dAdx_ij

def get_curl(a,b,c, x,y,z, **kwargs):
    '''
    get 3D curl vector ∇⨯A , A=[a,b,c]
    '''
    
    do_stack = kwargs.get('do_stack',True)
    hiOrder  = kwargs.get('hiOrder',True)
    verbose  = kwargs.get('verbose',False)
    
    print('get_curl() has been deprecated --> needs to be updated with turbx.gradient()')
    sys.exit(1)
    
    nx = x.size; ny = y.size; nz = z.size
    
    dtype = u.dtype
    
    ## gradients with O3 Spline + natural BCs
    if hiOrder:
        '''
        this could be easily parallelized
        '''
        
        #dadx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dady = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dadz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        #dbdy = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdy = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        #dcdz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        
        if verbose: progress_bar = tqdm(total=((nx*ny)+(ny*nz)+(nz*nx)), ncols=100, desc='get_curl()', leave=False, file=sys.stdout)
        
        for i in range(nx):
            for j in range(ny):
                dadz[i,j,:] = sp.interpolate.CubicSpline(z,a[i,j,:],bc_type='natural')(z,1)
                dbdz[i,j,:] = sp.interpolate.CubicSpline(z,b[i,j,:],bc_type='natural')(z,1)
                #dcdz[i,j,:] = sp.interpolate.CubicSpline(z,c[i,j,:],bc_type='natural')(z,1)
                if verbose: progress_bar.update()
        for j in range(ny):
            for k in range(nz):
                #dadx[:,j,k] = sp.interpolate.CubicSpline(x,a[:,j,k],bc_type='natural')(x,1)
                dbdx[:,j,k] = sp.interpolate.CubicSpline(x,b[:,j,k],bc_type='natural')(x,1)
                dcdx[:,j,k] = sp.interpolate.CubicSpline(x,c[:,j,k],bc_type='natural')(x,1)
                if verbose: progress_bar.update()
        for k in range(nz):
            for i in range(nx):
                dady[i,:,k] = sp.interpolate.CubicSpline(y,a[i,:,k],bc_type='natural')(y,1)
                #dbdy[i,:,k] = sp.interpolate.CubicSpline(y,b[i,:,k],bc_type='natural')(y,1)
                dcdy[i,:,k] = sp.interpolate.CubicSpline(y,c[i,:,k],bc_type='natural')(y,1)
                if verbose: progress_bar.update()
        
        if verbose: progress_bar.close()
    
    else: ## numpy.gradient() --> 1st (or 2nd?) order
        if verbose: progress_bar = tqdm(total=9, ncols=100, desc='get_curl()', leave=False, file=sys.stdout)
        
        #dadx = np.gradient(a, x, edge_order=2, axis=0)
        #if verbose: progress_bar.update()
        dady = np.gradient(a, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        dadz = np.gradient(a, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        dbdx = np.gradient(b, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        #dbdy = np.gradient(b, y, edge_order=2, axis=1)
        #if verbose: progress_bar.update()
        dbdz = np.gradient(b, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        dcdx = np.gradient(c, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        dcdy = np.gradient(c, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        #dcdz = np.gradient(c, z, edge_order=2, axis=2)
        #if verbose: progress_bar.update()
        
        if verbose: progress_bar.close()
    
    # ===
    curl_x = dcdy - dbdz
    curl_y = dadz - dcdx
    curl_z = dbdx - dady
    
    if do_stack:
        curl = np.stack((curl_x, curl_y, curl_z), axis=3)
        return curl
    else:
        curl = dict(zip(['curl_x', 'curl_y', 'curl_z'], 
                        [ curl_x,   curl_y,   curl_z]))
        return curl

# post-processing : spectral, statistical
# ======================================================================

def get_overlapping_window_size(asz, n_win, overlap_fac):
    '''
    get window length and overlap given a
    desired number of windows and a nominal overlap factor
    -----
    --> the output should be passed to get_overlapping_windows()
        to do the actual padding & windowing
    '''
    if not isinstance(asz, int):
        raise TypeError('arg asz must be type int')
    if not isinstance(n_win, int):
        raise TypeError('arg n_win must be type int')
    if (overlap_fac >= 1.):
        raise ValueError('arg overlap_fac must be <1')
    if (overlap_fac < 0.):
        raise ValueError('arg overlap_fac must be >0')
    n_ends = n_win+1
    n_mids = n_win
    
    # === solve for float-valued window 'mid' size & 'end' size
    def eqn(soltup, asz=asz, overlap_fac=overlap_fac):
        (endsz,midsz) = soltup
        eq1 = asz - n_ends*endsz - n_mids*midsz
        eq2 = overlap_fac*(midsz+2*endsz) - endsz
        return [eq1, eq2]
    
    guess = asz*0.5
    endsz,midsz = sp.optimize.fsolve(eqn, (guess,guess), (asz,overlap_fac))
    win_len     = midsz + 2*endsz
    overlap     = endsz
    
    win_len = max(math.ceil(win_len),1)
    overlap = max(math.floor(overlap),0)
    
    return win_len, overlap

def get_overlapping_windows(a, win_len, overlap):
    '''
    subdivide 1D array into overlapping windows
    '''
    #pad_mode = kwargs.get('pad_mode','append')
    ##
    if not isinstance(a, np.ndarray):
        raise TypeError('arg a must be type np.ndarray')
    if not isinstance(win_len, int):
        raise TypeError('arg win_len must be type int')
    if not isinstance(overlap, int):
        raise TypeError('arg overlap must be type int')
    ##
    asz   = a.size
    skip  = win_len - overlap
    n_pad = (win_len - asz%skip)%skip
    #a_pad = np.concatenate(( np.zeros(n_pad,dtype=a.dtype) , np.copy(a) )) ## prepend
    a_pad = np.concatenate(( np.copy(a) , np.zeros(n_pad,dtype=a.dtype) )) ## append
    ##
    b = np.lib.stride_tricks.sliding_window_view(a_pad, win_len, axis=0)
    b = np.copy(b[::skip,:])
    n_win = b.shape[0]
    ##
    if (n_pad > 0.5*win_len):
        print('WARNING: n_pad > overlap')
    ##
    return b, n_win, n_pad

def ccor(ui,uj,**kwargs):
    '''
    normalized cross-correlation
    '''
    if (ui.ndim!=1):
        raise AssertionError('ui.ndim!=1')
    if (uj.ndim!=1):
        raise AssertionError('uj.ndim!=1')
    ##
    mode     = kwargs.get('mode','full')
    get_lags = kwargs.get('get_lags',False)
    ##
    lags = sp.signal.correlation_lags(ui.shape[0], uj.shape[0], mode=mode)
    ccor = sp.signal.correlate(ui, uj, mode=mode, method='direct')
    norm = np.sqrt( np.sum( ui**2 ) ) * np.sqrt( np.sum( uj**2 ) )
    ##
    if (norm==0.):
        #ccor = np.ones((lags.shape[0],), dtype=ui.dtype)
        ccor = np.zeros((lags.shape[0],), dtype=ui.dtype)
    else:
        ccor /= norm
    ##
    if get_lags:
        return lags, ccor
    else:
        return ccor

def ccor_naive(u,v,**kwargs):
    '''
    normalized cross-correlation (naive version)
    - this kernel is designed as a check for ccor()
    '''
    if (u.ndim!=1):
        raise AssertionError('u.ndim!=1')
    if (v.ndim!=1):
        raise AssertionError('v.ndim!=1')
    
    ii = np.arange(u.shape[0],dtype=np.int32)
    jj = np.arange(v.shape[0],dtype=np.int32)
    
    ## lags (2D)
    ll     = np.stack(np.meshgrid(ii,jj,indexing='ij'), axis=-1)
    ll     = ll[:,:,0] - ll[:,:,1]
    
    ## lags (1D)
    lmin   = ll.min()
    lmax   = ll.max()
    n_lags = lmax-lmin+1
    lags   = np.arange(lmin,lmax+1)
    
    uu, vv = np.meshgrid(u,v,indexing='ij')
    uv     = np.stack((uu,vv), axis=-1)
    uvp    = np.prod(uv,axis=-1)
    
    c=-1
    R = np.zeros(n_lags, dtype=np.float64)
    for lag in lags:
        c+=1
        X = np.where(ll==lag)
        N = X[0].shape[0]
        R_ = np.sum(uvp[X]) / ( np.sqrt(np.sum(u**2)) * np.sqrt(np.sum(v**2)) )
        R[c] = R_
    return lags, R

# binary I/O
# ======================================================================

def gulp(fname, **kwargs):
    '''
    read a complete binary file into memory, return 'virtual file'
    -----
    - returned handle can be opened via h5py as if it were on disk (with very high performance)
    - of course this only works if the entire file fits into memory
    - best use case: medium size file, large number of high-frequency read ops
    '''
    verbose = kwargs.get('verbose',True)
    f_size_gb = os.path.getsize(fname)/1024**3
    if verbose: tqdm.write('>>> gulp() : %s : %0.2f [GB]'%(os.path.basename(fname),f_size_gb))
    t_start = timeit.default_timer()
    with open(fname, 'rb') as fnb:
        bytes_in_mem = io.BytesIO(fnb.read())
    t_delta = timeit.default_timer() - t_start
    if verbose: tqdm.write('>>> gulp() : %s : %0.2f [GB/s]'%(format_time_string(t_delta), (f_size_gb/t_delta)))
    return bytes_in_mem

# utilities
# ======================================================================

def format_time_string(tsec):
    '''
    format seconds as dd:hh:mm:ss
    '''
    m, s = divmod(tsec,60)
    h, m = divmod(m,60)
    d, h = divmod(h,24)
    time_str = '%dd:%dh:%02dm:%02ds'%(d,h,m,s)
    return time_str

def format_nbytes(size):
    '''
    format a number of bytes to [B],[KB],[MB],[GB],[TB]
    '''
    if not isinstance(size,(int,float)):
        raise ValueError('arg should be of type int or float')
    if (size<1024):
        size_fmt, size_unit = size, '[B]'
    elif (size>1024) and (size<=1024**2):
        size_fmt, size_unit = size/1024, '[KB]'
    elif (size>1024**2) and (size<=1024**3):
        size_fmt, size_unit = size/1024**2, '[MB]'
    elif (size>1024**3) and (size<=1024**4):
        size_fmt, size_unit = size/1024**3, '[GB]'
    else:
        size_fmt, size_unit = size/1024**4, '[TB]'
    return size_fmt, size_unit

def even_print(label, output, **kwargs):
    '''
    print/return a fixed width message
    '''
    terminal_width = kwargs.get('terminal_width',72)
    s              = kwargs.get('s',False) ## return string
    
    ndots = (terminal_width-2) - len(label) - len(output)
    text = label+' '+ndots*'.'+' '+output
    if s:
        return text
    else:
        #sys.stdout.write(text)
        print(text)
        return

# plotting & matplotlib
# ======================================================================

def set_mpl_env(**kwargs):
    '''
    Setup the matplotlib environment
    --------------------------------
    
    - styles   : https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html
    - rcParams : https://matplotlib.org/stable/tutorials/introductory/customizing.html
    
    TrueType / OpenType Fonts
    -------------------------
    
    - IBM Plex
        - https://github.com/IBM/plex/archive/refs/heads/master.zip
        - https://www.fontsquirrel.com/fonts/download/ibm-plex --> doesnt contain 'Condensed'
        - test at : https://www.ibm.com/plex/plexness/
        
        ----- Linux / Ubuntu / WSL2
        $> sudo apt install fonts-ibm-plex
        ----- Linux / Ubuntu / WSL2 (manual)
        $> wget https://github.com/IBM/plex/archive/refs/heads/master.zip
        $> unzip master.zip
        $> cd plex-master
        $> sudo mkdir -p /usr/share/fonts/opentype/ibm-plex
        $> sudo mkdir -p /usr/share/fonts/truetype/ibm-plex
        $> sudo find . -name '*.otf' -exec cp -v {} /usr/share/fonts/opentype/ibm-plex/ \;
        $> sudo find . -name '*.ttf' -exec cp -v {} /usr/share/fonts/truetype/ibm-plex/ \;
        $> fc-cache -f -v
        $> fc-list | grep 'IBM'
        ----- regenerate matplotlib font cache (just delete, gets regenerated)
        $> rm ~/.cache/matplotlib/fontlist-v330.json
        $> rm -rf ~/.cache/matplotlib/tex.cache
    
    - Latin Modern Math
        - http://www.gust.org.pl/projects/e-foundry/lm-math/download/latinmodern-math-1959.zip
    
    - Latin Modern (lmodern in LaTeX)
        - https://www.fontsquirrel.com/fonts/download/Latin-Modern-Roman.zip
        - http://www.gust.org.pl/projects/e-foundry/latin-modern/download/lm2.004otf.zip
        
        --> usually already installed at : /usr/share/texmf/fonts/opentype/public/lm/
        ----- Linux / Ubuntu / WSL2
        $> wget https://www.fontsquirrel.com/fonts/download/Latin-Modern-Roman.zip
        $> unzip Latin-Modern-Roman.zip -d Latin-Modern-Roman
        $> cd Latin-Modern-Roman
        $> sudo mkdir -p /usr/share/fonts/opentype/lmodern
        $> sudo find . -name '*.otf' -exec cp -v {} /usr/share/fonts/opentype/lmodern/ \;
        $> fc-cache -f -v
        $> fc-list | grep 'Latin'
        ----- regenerate matplotlib font cache (just delete, gets regenerated)
        $> rm ~/.cache/matplotlib/fontlist-v330.json
        $> rm -rf ~/.cache/matplotlib/tex.cache
    
    - Computer Modern (default in LaTeX)
        - https://www.fontsquirrel.com/fonts/download/computer-modern.zip
        - http://mirrors.ctan.org/fonts/cm/ps-type1/bakoma.zip
        ----- Linux / Ubuntu / WSL2
        $> sudo apt install fonts-cmu
        ----- Linux / Ubuntu / WSL2 (manual install)
        ## wget https://sourceforge.net/projects/cm-unicode/files/latest/download -O cm-unicode-0.7.0-ttf.tar.xz
        ## tar -xvf cm-unicode-0.7.0-ttf.tar.xz
        ## wget https://www.fontsquirrel.com/fonts/download/computer-modern -O computer-modern.zip
        $> wget https://mirrors.ctan.org/fonts/cm-unicode.zip
        $> unzip computer-modern.zip -d computer-modern
        $> cd computer-modern
        $> sudo mkdir -p /usr/share/fonts/truetype/cmu
        $> sudo find . -name '*.ttf' -exec cp -v {} /usr/share/fonts/truetype/cmu/ \;
        $> fc-cache -f -v
        $> fc-list | grep 'CMU'
        ----- regenerate matplotlib font cache (just delete, gets regenerated)
        $> rm ~/.cache/matplotlib/fontlist-v330.json
        $> rm -rf ~/.cache/matplotlib/tex.cache
    
    Windows
    -------
    --> download, install, then delete : C:/Users/%USERNAME%/.matplotlib/fontlist-v330.json
    --> this JSON file will get regenerated with newly installed fonts
    '''
    
    useTex   = kwargs.get('useTex',False) ## use LaTeX text rendering
    darkMode = kwargs.get('darkMode',True)
    font     = kwargs.get('font',None)
    
    ## mpl.rcParams.update(mpl.rcParamsDefault) ## reset rcparams to defaults
    
    if darkMode:
        mpl.style.use('dark_background') ## dark mode
    else:
        mpl.style.use('default')
    
    if useTex:
        
        ## 'Text rendering with LaTeX'
        ## https://matplotlib.org/stable/tutorials/text/usetex.html
        
        ## https://github.com/matplotlib/matplotlib/issues/5076/
        ## https://github.com/matplotlib/matplotlib/issues/17673/
        
        mpl.rcParams['text.usetex'] = True
        # mpl.rcParams['pgf.texsystem'] = 'pdflatex' ## 'xelatex', 'lualatex', 'pdflatex'
        
        # preamble_opts = [ r'' ]
        
        preamble_opts = [ #r'\usepackage[utf8]{inputenc}',
                          #r'\usepackage[T1]{fontenc}',
                          #r'\usepackage{amsmath}', ## AMS mathematical facilities for LaTeX
                          #r'\usepackage{amsfonts}',
                          r'\usepackage{amssymb}',
                          #r'\usepackage{textcomp}'
                          r'\usepackage{gensymb}', ## generic symbols 
                          #r'\usepackage{xfrac}',
                          #r'\usepackage{nicefrac}',
                          ]
        
        if (font==None): ## default
            mpl.rcParams['font.family']= 'serif'
            mpl.rcParams['font.serif'] = 'Computer Modern Roman'
        
        elif (font=='IBM Plex Sans') or (font=='IBM Plex') or (font=='IBM') or (font=='ibm'):
            preamble_opts +=  [ r'\usepackage{plex-sans}', ## IBM Plex Sans
                                r'\renewcommand{\familydefault}{\sfdefault}', ## sans as default family
                                r'\renewcommand{\seriesdefault}{c}', ## condensed {*} as default series
                                r'\usepackage[italic]{mathastext}', ## use default font in math mode
                              ]
        
        elif (font=='times') or (font=='Times') or (font=='Times New Roman'):
            mpl.rcParams['font.family'] = 'serif'
            mpl.rcParams['font.serif']  = 'Times'
            #preamble_opts +=  [ r'\usepackage{txfonts}' ] ## Times-like fonts mathtext symbols
            preamble_opts +=  [ r'\usepackage{newtxtext}',
                                r'\usepackage{newtxmath}',
                                #r'\usepackage[italic]{mathastext}', ## use default font in math mode
                              ]
        
        elif (font=='lmodern') or (font=='Latin Modern') or (font=='Latin Modern Roman') or (font=='lmr'):
            preamble_opts +=  [ r'\usepackage{lmodern}' ]
        
        elif (font=='Palatino') or (font=='palatino'):
            mpl.rcParams['font.family'] = 'serif'
            mpl.rcParams['font.serif']  = 'Palatino'
        
        elif (font=='Helvetica') or (font=='helvetica'):
            mpl.rcParams['font.family']      = 'sans-serif'
            mpl.rcParams['font.sans-serif']  = 'Helvetica'
            preamble_opts +=  [ r'\renewcommand{\familydefault}{\sfdefault}', ## sans as default family
                                r'\usepackage[italic]{mathastext}', ## use default font in math mode
                              ]
        
        elif (font=='Avant Garde'):
            mpl.rcParams['font.family']     = 'sans-serif'
            mpl.rcParams['font.sans-serif'] = 'Avant Garde'
            preamble_opts +=  [ r'\renewcommand{\familydefault}{\sfdefault}', ## sans as default family
                                r'\usepackage[italic]{mathastext}', ## use default font in math mode
                              ]
        
        elif (font=='Computer Modern Roman') or (font=='Computer Modern') or (font=='CMR') or (font=='cmr'):
            mpl.rcParams['font.family'] = 'serif'
            mpl.rcParams['font.serif']  = 'Computer Modern Roman'
        
        else:
            raise ValueError('font=%s not a valid option'%str(font))
        
        ## make preamble string
        mpl.rcParams['text.latex.preamble'] = '\n'.join(preamble_opts)
    
    else: ## use OpenType (OTF) / TrueType (TTF) fonts (and no TeX rendering)
        
        mpl.rcParams['text.usetex'] = False
        
        ## Register OTF/TTF Fonts (only necessary once)
        if False:
            # === register (new) fonts : Windows --> done automatically if you delete ~/.cache/matplotlib/fontlist-v330.json
            ##mpl.font_manager.findSystemFonts(fontpaths='C:/Windows/Fonts', fontext='ttf')
            #mpl.font_manager.findSystemFonts(fontpaths='C:/Users/'+os.path.expandvars('%USERNAME%')+'/AppData/Local/Microsoft/Windows/Fonts', fontext='ttf')
            mpl.font_manager.findSystemFonts(fontpaths=mpl.font_manager.win32FontDirectory(), fontext='ttf')
            
            # === register (new) fonts : Linux / WSL2 --> done automatically if you delete C:/Users/%USERNAME%/.matplotlib/fontlist-v330.json
            mpl.font_manager.findSystemFonts(fontpaths=None, fontext='ttf')
            mpl.font_manager.findSystemFonts(fontpaths=None, fontext='otf')
        
        ## example: list all TTF font properties
        if False:
            fonts = mpl.font_manager.fontManager.ttflist
            #fonts = [f for f in fonts if all([('IBM' in f.name),('Condensed' in f.name)])] ## filter list
            for f in fonts:
                print(f.name)
                print(Path(f.fname).stem)
                print('weight  : %s'%str(f.weight))
                print('style   : %s'%str(f.style))
                print('stretch : %s'%str(f.stretch))
                print('variant : %s'%str(f.variant))
                print('-----'+'\n')
        
        ## get list of names of all registered fonts
        if (font is not None):
            try:
                if hasattr(mpl.font_manager,'get_font_names'):
                    ## Matplotlib >3.6.X
                    fontnames = mpl.font_manager.get_font_names()
                elif hasattr(mpl.font_manager,'get_fontconfig_fonts'):
                    ## Matplotlib <=3.5.X
                    fontlist = mpl.font_manager.get_fontconfig_fonts()
                    fontnames = sorted(list(set([mpl.font_manager.FontProperties(fname=fname).get_name() for fname in fontlist])))
                else:
                    fontnames = None
            except:
                fontnames = None
        
        # === OTF/TTF fonts (when NOT using LaTeX rendering)
        
        if (font==None):
            pass ## do nothing, use system / matplotlib default font
        
        ## IBM Plex Sans
        elif (font=='IBM Plex Sans') or (font=='IBM Plex') or (font=='IBM') or (font=='ibm'):
            
            if (fontnames is not None) and ('IBM Plex Sans Condensed' in fontnames):
                
                ## condensed
                mpl.rcParams['font.family'] = 'IBM Plex Sans Condensed'
                mpl.rcParams['font.weight'] = 'regular' ## 200, 300/'light', 400, 450
                #mpl.rcParams['font.style'] = 'normal' ## 'normal', 'italic', 'oblique'
                #mpl.rcParams['font.variant'] = 'normal' ## 'normal', 'small-caps'
                #mpl.rcParams['font.stretch'] = 'condensed' ## always 'condensed' for family
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'IBM Plex Sans Condensed:regular'
                mpl.rcParams['mathtext.it'] = 'IBM Plex Sans Condensed:italic:regular'
                mpl.rcParams['mathtext.bf'] = 'IBM Plex Sans Condensed:bold'
                mpl.rcParams['mathtext.cal'] = 'IBM Plex Sans Condensed:italic'
                
                ## regular --> too wide
                # mpl.rcParams['font.family'] = 'IBM Plex Sans'
                # mpl.rcParams['font.weight'] = '400' ## 'light'
                # #mpl.rcParams['font.stretch'] = 'normal' ## always 'normal' for family
                # mpl.rcParams['mathtext.fontset'] = 'custom'
                # mpl.rcParams['mathtext.default'] = 'it'
                # mpl.rcParams['mathtext.rm'] = 'IBM Plex Sans:regular'
                # mpl.rcParams['mathtext.it'] = 'IBM Plex Sans:italic:regular'
                # mpl.rcParams['mathtext.bf'] = 'IBM Plex Sans:bold'
        
        ## Latin Modern Roman (lmodern in LaTeX, often used)
        elif (font=='lmodern') or (font=='Latin Modern') or (font=='Latin Modern Roman') or (font=='lmr'):
            if (fontnames is not None) and ('Latin Modern Roman' in fontnames):
                mpl.rcParams['font.family'] = 'Latin Modern Roman'
                mpl.rcParams['font.weight'] = '400'
                mpl.rcParams['font.style'] = 'normal'
                mpl.rcParams['font.variant'] = 'normal'
                #mpl.rcParams['font.stretch'] = 'condensed'
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'Latin Modern Roman:normal'
                mpl.rcParams['mathtext.it'] = 'Latin Modern Roman:italic'
                mpl.rcParams['mathtext.bf'] = 'Latin Modern Roman:bold'
        
        ## Times New Roman
        elif (font=='times') or (font=='Times') or (font=='Times New Roman'):
            if (fontnames is not None) and ('Times New Roman' in fontnames):
                mpl.rcParams['font.family'] = 'Times New Roman'
                mpl.rcParams['font.weight'] = 'normal'
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'Times New Roman:normal'
                mpl.rcParams['mathtext.it'] = 'Times New Roman:italic'
                mpl.rcParams['mathtext.bf'] = 'Times New Roman:bold'
        
        ## Computer Modern (LaTeX default)
        elif (font=='Computer Modern Roman') or (font=='CMU Serif') or (font=='Computer Modern') or (font=='CMR') or (font=='cmr') or (font=='cmu'):
            if (fontnames is not None) and ('CMU Serif' in fontnames):
                mpl.rcParams['font.family'] = 'CMU Serif'
                mpl.rcParams['font.weight'] = 'regular'
                mpl.rcParams['font.style'] = 'normal'
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'CMU Serif:regular'
                mpl.rcParams['mathtext.it'] = 'CMU Serif:italic:regular'
                mpl.rcParams['mathtext.bf'] = 'CMU Serif:bold'
        
        else:
            raise ValueError('font=%s not a valid option'%str(font))
        
        ## Manually point to a TTF/OTF file
        if False:
            #fe = mpl.font_manager.FontEntry(fname='/usr/share/texmf/fonts/opentype/public/lm/lmroman10-regular.otf', name='10erLatin')
            fe = mpl.font_manager.FontEntry(fname='C:/Users/'+os.path.expandvars('%USERNAME%')+'/AppData/Local/Microsoft/Windows/Fonts/lmroman10-regular.otf', name='10erLatin')
            mpl.font_manager.fontManager.ttflist.insert(0, fe)
            mpl.rcParams['font.family'] = fe.name
            mpl.rcParams['font.weight'] = '400'
            mpl.rcParams['font.style'] = 'normal'
            mpl.rcParams['font.variant'] = 'normal'
            mpl.rcParams['mathtext.fontset'] = 'custom'
            mpl.rcParams['mathtext.default'] = 'it'
            mpl.rcParams['mathtext.rm'] = fe.name+':normal'
            mpl.rcParams['mathtext.it'] = fe.name+':italic'
            mpl.rcParams['mathtext.bf'] = fe.name+':bold'
    
    # ===
    
    ## ## list all options
    ## print(mpl.rcParams.keys())
    
    fontsize = 10
    axesAndTickWidth = 0.5
    
    mpl.rcParams['figure.figsize'] = 3*(16/9), 3
    mpl.rcParams['figure.dpi']     = 300
    #mpl.rcParams['figure.facecolor'] = 'k'
    #mpl.rcParams['figure.autolayout'] = True ### tight_layout() --> just use instead : fig1.tight_layout(pad=0.20)
    
    #mpl.rcParams['figure.constrained_layout.use'] = True
    #mpl.rcParams['figure.constrained_layout.h_pad']  = 0.0 ## Padding around axes objects. Float representing
    #mpl.rcParams['figure.constrained_layout.w_pad']  = 0.0 ## inches. Default is 3/72 inches (3 points)
    #mpl.rcParams['figure.constrained_layout.hspace'] = 0.2 ## Space between subplot groups. Float representing
    #mpl.rcParams['figure.constrained_layout.wspace'] = 0.2 ## a fraction of the subplot widths being separated.
    
    #mpl.rcParams['figure.subplot.bottom'] = 0.02
    #mpl.rcParams['figure.subplot.top'] = 0.98
    #mpl.rcParams['figure.subplot.left'] = 0.02
    #mpl.rcParams['figure.subplot.right'] = 0.98
    #mpl.rcParams['figure.subplot.hspace'] = 0.02
    #mpl.rcParams['figure.subplot.wspace'] = 0.2
    
    mpl.rcParams['pdf.compression'] = 2 ## 0-9
    #mpl.rcParams['pdf.fonttype'] = 42  # Output Type 3 (Type3) or Type 42 (TrueType)
    #mpl.rcParams['pdf.use14corefonts'] = False
    
    mpl.rcParams['savefig.pad_inches'] = 0.20
    mpl.rcParams['savefig.dpi']        = 300
    
    mpl.rcParams['xtick.major.size']  = 2.5
    mpl.rcParams['xtick.major.width'] = axesAndTickWidth
    mpl.rcParams['xtick.minor.size']  = 1.4
    mpl.rcParams['xtick.minor.width'] = axesAndTickWidth*1.0
    #mpl.rcParams['xtick.color'] = 'k' ## set with mpl.style.use()
    mpl.rcParams['xtick.direction']   = 'in'
    
    mpl.rcParams['ytick.major.size']  = 2.5
    mpl.rcParams['ytick.major.width'] = axesAndTickWidth
    mpl.rcParams['ytick.minor.size']  = 1.4
    mpl.rcParams['ytick.minor.width'] = axesAndTickWidth*1.0
    #mpl.rcParams['ytick.color'] = 'k' ## set with mpl.style.use()
    mpl.rcParams['ytick.direction'] = 'in'
    
    mpl.rcParams['xtick.labelsize'] = fontsize
    mpl.rcParams['ytick.labelsize'] = fontsize
    
    mpl.rcParams['xtick.major.pad'] = 3.0
    mpl.rcParams['ytick.major.pad'] = 2.0
    
    mpl.rcParams['lines.linewidth']  = 1.0
    mpl.rcParams['lines.linestyle']  = 'solid'
    mpl.rcParams['lines.marker']     = 'None' #'o'
    mpl.rcParams['lines.markersize'] = 1.2
    mpl.rcParams['lines.markeredgewidth'] = 0.
    
    #mpl.rcParams['axes.facecolor'] = 'k' ## set with mpl.style.use()
    mpl.rcParams['axes.linewidth'] = axesAndTickWidth
    mpl.rcParams['axes.labelpad']  = 3.0
    mpl.rcParams['axes.titlesize'] = fontsize
    mpl.rcParams['axes.labelsize'] = fontsize
    mpl.rcParams['axes.formatter.use_mathtext'] = True
    mpl.rcParams['axes.axisbelow'] = False ## dont allow axes, ticks to be under lines --> doesn't work for artist objects with zorder >2.5
    
    mpl.rcParams['legend.fontsize'] = fontsize
    mpl.rcParams['legend.shadow']   = False
    mpl.rcParams['legend.borderpad'] = 0.3
    mpl.rcParams['legend.framealpha'] = 1.0
    mpl.rcParams['legend.edgecolor']  = 'inherit'
    mpl.rcParams['legend.handlelength'] = 1.0
    mpl.rcParams['legend.handletextpad'] = 0.4
    mpl.rcParams['legend.borderaxespad'] = 0.6
    mpl.rcParams['legend.columnspacing'] = 0.5
    mpl.rcParams['legend.fancybox'] = False
    
    return

def colors_table():
    '''
    a table of color hues
    -----
    clrs = colors_table()
    red = clrs['red'][9]
    -----
    https://yeun.github.io/open-color/
    https://twitter.com/nprougier/status/1323575342204936192
    https://github.com/rougier/scientific-visualization-book/blob/master/code/colors/open-colors.py
    '''
    
    colors = {  "gray"   :  { 0: '#f8f9fa', 1: '#f1f3f5', 2: '#e9ecef', 3: '#dee2e6', 4: '#ced4da', 
                              5: '#adb5bd', 6: '#868e96', 7: '#495057', 8: '#343a40', 9: '#212529', },
                "red"    :  { 0: '#fff5f5', 1: '#ffe3e3', 2: '#ffc9c9', 3: '#ffa8a8', 4: '#ff8787', 
                              5: '#ff6b6b', 6: '#fa5252', 7: '#f03e3e', 8: '#e03131', 9: '#c92a2a', },
                "pink"   :  { 0: '#fff0f6', 1: '#ffdeeb', 2: '#fcc2d7', 3: '#faa2c1', 4: '#f783ac',
                              5: '#f06595', 6: '#e64980', 7: '#d6336c', 8: '#c2255c', 9: '#a61e4d', },
                "grape"  :  { 0: '#f8f0fc', 1: '#f3d9fa', 2: '#eebefa', 3: '#e599f7', 4: '#da77f2',
                              5: '#cc5de8', 6: '#be4bdb', 7: '#ae3ec9', 8: '#9c36b5', 9: '#862e9c', },
                "violet" :  { 0: '#f3f0ff', 1: '#e5dbff', 2: '#d0bfff', 3: '#b197fc', 4: '#9775fa',
                              5: '#845ef7', 6: '#7950f2', 7: '#7048e8', 8: '#6741d9', 9: '#5f3dc4', },
                "indigo" :  { 0: '#edf2ff', 1: '#dbe4ff', 2: '#bac8ff', 3: '#91a7ff', 4: '#748ffc', 
                              5: '#5c7cfa', 6: '#4c6ef5', 7: '#4263eb', 8: '#3b5bdb', 9: '#364fc7', },
                "blue"   :  { 0: '#e7f5ff', 1: '#d0ebff', 2: '#a5d8ff', 3: '#74c0fc', 4: '#4dabf7',
                              5: '#339af0', 6: '#228be6', 7: '#1c7ed6', 8: '#1971c2', 9: '#1864ab', },
                "cyan"   :  { 0: '#e3fafc', 1: '#c5f6fa', 2: '#99e9f2', 3: '#66d9e8', 4: '#3bc9db',
                              5: '#22b8cf', 6: '#15aabf', 7: '#1098ad', 8: '#0c8599', 9: '#0b7285', },
                "teal"   :  { 0: '#e6fcf5', 1: '#c3fae8', 2: '#96f2d7', 3: '#63e6be', 4: '#38d9a9',
                              5: '#20c997', 6: '#12b886', 7: '#0ca678', 8: '#099268', 9: '#087f5b', },
                "green"  :  { 0: '#ebfbee', 1: '#d3f9d8', 2: '#b2f2bb', 3: '#8ce99a', 4: '#69db7c',
                              5: '#51cf66', 6: '#40c057', 7: '#37b24d', 8: '#2f9e44', 9: '#2b8a3e', },
                "lime"   :  { 0: '#f4fce3', 1: '#e9fac8', 2: '#d8f5a2', 3: '#c0eb75', 4: '#a9e34b',
                              5: '#94d82d', 6: '#82c91e', 7: '#74b816', 8: '#66a80f', 9: '#5c940d', },
                "yellow" :  { 0: '#fff9db', 1: '#fff3bf', 2: '#ffec99', 3: '#ffe066', 4: '#ffd43b',
                              5: '#fcc419', 6: '#fab005', 7: '#f59f00', 8: '#f08c00', 9: '#e67700', },
                "orange" :  { 0: '#fff4e6', 1: '#ffe8cc', 2: '#ffd8a8', 3: '#ffc078', 4: '#ffa94d',
                              5: '#ff922b', 6: '#fd7e14', 7: '#f76707', 8: '#e8590c', 9: '#d9480f', }, }
    
    return colors

def get_standard_colors():
    '''
    a convenience func to return some colors from colors_table()
    '''
    
    cc = colors_table()
    
    red,pink,grape,violet,indigo,blue,cyan,teal,green,lime,yellow,orange = \
    cc['red'][7],\
    cc['pink'][3],\
    cc['grape'][6],\
    cc['violet'][6],\
    cc['indigo'][8],\
    cc['blue'][6],\
    cc['cyan'][3],\
    cc['teal'][3],\
    cc['green'][7],\
    cc['lime'][4],\
    cc['yellow'][4],\
    cc['orange'][6]
    
    colors_str = [ 'red','pink','grape','violet','indigo','blue','cyan','teal','green','lime','yellow','orange' ]
    colors = [ red,pink,grape,violet,indigo,blue,cyan,teal,green,lime,yellow,orange ]
    
    red_dk,pink_dk,grape_dk,violet_dk,indigo_dk,blue_dk,cyan_dk,teal_dk,green_dk,lime_dk,yellow_dk,orange_dk = \
    cc['red'][6+2],\
    cc['pink'][3+2],\
    cc['grape'][6+2],\
    cc['violet'][6+2],\
    cc['indigo'][8+2-1],\
    cc['blue'][6+2],\
    cc['cyan'][3+2],\
    cc['teal'][3+2],\
    cc['green'][7+2],\
    cc['lime'][4+2],\
    cc['yellow'][4+2],\
    cc['orange'][6+2]
    
    colors_dark_str = [ 'red_dk','pink_dk','grape_dk','violet_dk','indigo_dk','blue_dk','cyan_dk','teal_dk','green_dk','lime_dk','yellow_dk','orange_dk' ]
    colors_dark = [red_dk,pink_dk,grape_dk,violet_dk,indigo_dk,blue_dk,cyan_dk,teal_dk,green_dk,lime_dk,yellow_dk,orange_dk]
    
    ## to dict
    colors    = dict(zip(colors_str, colors))
    colors_dk = dict(zip(colors_dark_str, colors_dark))
    
    return colors, colors_dk

def colors_test_plot(color_dict):
    '''
    plot test image for color palette
    '''
    
    plt.close('all')
    fig1 = plt.figure(figsize=(2.5*(16/9),2.5), dpi=200)
    ax1 = plt.gca()
    ax1.tick_params(axis='x', which='both', direction='in')
    ax1.tick_params(axis='y', which='both', direction='in')
    ax1.xaxis.set_ticks_position('both')
    ax1.yaxis.set_ticks_position('both')
    x_ = np.linspace(0,np.pi,500)
    n_colors = len(color_dict)
    i=-1
    for color_name, color_val in color_dict.items():
        i+=1
        phase_ = - np.pi*(i/(n_colors-1))
        ln1, = ax1.plot( x_,
                         np.cos(2*x_ + phase_ ),
                         c=color_val,
                         lw=1.0, ls='solid', zorder=1 )
        print(f'{color_name} {str(color_val)}')
    fig1.tight_layout(pad=0.25)
    fig1.tight_layout(pad=0.25)
    plt.show()
    
    return

def color_dict_to_tex(color_dict):
    '''
    print colors as Tex definitions
    '''
    for color_name, color_val in color_dict.items():
        print(f'\definecolor{{{color_name}}}{{HTML}}{{{color_val.replace("#","")}}}')
    return

def get_Lch_colors(hues,**kwargs):
    '''
    given a list of hues [0-360], chroma & luminance, return
        colors in hex (html) or rgb format
    -----
    Lch color picker : https://css.land/lch/
    '''
    
    c = kwargs.get('c',110) ## chroma
    L = kwargs.get('L',65) ## luminance
    fmt = kwargs.get('fmt','rgb') ## output format : hex or rgb tuples
    test_plot = kwargs.get('test_plot',False) ## plot to test colors
    
    colors_rgb=[]
    colors_Lab=[]
    for h in hues:
        hX = h*np.pi/180
        LX,a,b = skimage.color.lch2lab([L,c,hX])
        colors_Lab.append([LX,a,b])
        r,g,b = skimage.color.lab2rgb([LX,a,b])
        colors_rgb.append([r,g,b])
    
    if test_plot:
        nc = len(colors_rgb)
        x = np.linspace(0,2*np.pi,1000)
        plt.close('all')
        fig1 = plt.figure(figsize=(4, 4*(9/16)), dpi=300)
        ax1 = plt.gca()
        for i in range(nc):
            ax1.plot(x, np.cos(x-i*np.pi/nc), c=colors_rgb[i])
        fig1.tight_layout(pad=0.15)
        plt.show()
    
    if (fmt=='rgb'):
        colors=colors_rgb
    elif (fmt=='hex'):
        colors=[mpl.colors.to_hex(c) for c in colors_rgb]
    elif (fmt=='Lab'):
        colors=colors_Lab
    else:
        raise NameError('fmt=%s not a valid option'%str(fmt))
    
    return colors

def hex2rgb(hexstr,**kwargs):
    '''
    return (r,g,b) [0-1] from html/hexadecimal
    '''
    base = kwargs.get('base',1)
    hexstr = hexstr.lstrip('#')
    c = tuple(int(hexstr[i:i+2], 16) for i in (0, 2, 4))
    if (base==1):
        c = tuple(i/255. for i in c)
    return c

def hsv_adjust_hex(hex_list,h_fac,s_fac,v_fac,**kwargs):
    '''
    adjust the (h,s,v) values of a list of html color codes (#XXXXXX)
    --> if single #XXXXXX is passed, returns single
    --> margin : adjust proportional to available margin
    '''
    margin = kwargs.get('margin',False)
    
    if isinstance(hex_list, str):
        single=True
        hex_list = [hex_list]
    else:
        single=False
    
    colors_rgb = [ hex2rgb(c) for c in hex_list ]
    colors_hsv = mpl.colors.rgb_to_hsv(colors_rgb)
    for ci in range(len(colors_hsv)):
        c = colors_hsv[ci]
        h, s, v = c
        if margin:
            h_margin = 1. - h
            s_margin = 1. - s
            v_margin = 1. - v
        else:
            h_margin = 1.
            s_margin = 1.
            v_margin = 1.
        h = max(0.,min(1.,h + h_margin*h_fac))
        s = max(0.,min(1.,s + s_margin*s_fac))
        v = max(0.,min(1.,v + v_margin*v_fac))
        colors_hsv[ci] = (h,s,v)
    colors_rgb = mpl.colors.hsv_to_rgb(colors_hsv)
    hex_list_out = [mpl.colors.to_hex(c).upper() for c in colors_rgb]
    if single:
        hex_list_out=hex_list_out[0] ## just the one tuple
    return hex_list_out

def analytical_u_plus_y_plus():
    '''
    return viscous, transitional (Spalding), and log law curves
    '''
    y_plus_viscousLayer = np.logspace(np.log10(0.1), np.log10(12.1), num=200, base=10.)
    u_plus_viscousLayer = np.logspace(np.log10(0.1), np.log10(12.1), num=200, base=10.)
    
    y_plus_logLaw = np.logspace(np.log10(9), np.log10(3000), num=20, base=10.)
    u_plus_logLaw = 1 / 0.41 * np.log(y_plus_logLaw) + 5.2
    
    u_plus_spalding = np.logspace(np.log10(0.1), np.log10(13.1), num=200, base=10.)
    y_plus_spalding = u_plus_spalding + \
                      0.1108*(np.exp(0.4*u_plus_spalding) - 1 \
                      -  0.4*u_plus_spalding \
                      - (0.4*u_plus_spalding)**2/(2*1) \
                      - (0.4*u_plus_spalding)**3/(3*2*1) \
                      - (0.4*u_plus_spalding)**4/(4*3*2*1) )
    
    return y_plus_viscousLayer, u_plus_viscousLayer, y_plus_logLaw , u_plus_logLaw, y_plus_spalding, u_plus_spalding

def nice_log_labels(L):
    '''
    nice compact axis labels for log plots
    '''
    L_out=[]
    for l in L:
        if (l<0.001):
            a = '%0.0e'%l
            a = a.replace('e-0','e-')
            L_out.append(a)
        elif (l>=0.001) and (l<0.01):
            a = '%0.0e'%l
            a = a.replace('e-0','e-')
            L_out.append(a)
        elif (l>=0.01) and (l<0.1):
            a = '%0.0e'%l
            a = a.replace('e-0','e-')
            L_out.append(a)
        elif (l>=0.1) and (l<1):
            L_out.append('%0.1f'%l)
        elif (l>=1):
            L_out.append('%i'%l)
        else:
            print(l)
            sys.exit('uh-oh')
    return L_out

def fig_trim_y(fig, list_of_axes, **kwargs):
    '''
    trims the figure in (y) / height dimension
    - typical use case : single equal aspect figure needs to be scooted / trimmed
    '''
    
    offset_px = kwargs.get('offset_px',10)
    dpi_out   = kwargs.get('dpi',None) ## this can be used to make sure output png px dims is divisible by N
    if (dpi_out is None):
        dpi_out = fig.dpi
    
    fig_px_x, fig_px_y = fig.get_size_inches()*fig.dpi
    #print('fig size px : %i %i'%(fig_px_x, fig_px_y))
    transFigInv = fig.transFigure.inverted()
    mainAxis = list_of_axes[0]
    ##
    x0,  y0,  dx,  dy  = mainAxis.get_position().bounds
    x0A, y0A, dxA, dyA = mainAxis.get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds ### pixel values of the axis tightbox
    #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
    dy_pct = dyA / fig_px_y #; print('dy_pct : %0.6f' % dy_pct)
    x0A, y0A = transFigInv.transform_point([x0A, y0A])
    dxA, dyA = transFigInv.transform_point([dxA, dyA])
    #y_shift = 1.0 - (y0A+dyA)
    y_shift = y0A
    ##
    w = fig.get_figwidth()
    h = fig.get_figheight()
    w_inch_nom = w
    h_inch_nom = h*1.08*dy_pct ## cropped height [in]
    w_px_nom   = w_inch_nom * dpi_out
    h_px_nom   = h_inch_nom * dpi_out
    px_base    = 4
    h_px       = math.ceil(h_px_nom/px_base)*px_base ## make sure height in px divisible by N (video encoding)
    w_px       = int(round(w_px_nom))
    w_inch     = w_px / dpi_out
    h_inch     = h_px / dpi_out
    ##
    fig.set_size_inches(w_inch,h_inch,forward=True)
    fig_px_x, fig_px_y = fig.get_size_inches()*dpi_out # ; print('fig size px : %0.6f %0.6f'%(fig_px_x, fig_px_y))
    w_adj = fig.get_figwidth()
    h_adj = fig.get_figheight()
    ## do shift
    for axis in list_of_axes:
        x0, y0, dx, dy  = axis.get_position().bounds
        x0n = x0
        y0n = y0-y_shift+(offset_px/fig_px_y)
        dxn = dx
        dyn = dy
        axis.set_position([x0n,y0n*(h/h_adj),dxn,dyn*(h/h_adj)])
    return

def fig_trim_x(fig, list_of_axes, **kwargs):
    '''
    trims the figure in (x) / width dimension
    - typical use case : single equal aspect figure needs to be scooted / trimmed
    '''
    
    offset_px = kwargs.get('offset_px',10)
    dpi_out   = kwargs.get('dpi',None) ## this is used to make sure OUTPUT png px dims are divisible by N
    if (dpi_out is None):
        dpi_out = fig.dpi
    
    fig_px_x, fig_px_y = fig.get_size_inches()*dpi_out
    #print('fig size px : %i %i'%(fig_px_x, fig_px_y))
    transFigInv = fig.transFigure.inverted()
    
    w = fig.get_figwidth()
    h = fig.get_figheight()
    #print('w, h : %0.2f %0.2f'%(w, h))
    
    nax = len(list_of_axes)
    
    ax_tb_pct = np.zeros( (nax,4) , dtype=np.float64 )
    ax_tb_px  = np.zeros( (nax,4) , dtype=np.float64 )
    for ai, axis in enumerate(list_of_axes):
        
        ## percent values of the axis tightbox
        x0, y0, dx, dy  = axis.get_position().bounds
        #print('x0, y0, dx, dy : %0.2f %0.2f %0.2f %0.2f'%(x0, y0, dx, dy))
        ax_tb_pct[ai,:] = np.array([x0, y0, dx, dy])
        
        ## pixel values of the axis tightbox
        x0, y0, dx, dy = axis.get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=True).bounds
        #print('x0, y0, dx, dy : %0.2f %0.2f %0.2f %0.2f'%(x0, y0, dx, dy))
        axis_tb_px  = np.array([x0, y0, dx, dy])
        axis_tb_px *= (dpi_out/fig.dpi) ## scale by dpi ratio [png:screen]
        ax_tb_px[ai,:] = axis_tb_px
    
    ## current width of (untrimmed) margins (in [x])
    marg_R_px = fig_px_x - (ax_tb_px[:,0] + ax_tb_px[:,2]).max()
    marg_L_px = ax_tb_px[:,0].min()
    #print('marg_L_px : %0.2f'%(marg_L_px,))
    #print('marg_R_px : %0.2f'%(marg_R_px,))
    
    ## n pixels to move all axes left by (fig canvas is 'trimmed' from right)
    x_shift_px = marg_L_px - offset_px
    #print('x_shift_px : %0.2f'%(x_shift_px,))
    
    ## get new canvas size
    ## make sure height in px divisible by N (important for video encoding)
    px_base = 8
    w_px    = fig_px_x - marg_L_px - marg_R_px + 2*offset_px
    w_px    = math.ceil(w_px/px_base)*px_base
    #w_px   += 1*px_base ## maybe helpful in case labels have \infty etc., where get_position() slightly underestimates
    h_px    = fig_px_y
    #print('w_px, h_px : %0.2f %0.2f'%(w_px, h_px))
    w_inch  = w_px / dpi_out
    h_inch  = h_px / dpi_out
    
    ## get shifted axis bound values
    for ai, axis in enumerate(list_of_axes):
        x0, y0, dx, dy = axis.get_position().bounds
        x0n = x0 - x_shift_px/fig_px_x
        y0n = y0
        dxn = dx
        dyn = dy
        ax_tb_pct[ai,:] = np.array([x0n, y0n, dxn, dyn])
        #print('x0n, y0n, dxn, dyn : %0.4f %0.4f %0.4f %0.4f'%(x0n, y0n, dxn, dyn))
    
    ## resize canvas
    fig.set_size_inches( w_inch, h_inch, forward=True )
    fig_px_x, fig_px_y = fig.get_size_inches()*dpi_out
    w_adj = fig.get_figwidth()
    h_adj = fig.get_figheight()
    #print('w_adj, h_adj : %0.2f %0.2f'%(w_adj, h_adj))
    #print('w_adj, h_adj : %0.2f %0.2f'%(w_adj*dpi_out, h_adj*dpi_out))
    
    ## do shift
    for ai, axis in enumerate(list_of_axes):
        x0n, y0n, dxn, dyn = ax_tb_pct[ai,:]
        axis.set_position( [ x0n*(w/w_adj) , y0n , dxn*(w/w_adj) , dyn ] )
    
    return

def axs_grid_initializer(**kwargs):
    '''
    initialize several axes in a grid
    '''
    
    w_prop    = kwargs.get('w_prop',True) ## peg to width vals
    figsize_x = kwargs.get('figsize_x',6)
    figsize_y = kwargs.get('figsize_y',4) ## figsizes in 
    mrg_l     = kwargs.get('mrg_l',0.05)
    mrg_r     = kwargs.get('mrg_r',0.05)
    mrg_w     = kwargs.get('mrg_w',0.05)
    mrg_t     = kwargs.get('mrg_t',0.05)
    mrg_b     = kwargs.get('mrg_b',0.05)
    mrg_h     = kwargs.get('mrg_h',0.05)
    
    cols = kwargs.get('cols',2)
    rows = kwargs.get('rows',1)
    
    dpi_fig = kwargs.get('dpi_fig',110)
    
    
    fig = plt.figure(figsize=(figsize_x,figsize_y), dpi=dpi_fig)
    aspect = figsize_x/figsize_y
    if w_prop:
        mrg_t *= aspect
        mrg_b *= aspect
        mrg_h *= aspect
    
    ax_width  = (1.0 - (cols-1)*mrg_w - mrg_l - mrg_r)/cols
    ax_height = (1.0 - (rows-1)*mrg_h - mrg_t - mrg_b)/rows
    axs          = np.zeros(shape=(cols,rows),   dtype=object)
    ax_grid_dims = np.zeros(shape=(cols,rows,4), dtype=np.float64) ## the original axis positions
    for j in range(rows):
        for i in range(cols):
            ii = i ## left to right
            jj = (rows-1) - j ## top to bottom
            x0 = mrg_l + ii*mrg_w + ii*ax_width
            dx = ax_width
            y0 = mrg_b + jj*mrg_h + jj*ax_height
            dy = ax_height
            ax = fig.add_axes([x0,y0,dx,dy])
            
            ax_grid_dims[i,j,:] = x0,y0,dx,dy
            axs[i,j] = ax
    
    ## axs          = np.asarray(axs)
    ## ax_grid_dims = np.asarray(ax_grid_dims, dtype=object)
    ## axs          = np.reshape(axs,          (cols, rows),    order='F')
    ## ax_grid_dims = np.reshape(ax_grid_dims, (cols, rows, 4), order='F') ; print(ax_grid_dims.shape)
    
    return fig, axs, ax_grid_dims

def axs_grid_compress(fig,axs,**kwargs):
    '''
    compress ax grid
    '''
    dim = kwargs.get('dim',1)
    offset_px = kwargs.get('offset_px',5)
    transFigInv = fig.transFigure.inverted()
    
    ## on screen pixel size
    fig_px_x, fig_px_y = fig.get_size_inches()*fig.dpi
    #print('fig size px : %0.3f %0.3f'%(fig_px_x, fig_px_y))
    
    cols, rows = axs.shape
    for j in range(rows-1):
        
        ### determine the min x0 in each row
        top_row_y0s = []
        low_row_y1s = []
        for i in range(cols):
            
            x0,  y0,  dx,  dy  = axs[i,j+1].get_position().bounds
            
            ### pixel values of the axis tightbox
            x0A, y0A, dxA, dyA = axs[i,j+0].get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds
            x0B, y0B, dxB, dyB = axs[i,j+1].get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds
            #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
            #print('x0B, y0B, dxB, dyB : %0.2f %0.2f %0.2f %0.2f'%(x0B, y0B, dxB, dyB))
            
            ### convert pixel vals to dimless
            x0A, y0A = transFigInv.transform_point([x0A, y0A])
            dxA, dyA = transFigInv.transform_point([dxA, dyA])
            x0B, y0B = transFigInv.transform_point([x0B, y0B])
            dxB, dyB = transFigInv.transform_point([dxB, dyB])
            #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
            #print('x0B, y0B, dxB, dyB : %0.2f %0.2f %0.2f %0.2f'%(x0B, y0B, dxB, dyB))
            #print('\n')
            
            top_row_y0s.append(y0A)
            low_row_y1s.append(y0B+dyB)
        
        y_shift = min(top_row_y0s) - max(low_row_y1s)
        
        # =====
        
        for i in range(cols):
            
            x0,  y0,  dx,  dy  = axs[i,j+1].get_position().bounds
            
            # x0A, y0A, dxA, dyA = axs[i,j+0].get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds
            # x0B, y0B, dxB, dyB = axs[i,j+1].get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds
            # #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
            # #print('x0B, y0B, dxB, dyB : %0.2f %0.2f %0.2f %0.2f'%(x0B, y0B, dxB, dyB))
            # 
            # ### convert pixel vals to dimless
            # x0A, y0A = transFigInv.transform_point([x0A, y0A])
            # dxA, dyA = transFigInv.transform_point([dxA, dyA])
            # x0B, y0B = transFigInv.transform_point([x0B, y0B])
            # dxB, dyB = transFigInv.transform_point([dxB, dyB])
            # #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
            # #print('x0B, y0B, dxB, dyB : %0.2f %0.2f %0.2f %0.2f'%(x0B, y0B, dxB, dyB))
            # #print('\n')
            
            x0n = x0
            #y0n = y0+(y0A-(y0B+dyB))-(offset_px/fig_px_y)
            y0n = y0+y_shift-(offset_px/fig_px_y)
            dxn = dx
            dyn = dy
            
            axs[i,j+1].set_position([x0n,y0n,dxn,dyn])
    
    return

def tight_layout_helper_ax_with_cbar(fig,ax,cax,**kwargs):
    '''
    shift cbar in x, then expand main axis dx to fill space
    --> call tight_layout(pad=X) first to set base padding
    '''
    
    transFigInv = fig.transFigure.inverted()
    transFig    = fig.transFigure
    
    ### this is the ON SCREEN pixels size... has nothing to do with output
    fig_px_x, fig_px_y = fig.get_size_inches()*fig.dpi
    #print('fig size px : %0.3f %0.3f'%(fig_px_x, fig_px_y))
    
    ### axis position (dimless) --> NOT tightbox!
    x0A, y0A, dxA, dyA  = ax.get_position().bounds
    #print('x0A, y0A, dxA, dyA : %0.6f %0.6f %0.6f %0.6f'%(x0A, y0A, dxA, dyA))
    
    ### pixel values of axis frame (NOT tightbox)
    x0App, y0App = transFig.transform_point([x0A, y0A])
    dxApp, dyApp = transFig.transform_point([dxA, dyA])
    #x0Bpp, y0Bpp = transFig.transform_point([x0B, y0B])
    #dxBpp, dyBpp = transFig.transform_point([dxB, dyB])
    #print('x0App, y0App, dxApp, dyApp : %0.6f %0.6f %0.6f %0.6f'%(x0App, y0App, dxApp, dyApp))
    #print('x0Bpp, y0Bpp, dxBpp, dyBpp : %0.6f %0.6f %0.6f %0.6f'%(x0Bpp, y0Bpp, dxBpp, dyBpp))
    
    x0B, y0B, dxB, dyB  = cax.get_position().bounds
    #print('x0B, y0B, dxB, dyB : %0.6f %0.6f %0.6f %0.6f'%(x0B, y0B, dxB, dyB))
    
    ### pixel values of the axis tightbox (NOT the axis position)
    x0Ap, y0Ap, dxAp, dyAp = ax.get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=True).bounds
    #print('x0Ap, y0Ap, dxAp, dyAp : %0.6f %0.6f %0.6f %0.6f'%(x0Ap, y0Ap, dxAp, dyAp))
    
    x0Bp, y0Bp, dxBp, dyBp = cax.get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=True).bounds
    #print('x0Bp, y0Bp, dxBp, dyBp : %0.6f %0.6f %0.6f %0.6f'%(x0Bp, y0Bp, dxBp, dyBp))
    
    ### in pixels, figure out the x0 of the cbar axis
    ### use x0Ap : the number of pixels to the L of the main axis --> determined independently by tight_layout()
    x0Bpn = fig_px_x - x0Ap - dxBp
    x0Bn, _ = transFigInv.transform_point([x0Bpn, 0.])
    cax.set_position([x0Bn, y0B, dxB, dyB])
    
    end_ax_px_target = x0Bpn-2*x0Ap
    end_ax_px        = x0Ap + dxAp
    delta_px = end_ax_px_target - end_ax_px
    
    fac = (dxApp + delta_px)/dxApp
    
    ax.set_position([x0A, y0A, dxA*fac, dyA])
    x0Ap, y0Ap, dxAp, dyAp = ax.get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=True).bounds
    #print('x0Ap, y0Ap, dxAp, dyAp : %0.6f %0.6f %0.6f %0.6f'%(x0Ap, y0Ap, dxAp, dyAp))
    #print(x0Ap+dxAp)
    
    return

def cmap_convert_mpl_to_pview(cmap,fname,cmap_name,**kwargs):
    '''
    convert python/matplotlib cmap object to JSON for Paraview
    '''
    
    # === we dont want to mess up the current cmap/norm when we set_array() and autoscale()
    cmapX = copy.deepcopy(cmap)
    #normX = copy.deepcopy(norm)
    
    N = kwargs.get('N',256)
    #lo = kwargs.get('lo',0.)
    #hi = kwargs.get('hi',1.)
    #norm = kwargs.get('norm',mpl.colors.Normalize(vmin=0.0, vmax=1.0))
    
    sclMap = mpl.cm.ScalarMappable(cmap=cmapX)
    x      = np.linspace(0,1,N+1)
    sclMap.set_array(x)
    sclMap.autoscale()
    colors = sclMap.to_rgba(x)
    
    ## output .json formatted ascii file
    #f = open(fname,'w')
    f = io.open(fname,'w',newline='\n')
    
    out_str='''[
    {
        "ColorSpace" : "RGB",
        "Name" : "%s",
        "RGBPoints" : 
        ['''%(cmap_name, )
    
    f.write(out_str)
    
    for i in range(len(x)):
        c = x[i]
        if (i==len(x)-1):
            maybeComma=''
        else:
            maybeComma=','
        color=colors[i]
        out_str='''
            %0.6f,
            %0.17f,
            %0.17f,
            %0.17f%s'''%(c,color[0],color[1],color[2],maybeComma)
        f.write(out_str)
    
    out_str = '''\n%s]\n%s}\n]'''%(8*' ',4*' ')
    f.write(out_str)
    f.close()
    print('--w-> %s'%fname)
    return

# main()
# ======================================================================

if __name__ == '__main__':
    pass
