#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os, sys, re, glob, time, traceback
import h5py
import mpi4py
from mpi4py import MPI
import pickle
import gzip
import shutil
import psutil
import gc
import pathlib
from pathlib import Path, PurePosixPath
import subprocess
import io
import copy
import datetime
import timeit
import textwrap
from tqdm import tqdm
import math
import numpy as np
import scipy as sp
from scipy import interpolate, integrate, signal, stats, special

from concurrent.futures import ThreadPoolExecutor

import matplotlib as mpl
import matplotlib.pyplot as plt
import cmocean, colorcet, cmasher

import colorspacious
from colorspacious import cspace_converter

## required for EAS3
import struct

'''
========================================================================

turbx: an extensible toolkit for analyzing turbulent flow datasets
--> version Spring/Summer 2025

$> wget https://raw.githubusercontent.com/iagappel/turbx/main/turbx/turbx.py
$> git clone git@gitlab.iag.uni-stuttgart.de:transi/turbx.git
$> git clone git@github.com:iagappel/turbx.git

h5py Documentation:
https://docs.h5py.org/_/downloads/en/3.13.0/pdf/

compiling parallel HDF5 & h5py:
https://docs.h5py.org/en/stable/mpi.html#building-against-parallel-hdf5

========================================================================
'''

# data container interface classes for HDF5 containers
# ======================================================================

class cgd(h5py.File):
    '''
    Curvilinear Grid Data (CGD)
    ---------------------------
    - super()'ed h5py.File class
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.cgd()')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## cgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 16    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2     )
        perms          = kwargs.pop('perms'          , '640' )
        
        ## passthrough arg to get_header() to automatically read the full 3D grid on every MPI rank
        ## upon opening. This is in general a bad idea for CGD, which has a 3D grid. If every rank reads the
        ## full grid, RAM can very quickly fill up.
        read_grid = kwargs.pop('read_grid', False)
        
        if not isinstance(stripe_count, int):
            raise ValueError
        if not isinstance(stripe_size_mb, int):
            raise ValueError
        if not isinstance(perms, str):
            raise ValueError
        if not len(perms)==3:
            raise ValueError
        if not re.fullmatch(r'\d{3}',perms):
            raise ValueError
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using romio --> causes crash
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        
        ## set ROMIO hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(16*1024**2))) ) ## 16 [MB]
                ##
                #mpi_info.Set('romio_no_indep_rw' , 'true' ) ## deferred open + only collective I/O 
                #mpi_info.Set('cb_nodes' , str(int(round(1*self.n_ranks))) )
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        ## | rdcc_nbytes:
        ## | ------------
        ## | Integer setting the total size of the raw data chunk cache for this dataset in bytes.
        ## | In most cases increasing this number will improve performance, as long as you have 
        ## | enough free memory. The default size is 1 MB
        
        ## --> gets passed to H5Pset_chunk_cache
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        ## | rdcc_nslots:
        ## | ------------
        ## | Integer defining the number of chunk slots in the raw data chunk cache for this dataset.
        
        ## if ('rdcc_nslots' not in kwargs):
        ##     kwargs['rdcc_nslots'] = 521
        
        ## rgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with cgd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(0.1)
                Path(self.fname).touch() ## touch a new file
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        ## call actual h5py.File.__init__()
        super(cgd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose, read_grid=read_grid)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(cgd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed CGD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(cgd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of CGD class instance
        '''
        
        verbose = kwargs.get('verbose',True)
        
        ## by default, do NOT read CGD grid upon opening
        ## leave it up to the individual functions to read grid into RAM
        read_grid = kwargs.get('read_grid',False) 
        
        if (self.rank!=0):
            verbose=False
        
        # if verbose: print('\n'+'cgd.get_header()'+'\n'+72*'-')
        # t_start_func = timeit.default_timer()
        
        # === attrs
        if ('duration_avg' in self.attrs.keys()):
            self.duration_avg = self.attrs['duration_avg']
        if ('rectilinear' in self.attrs.keys()):
            self.rectilinear = self.attrs['rectilinear']
        if ('curvilinear' in self.attrs.keys()):
            self.curvilinear = self.attrs['curvilinear']
        
        ## these should be set in the (init_from_() funcs)
        if ('fclass' in self.attrs.keys()):
            self.fclass = self.attrs['fclass'] ## 'rgd','cgd',...
        if ('fsubtype' in self.attrs.keys()):
            self.fsubtype = self.attrs['fsubtype'] ## 'unsteady','mean','prime',...
        
        # === udef
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## make dict where keys are udef_char and values are udef_real
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            self.S_Suth      = self.udef['S_Suth']
            #self.C_Suth      = self.udef['C_Suth']
            
            self.C_Suth = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## [kg/(m·s·√K)]
            self.udef['C_Suth'] = self.C_Suth
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            
            # === characteristic values : derived
            
            ## mu_inf_1 = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            ## mu_inf_2 = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            ## mu_inf_3 = self.C_Suth*self.T_inf**(3/2)/(self.T_inf+self.S_Suth)
            ## if not np.isclose(mu_inf_1, mu_inf_2, rtol=1e-14):
            ##     raise AssertionError('inconsistency in Sutherland calc --> check')
            ## if not np.isclose(mu_inf_2, mu_inf_3, rtol=1e-14):
            ##     raise AssertionError('inconsistency in Sutherland calc --> check')
            ## mu_inf = self.mu_inf = mu_inf_2
            
            self.mu_inf    = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            self.rho_inf   = self.p_inf/(self.R*self.T_inf)
            self.nu_inf    = self.mu_inf/self.rho_inf
            self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            self.U_inf     = self.Ma*self.a_inf
            self.cp        = self.R*self.kappa/(self.kappa-1.)
            self.cv        = self.cp/self.kappa
            self.recov_fac = self.Pr**(1/3)
            self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            self.tchar = self.lchar / self.U_inf
            self.uchar = self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            if verbose: even_print('tchar'           , '%0.6E [s]'        % self.tchar     )
            if verbose: print(72*'-')
            #if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the CGD instance
            self.udef_deriv = { 'rho_inf':self.rho_inf,
                                'mu_inf':self.mu_inf,
                                'nu_inf':self.nu_inf,
                                'a_inf':self.a_inf,
                                'U_inf':self.U_inf,
                                'cp':self.cp,
                                'cv':self.cv,
                                'recov_fac':self.recov_fac,
                                'Taw':self.Taw,
                                'lchar':self.lchar,
                              }
        
        else:
            pass
        
        # === read coordinate vectors
        # - (full) grid will only be read (to every rank!!!) if read_grid=True
        # - reading the grid is often not necessary. when it is necessary, just read it directly from the HDF5/h5py handle in
        #    the corresponding function
        
        if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
            
            if read_grid:
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## read 3D/1D coordinate arrays
                ## ( dont transpose right away --> allows for 1D storage )
                
                if self.usingmpi:
                    
                    dset = self['dims/x']
                    with dset.collective:
                        x = self.x = np.copy( dset[()] )
                    
                    self.comm.Barrier()
                    
                    dset = self['dims/y']
                    with dset.collective:
                        y = self.y = np.copy( dset[()] )
                    
                    self.comm.Barrier()
                    
                    dset = self['dims/z']
                    with dset.collective:
                        z = self.z = np.copy( dset[()] )
                
                else:
                    
                    x = self.x = np.copy( self['dims/x'][()] )
                    y = self.y = np.copy( self['dims/y'][()] )
                    z = self.z = np.copy( self['dims/z'][()] )
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ( x.nbytes + y.nbytes + z.nbytes ) * self.n_ranks / 1024**3
                if verbose:
                    even_print('read x,y,z (full)', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                if True: ## transpose the coordinate arrays
                    
                    '''
                    nx,ny,nz should probably just be stored as attributes
                    '''
                    
                    if (x.ndim==1):
                        nx = self.nx = x.shape[0]
                    elif (x.ndim==3):
                        x  = self.x  = np.copy( x.T )
                        nx = self.nx = x.shape[0]
                    else:
                        raise AssertionError('x.ndim=%i'%(x.ndim,))
                    
                    if (y.ndim==1):
                        ny = self.ny = y.shape[0]
                    elif (y.ndim==3):
                        y  = self.y  = np.copy( y.T )
                        ny = self.ny = y.shape[1]
                    else:
                        raise AssertionError('y.ndim=%i'%(y.ndim,))
                    
                    if (z.ndim==1):
                        nz = self.nz = z.shape[0]
                    elif (z.ndim==3):
                        z  = self.z  = np.copy( z.T )
                        nz = self.nz = z.shape[2]
                    else:
                        raise AssertionError('z.ndim=%i'%(z.ndim,))
            
            else:
                
                x_shp = self['dims/x'].shape
                y_shp = self['dims/y'].shape
                z_shp = self['dims/z'].shape
                
                ## assert 3D
                if (self['dims/x'].ndim != 3):
                    raise ValueError
                
                ## assert shapes agree 
                if (x_shp != y_shp):
                    raise AssertionError
                if (y_shp != z_shp):
                    raise AssertionError
                
                ## datasets are stored transposed!
                nz,ny,nx = self['dims/x'].shape
                self.nx = nx
                self.ny = ny
                self.nz = nz
                
                self.x = None
                self.y = None
                self.z = None
            
            ngp = self.ngp = nx*ny*nz
            
            if verbose: even_print('nx',  '%i'%nx  )
            if verbose: even_print('ny',  '%i'%ny  )
            if verbose: even_print('nz',  '%i'%nz  )
            if verbose: even_print('ngp', '%i'%ngp )
            #if verbose: print(72*'-')
            
            if False:
                if verbose: even_print('x_min', '%0.2f'%x.min())
                if verbose: even_print('x_max', '%0.2f'%x.max())
                if verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (x[1]-x[0]), (x[-1]-x[-2]) ))
                if verbose: even_print('y_min', '%0.2f'%y.min())
                if verbose: even_print('y_max', '%0.2f'%y.max())
                if verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (y[1]-y[0]), (y[-1]-y[-2]) ))
                if verbose: even_print('z_min', '%0.2f'%z.min())
                if verbose: even_print('z_max', '%0.2f'%z.max())        
                if verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (z[1]-z[0]), (z[-1]-z[-2]) ))
                if verbose: print(72*'-'+'\n')
        
        # === time vector
        
        if ('dims/t' in self):
            
            self.t = np.copy(self['dims/t'][()])
            
            if ('data' in self): ## check t dim and data arr agree
                nt,_,_,_ = self['data/%s'%list(self['data'].keys())[0]].shape
                if (nt!=self.t.size):
                    raise AssertionError('nt!=self.t.size : %i!=%i'%(nt,self.t.size))
            
            try:
                self.dt = self.t[1] - self.t[0]
            except IndexError:
                self.dt = 0.
            
            self.nt       = nt       = self.t.size
            self.duration = duration = self.t[-1] - self.t[0]
            self.ti       = ti       = np.arange(self.nt, dtype=np.int64)
        
        elif all([ ('data' in self) , ('dims/t' not in self) ]): ## data but no time --> make dummy time vector
            self.scalars = list(self['data'].keys())
            nt,_,_,_ = self['data/%s'%self.scalars[0]].shape
            self.nt  = nt
            self.t   =      np.arange(self.nt, dtype=np.float64)
            self.ti  = ti = np.arange(self.nt, dtype=np.int64)
            self.dt  = 1.
            self.duration = duration = self.t[-1]-self.t[0]
        
        else: ## no data, no time
            self.t  = np.array([], dtype=np.float64)
            self.ti = np.array([], dtype=np.int64)
            self.nt = nt = 0
            self.dt = 0.
            self.duration = duration = 0.
        
        #if verbose: print(72*'-')
        if verbose: even_print('nt', '%i'%self.nt )
        if verbose: even_print('dt', '%0.6f'%self.dt)
        if verbose: even_print('duration', '%0.2f'%self.duration )
        if hasattr(self, 'duration_avg'):
            if verbose: even_print('duration_avg', '%0.2f'%self.duration_avg )
        #if verbose: print(72*'-'+'\n')
        
        if hasattr(self,'rectilinear'):
            if verbose: even_print('rectilinear', str(self.rectilinear) )
        if hasattr(self,'curvilinear'):
            if verbose: even_print('curvilinear', str(self.curvilinear) )
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys()) ## 4D : string names of scalars : ['u','v','w'],...
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self[f'data/{scalar}'].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    # === I/O
    
    def init_from_eas4(self, fn_eas4, **kwargs):
        '''
        initialize a CGD from an EAS4 (NS3D output format)
        '''
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        verbose  = kwargs.get('verbose',True)
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        if (self.rank!=0):
            verbose=False
        
        # === spatial resolution filter : take every nth grid point
        sx = kwargs.get('sx',1)
        sy = kwargs.get('sy',1)
        sz = kwargs.get('sz',1)
        #st = kwargs.get('st',1)
        
        # === spatial resolution filter : set x/y/z bounds
        xi_min = kwargs.get('xi_min',None)
        yi_min = kwargs.get('yi_min',None)
        zi_min = kwargs.get('zi_min',None)
        
        xi_max = kwargs.get('xi_max',None)
        yi_max = kwargs.get('yi_max',None)
        zi_max = kwargs.get('zi_max',None)
        
        ## grid filters are currently not supported for CGD
        if (xi_min is not None):
            raise NotImplementedError
        if (yi_min is not None):
            raise NotImplementedError
        if (zi_min is not None):
            raise NotImplementedError
        
        if (xi_max is not None):
            raise NotImplementedError
        if (yi_max is not None):
            raise NotImplementedError
        if (zi_max is not None):
            raise NotImplementedError
        
        ## grid filters are currently not supported for CGD
        self.hasGridFilter=False
        
        ## set default attributes
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'cgd'
        
        if verbose: print('\n'+'cgd.init_from_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        #if verbose: even_print('infile', f'{os.path.basename(os.path.dirname(fn_eas4))}/{os.path.basename(fn_eas4)}')
        #if verbose: even_print('infile', f'{Path(fn_eas4).parent.resolve().name}/{Path(fn_eas4).name}')
        if verbose: even_print('infile', str(Path(fn_eas4).relative_to(Path())))
        if verbose: even_print('infile size', '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3))
        if verbose: even_print('outfile', self.fname)
        
        with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=MPI.COMM_WORLD, read_3d_grid=False) as hf_eas4:
            
            ## original GMODE in eas4 file
            ## remember than 1,2 get automatically expanded to 4 during eas4.get_header()
            # if verbose: even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1_orig, gmode_dict[hf_eas4.gmode_dim1_orig] ) )
            # if verbose: even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2_orig, gmode_dict[hf_eas4.gmode_dim2_orig] ) )
            # if verbose: even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3_orig, gmode_dict[hf_eas4.gmode_dim3_orig] ) )
            
            if verbose: even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1, gmode_dict[hf_eas4.gmode_dim1] ) )
            if verbose: even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2, gmode_dict[hf_eas4.gmode_dim2] ) )
            if verbose: even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3, gmode_dict[hf_eas4.gmode_dim3] ) )
            
            if verbose: even_print( 'rectilinear' , str(hf_eas4.is_rectilinear) )
            if verbose: even_print( 'curvilinear' , str(hf_eas4.is_curvilinear) )
            
            self.attrs['rectilinear'] = hf_eas4.is_rectilinear
            self.attrs['curvilinear'] = hf_eas4.is_curvilinear
            #self.rectilinear = self.attrs['rectilinear']
            #self.curvilinear = self.attrs['curvilinear']
            
            # === copy over header info if needed
            
            if all([('header/udef_real' in self),('header/udef_char' in self)]):
                raise ValueError('udef already present')
            else:
                udef         = hf_eas4.udef
                udef_real    = list(udef.values())
                udef_char    = list(udef.keys())
                udef_real_h5 = np.array(udef_real, dtype=np.float64)
                udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
                
                self.create_dataset('header/udef_real', data=udef_real_h5, dtype=np.float64)
                self.create_dataset('header/udef_char', data=udef_char_h5, dtype='S128')
                self.udef      = udef
                self.udef_real = udef_real
                self.udef_char = udef_char
            
            # === copy over dims info
            
            if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
                raise ValueError('dims/x, dims/y, dims/z already in CGD file')
            
            ## do NOT copy EAS4 grid into memory at this point!
            #x = np.copy(hf_eas4.x)
            #y = np.copy(hf_eas4.y)
            #z = np.copy(hf_eas4.z)
            
            self.nx = nx = hf_eas4.nx
            self.ny = ny = hf_eas4.ny
            self.nz = nz = hf_eas4.nz
            
            ngp = nx*ny*nz
            ## nt = hf_eas4.nt --> no time data yet
            
            # === rank 3D grid ranges
            
            if self.usingmpi:
                
                comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                t4d = comm4d.Get_coords(self.rank)
                
                rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
                ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
                rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
                #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
                
                rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                
                rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
            
            else:
                
                rx1 = 0
                rx2 = self.nx
                ry1 = 0
                ry2 = self.ny
                rz1 = 0
                rz2 = self.nz
                
                nxr = self.nx
                nyr = self.ny
                nzr = self.nz
                #ntr = self.nt
            
            # === 3D grid expansion --> the grid is expanded rank-local for CGD, can easily be too big otherwise
            
            ## if ANY grid dimension has FULL_G
            if any([ (hf_eas4.gmode_dim1==5) , (hf_eas4.gmode_dim2==5) , (hf_eas4.gmode_dim3==5) ]):
                
                ## GMODE=(5,5,5)
                if all([ (hf_eas4.gmode_dim1==5) , (hf_eas4.gmode_dim2==5) , (hf_eas4.gmode_dim3==5) ]):
                    
                    raise NotImplementedError ## sorry :( this needs to be implemented as a collective read
                    
                    ## collective
                    # ...
                    
                    ## independent
                    # dset = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim01']
                    # x = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                    # dset = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim02']
                    # y = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                    # dset = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim03']
                    # z = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                
                ## GMODE=(5,5,4)
                elif all([ (hf_eas4.gmode_dim1==5) , (hf_eas4.gmode_dim2==5) , (hf_eas4.gmode_dim3==4) ]):
                    
                    x_ = np.copy(hf_eas4.x) ## will be 2D+1 from eas4() --> (nx,ny,1)
                    y_ = np.copy(hf_eas4.y) ## will be 2D+1 from eas4() --> (nx,ny,1)
                    z_ = np.copy(hf_eas4.z) ## will be 1D   from eas4() --> (nz,)
                    
                    ## assert that ndim is indeed correct
                    if (x_.ndim!=3):
                        raise ValueError
                    if (y_.ndim!=3):
                        raise ValueError
                    if (z_.ndim!=1):
                        raise ValueError
                    
                    ## rank-local 3D coordinate buffer
                    x = np.zeros((nxr,nyr,nzr), dtype=np.float64)
                    y = np.zeros((nxr,nyr,nzr), dtype=np.float64)
                    z = np.zeros((nxr,nyr,nzr), dtype=np.float64)
                    
                    ## broadcast sections of 1D/2D to rank-local 3D
                    x[:,:,:] = x_[rx1:rx2,ry1:ry2,:] ## broadcast on axis 2
                    y[:,:,:] = y_[rx1:rx2,ry1:ry2,:] ## broadcast on axis 2
                    z[:,:,:] = z_[rz1:rz2]           ## broadcast on axes (0,1)
                
                else:
                    
                    ## GMODE combos like (4,5,5) / (5,4,5) not yet implemented
                    raise NotImplementedError
            
            else: ## if no dim has FULL_G --> this means a rectilinear grid is being read, so rgd() could actually be used
                
                ## GMODE=(4,4,4)
                ## remember that eas4() automatically expands 1,2 to 4, so e.g. EAS4 (4,4,2) automatically becomes (4,4,4)
                if all([ (hf_eas4.gmode_dim1==4) , (hf_eas4.gmode_dim2==4) , (hf_eas4.gmode_dim3==4) ]):
                    
                    x = np.copy(hf_eas4.x)
                    y = np.copy(hf_eas4.y)
                    z = np.copy(hf_eas4.z)
                    
                    ## assert 1D
                    if (x.ndim!=1):
                        raise ValueError
                    if (y.ndim!=1):
                        raise ValueError
                    if (z.ndim!=1):
                        raise ValueError
                    
                    ## expand to 3D
                    #x, y, z = np.meshgrid(x, y, z, indexing='ij')
                    x, y, z = np.meshgrid(x[rx1:rx2], y[ry1:ry2], z[rz1:rz2], indexing='ij') ## rank-local
                
                else:
                    raise NotImplementedError
            
            # === by this point, the grid should have been expanded to 3D
            
            if (x.ndim!=3):
                raise ValueError('grid should be 3D expanded (rank-local) at this point')
            if (y.ndim!=3):
                raise ValueError('grid should be 3D expanded (rank-local) at this point')
            if (z.ndim!=3):
                raise ValueError('grid should be 3D expanded (rank-local) at this point')
            
            ## old
            ## broadcast in dimensions with shape=1
            ## this can easily eat up all RAM in MPI parallel mode
            if False:
                if ( hf_eas4.gmode_dim1==5 ) and ( x.shape != (nx,ny,nz) ):
                    x = np.broadcast_to(x, (nx,ny,nz))
                    if verbose: print('broadcasted x')
                
                if ( hf_eas4.gmode_dim2==5 ) and ( y.shape != (nx,ny,nz) ):
                    y = np.broadcast_to(y, (nx,ny,nz))
                    if verbose: print('broadcasted y')
                
                if ( hf_eas4.gmode_dim3==5 ) and ( z.shape != (nx,ny,nz) ):
                    z = np.broadcast_to(z, (nx,ny,nz))
                    if verbose: print('broadcasted z')
            
            shape  = (nz,ny,nx)
            chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=4, itemsize=8)
            
            # === write coord arrays
            
            if ('dims/x' in self):
                del self['dims/x']
            if ('dims/y' in self):
                del self['dims/y']
            if ('dims/z' in self):
                del self['dims/z']
            
            if False: ## serial write
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                dset = self.create_dataset('dims/x', data=x.T, shape=shape, chunks=chunks)
                dset = self.create_dataset('dims/y', data=y.T, shape=shape, chunks=chunks)
                dset = self.create_dataset('dims/z', data=z.T, shape=shape, chunks=chunks)
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                if verbose:
                    even_print('write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            if True: ## collective write
                
                ## initialize datasets
                dset_x = self.create_dataset('dims/x', shape=shape, chunks=chunks, dtype=x.dtype)
                dset_y = self.create_dataset('dims/y', shape=shape, chunks=chunks, dtype=y.dtype)
                dset_z = self.create_dataset('dims/z', shape=shape, chunks=chunks, dtype=z.dtype)
                
                chunk_kb_ = np.prod(dset_x.chunks)*8 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (z,y,x)','%s'%str(dset_x.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi: 
                    with dset_x.collective:
                        #dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x.T
                else:
                    dset_x[:,:,:] = x.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_y.collective:
                        #dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y.T
                else:
                    dset_y[:,:,:] = y.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_z.collective:
                        #dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z.T
                else:
                    dset_z[:,:,:] = z.T
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                #data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                data_gb = 3 * 8 * nx * ny * nz / 1024**3
                
                if verbose:
                    even_print('write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                # === write a preliminary time array --> e.g. for baseflow
                
                if ('dims/t' in self):
                    del self['dims/t']
                dset = self.create_dataset( 'dims/t', data=hf_eas4.t )
        
        self.x = np.copy(x)
        self.y = np.copy(y)
        self.z = np.copy(z)
        
        if verbose: print(72*'-')
        self.get_header(verbose=True, read_grid=False)
        if verbose: print(72*'-')
        
        return
    
    def init_from_cgd(self, fn_cgd, **kwargs):
        '''
        initialize an CGD from an CGD (copy over header data & coordinate data)
        '''
        
        t_info = kwargs.get('t_info',True)
        copy_grid = kwargs.get('copy_grid',True)
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        
        verbose = kwargs.get('verbose',True)
        if (self.rank!=0):
            verbose=False
        
        ## set default attributes
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'cgd'
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        with cgd(fn_cgd, 'r', driver=self.driver, comm=self.comm) as hf_ref:
            
            ## copy over fsubtype
            if hasattr(hf_ref,'fsubtype'):
                self.attrs['fsubtype'] = hf_ref.fsubtype
            
            # === copy over header info if needed
            
            if all([('header/udef_real' in self),('header/udef_char' in self)]):
                raise ValueError('udef already present')
            else:
                udef         = hf_ref.udef
                udef_real    = list(udef.values())
                udef_char    = list(udef.keys())
                udef_real_h5 = np.array(udef_real, dtype=np.float64)
                udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
                
                self.create_dataset('header/udef_real', data=udef_real_h5, maxshape=np.shape(udef_real_h5), dtype=np.float64)
                self.create_dataset('header/udef_char', data=udef_char_h5, maxshape=np.shape(udef_char_h5), dtype='S128')
                self.udef      = udef
                self.udef_real = udef_real
                self.udef_char = udef_char
            
            # === copy over dims info
            
            # if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
            #     raise ValueError('dims/x, dims/y, dims/z already in CGD file')
            
            ## make sure source CGD file has grid
            if ('dims/x' not in hf_ref):
                raise ValueError(f'{hf_ref.fname} does not have dset dims/x')
            if ('dims/y' not in hf_ref):
                raise ValueError(f'{hf_ref.fname} does not have dset dims/y')
            if ('dims/z' not in hf_ref):
                raise ValueError(f'{hf_ref.fname} does not have dset dims/z')
            
            ## make sure source CGD grid is 3D
            if (hf_ref['dims/x'].ndim != 3):
                raise ValueError('CGD file x grid has ndim!=3')
            if (hf_ref['dims/y'].ndim != 3):
                raise ValueError('CGD file y grid has ndim!=3')
            if (hf_ref['dims/z'].ndim != 3):
                raise ValueError('CGD file z grid has ndim!=3')
            
            ## no!
            #x   = self.x   = hf_ref.x
            #y   = self.y   = hf_ref.y
            #z   = self.z   = hf_ref.z
            
            nx  = self.nx  = hf_ref.nx
            ny  = self.ny  = hf_ref.ny
            nz  = self.nz  = hf_ref.nz
            ngp = self.ngp = hf_ref.ngp
            
            # === rank 3D grid ranges
            
            if self.usingmpi:
                
                comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                t4d = comm4d.Get_coords(self.rank)
                
                rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
                ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
                rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
                #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
                
                rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                
                rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
            
            else:
                
                nxr = self.nx
                nyr = self.ny
                nzr = self.nz
                #ntr = self.nt
            
            # ===
            
            if copy_grid:
                
                if ('dims/x' in self):
                    del self['dims/x']
                if ('dims/y' in self):
                    del self['dims/y']
                if ('dims/z' in self):
                    del self['dims/z']
                
                ## 1D
                #self.create_dataset('dims/x', data=x)
                #self.create_dataset('dims/y', data=y)
                #self.create_dataset('dims/z', data=z)
                
                ## 3D
                shape  = (nz,ny,nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=4, itemsize=8)
                
                ## if (x is not None) and (y is not None) and (z is not None):
                ##     if self.usingmpi: self.comm.Barrier()
                ##     t_start = timeit.default_timer()
                ##     dset = self.create_dataset('dims/x', data=x.T, shape=shape, chunks=chunks)
                ##     dset = self.create_dataset('dims/y', data=y.T, shape=shape, chunks=chunks)
                ##     dset = self.create_dataset('dims/z', data=z.T, shape=shape, chunks=chunks)
                ##     if self.usingmpi: self.comm.Barrier()
                ##     t_delta = timeit.default_timer() - t_start
                ##     data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                ##     if verbose:
                ##         even_print('write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                ## collective 3D grid read
                dset = hf_ref['dims/x']
                if self.usingmpi:
                    with dset.collective:
                        x = np.copy(dset[rz1:rz2,ry1:ry2,rx1:rx2].T)
                else:
                    x = np.copy(dset[()].T)
                
                dset = hf_ref['dims/y']
                if self.usingmpi:
                    with dset.collective:
                        y = np.copy(dset[rz1:rz2,ry1:ry2,rx1:rx2].T)
                else:
                    y = np.copy(dset[()].T)
                
                dset = hf_ref['dims/z']
                if self.usingmpi:
                    with dset.collective:
                        z = np.copy(dset[rz1:rz2,ry1:ry2,rx1:rx2].T)
                else:
                    z = np.copy(dset[()].T)
                
                ## initialize datasets for write
                dset_x = self.create_dataset('dims/x', shape=shape, chunks=chunks, dtype=np.float64) # dtype=x.dtype)
                dset_y = self.create_dataset('dims/y', shape=shape, chunks=chunks, dtype=np.float64)
                dset_z = self.create_dataset('dims/z', shape=shape, chunks=chunks, dtype=np.float64)
                
                chunk_kb_ = np.prod(dset_x.chunks)*8 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (z,y,x)','%s'%str(dset_x.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi: 
                    with dset_x.collective:
                        #dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x.T
                else:
                    dset_x[:,:,:] = x.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_y.collective:
                        #dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y.T
                else:
                    dset_y[:,:,:] = y.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_z.collective:
                        #dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z.T
                else:
                    dset_z[:,:,:] = z.T
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                #data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                data_gb = 3 * 8 * nx * ny * nz / 1024**3
                
                if verbose:
                    even_print('write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            # === copy over temporal dim info
            
            if t_info:
                self.t  = hf_ref.t
                self.nt = self.t.size
                self.create_dataset('dims/t', data=hf_ref.t)
            else:
                t = np.array([0.], dtype=np.float64)
                if ('dims/t' in self):
                    del self['dims/t']
                self.create_dataset('dims/t', data=t)
            
            # === 
            
            ## add additional [dims/<>] dsets
            for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                if (dsn in hf_ref):
                    if (dsn in self):
                        del self[dsn]
                    data = np.copy(hf_ref[dsn][()])
                    
                    if (dsn=='dims/stang') and (data.shape!=(self.nx,)):
                        raise ValueError
                    if (dsn=='dims/snorm') and (data.shape!=(self.ny,)):
                        raise ValueError
                    
                    ds = self.create_dataset(dsn, data=data, chunks=None)
                    #if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## add additional [csys/<>] dsets
            for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                if (dsn in hf_ref):
                    if (dsn in self):
                        del self[dsn]
                    data = np.copy(hf_ref[dsn][()])
                    
                    if (data.ndim!=3):
                        raise ValueError
                    if (data.shape!=(self.nx,self.ny,2)):
                        raise ValueError
                    
                    ds = self.create_dataset(dsn, data=data, chunks=None)
                    #if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## copy over [data_dim/<>] dsets if present
            if ('data_dim' in hf_ref):
                for dsn in hf_ref['data_dim'].keys():
                    data = np.copy( hf_ref[f'data_dim/{dsn}'][()] ) 
                    self.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                    if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False)
        return
    
    def import_eas4(self, fn_eas4_list, **kwargs):
        '''
        import data from a series of EAS4 files to a CGD
        '''
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        if verbose: print('\n'+'cgd.import_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        tt_min = kwargs.get('tt_min',None)
        tt_max = kwargs.get('tt_max',None)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        # === check for an often made mistake
        ts_min = kwargs.get('ts_min',None)
        ts_max = kwargs.get('ts_max',None)
        if (ts_min is not None):
            raise AssertionError('ts_min is not an option --> did you mean ti_min or tt_min?')
        if (ts_max is not None):
            raise AssertionError('ts_max is not an option --> did you mean ti_max or tt_max?')
        
        # === check that iterable of EAS4 files is OK
        if not hasattr(fn_eas4_list, '__iter__'):
            raise AssertionError('first arg \'fn_eas4_list\' must be iterable')
        for fn_eas4 in fn_eas4_list:
            if not os.path.isfile(fn_eas4):
                raise FileNotFoundError('%s not found!'%fn_eas4)
        
        # === ranks
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        ## skip dimensions --> spatial skips done in init_from_XXX()
        # sx = kwargs.get('sx',1)
        # sy = kwargs.get('sy',1)
        # sz = kwargs.get('sz',1)
        st = kwargs.get('st',1)
        
        ## update this CGD's header and attributes
        self.get_header(verbose=False, read_grid=False)
        
        ## make a communicator for the EAS4 files
        if self.usingmpi:
            comm_eas4 = MPI.COMM_WORLD
            #comm_eas4 = self.comm
        else:
            comm_eas4 = None
        
        ### # === get all time info & check --> having every process open in MPI mode is not optimal
        ### comm_eas4 = MPI.COMM_WORLD
        ### t = np.array([], dtype=np.float64)
        ### for fn_eas4 in fn_eas4_list:
        ###     with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
        ###         t = np.concatenate((t, hf_eas4.t))
        ###         if verbose:
        ###             print(hf_eas4.nt)
        ### comm_eas4.Barrier()
        
        # === get all time info & check
        if (self.rank==0):
            t = np.array([], dtype=np.float64)
            for fn_eas4 in fn_eas4_list:
                with eas4(fn_eas4, 'r', verbose=False) as hf_eas4:
                    t_ = np.copy(hf_eas4.t)
                t = np.concatenate((t,t_))
        else:
            t = np.array([], dtype=np.float64) ## 't' must exist on all ranks
        
        if self.usingmpi:
            self.comm.Barrier()
        
        ## broadcast concatenated time vector to all ranks
        if self.usingmpi:
            t = self.comm.bcast(t, root=0)
        
        if verbose: even_print('n EAS4 files','%i'%len(fn_eas4_list))
        if verbose: even_print('nt all files','%i'%t.size)
        
        ## check the time vector that was just created
        if (t.size>1):
            
            ## check no zero distance elements
            if (np.diff(t).size - np.count_nonzero(np.diff(t))) != 0.:
                raise AssertionError('t arr has zero-distance elements')
            else:
                if verbose: even_print('check: Δt!=0','passed')
            
            ## check monotonically increasing
            if not np.all(np.diff(t) > 0.):
                raise AssertionError('t arr not monotonically increasing')
            else:
                if verbose: even_print('check: t mono increasing','passed')
            
            ## check constant Δt
            dt0 = np.diff(t)[0]
            if not np.all(np.isclose(np.diff(t), dt0, rtol=1e-3)):
                if (self.rank==0): print(np.diff(t))
                raise AssertionError('t arr not uniformly spaced')
            else:
                if verbose: even_print('check: constant Δt','passed')
        
        # === get all grid info & check
        
        # TODO : compare coordinate arrays for series of EAS4 files
        
        # === resolution filter (skip every n timesteps)
        tfi = self.tfi = np.arange(t.size, dtype=np.int64)
        if (st!=1):
            if verbose: even_print('st', '%i'%st)
            #print('>>> st : %i'%st)
            tfi = self.tfi = tfi[::st]
        
        # === get doRead vector
        doRead = np.full((t.size,), True, dtype=bool)
        
        ## skip filter
        if hasattr(self, 'tfi'):
            doRead[np.isin(np.arange(t.size),self.tfi,invert=True)] = False
        
        ## min/max index filter
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
            doRead[:ti_min] = False
        if (ti_max is not None):
            if not isinstance(ti_max, int):
                raise TypeError('ti_max must be type int')
            doRead[ti_max:] = False
        
        if (tt_min is not None):
            if (tt_min>=0.):
                doRead[np.where((t-t.min())<tt_min)] = False
            elif (tt_min<0.):
                doRead[np.where((t-t.max())<tt_min)] = False
        
        if (tt_max is not None):
            if (tt_max>=0.):
                doRead[np.where((t-t.min())>tt_max)] = False
            elif (tt_max<0.):
                doRead[np.where((t-t.max())>tt_max)] = False
        
        # === CGD times
        self.t  = np.copy(t[doRead])
        self.nt = self.t.size
        self.ti = np.arange(self.nt, dtype=np.int64)
        
        # === write back self.t to file
        if ('dims/t' in self):
            del self['dims/t']
        self.create_dataset('dims/t', data=self.t)
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === determine CGD scalars (from EAS4 scalars)
        if not hasattr(self, 'scalars') or (len(self.scalars)==0):
            with eas4(fn_eas4_list[0], 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                self.scalars   = hf_eas4.scalars
                self.n_scalars = len(self.scalars)
                
                ## decide dtypes
                for scalar in hf_eas4.scalars:
                    
                    domainName = hf_eas4.domainName
                    ti = 0
                    dset_path = 'Data/%s/ts_%06d/par_%06d'%(domainName,ti,hf_eas4.scalar_n_map[scalar])
                    dset = hf_eas4[dset_path]
                    dtype = dset.dtype
                    
                    ## if (prec=='same'):
                    ##     self.scalars_dtypes_dict[scalar] = dtype
                    ## elif (prec=='single'):
                    ##     if (dtype!=np.float32) and (dtype!=np.float64): ## make sure its either a single or double float
                    ##         raise ValueError
                    ##     self.scalars_dtypes_dict[scalar] = np.dtype(np.float32)
                    ## else:
                    ##     raise ValueError
                    
                    self.scalars_dtypes_dict[scalar] = np.dtype(np.float32)
        
        if self.usingmpi: comm_eas4.Barrier()
        
        # === initialize datasets
        
        for scalar in self.scalars:
            
            dtype = self.scalars_dtypes_dict[scalar]
            float_bytes = dtype.itemsize
            data_gb = float_bytes*self.nt*self.nz*self.ny*self.nx / 1024**3
            shape  = (self.nt,self.nz,self.ny,self.nx)
            chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
            
            dsn = f'data/{scalar}'
            
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            
            if verbose:
                even_print(f'initializing data/{scalar}', f'{data_gb:0.2f} [GB]')
            
            dset = self.create_dataset(dsn, 
                                       shape=shape, 
                                       dtype=dtype,
                                       chunks=chunks,
                                       )
            
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            if verbose: even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]')
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size', f'{int(round(chunk_kb_)):d} [KB]')
        
        if verbose: print(72*'-')
        
        # === report size of CGD after initialization
        if verbose: tqdm.write(even_print(os.path.basename(self.fname), '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3), s=True))
        if verbose: print(72*'-')
        
        # === open EAS4s, read, write to CGD
        
        if verbose:
            progress_bar = tqdm(total=(self.nt*self.n_scalars), ncols=100, desc='import', leave=False, file=sys.stdout, smoothing=0.)
        
        data_gb_read  = 0.
        data_gb_write = 0.
        t_read  = 0.
        t_write = 0.
        
        tii  = -1 ## counter full series
        tiii = -1 ## counter CGD-local
        for fn_eas4 in fn_eas4_list:
            with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                
                #if verbose: tqdm.write( even_print( os.path.basename(fn_eas4)              , '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3), s=True))
                if verbose: tqdm.write( even_print( str(Path(fn_eas4).relative_to(Path())) , '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3), s=True))
                
                # if verbose: tqdm.write(even_print('gmode_dim1' , '%i'%hf_eas4.gmode_dim1  , s=True))
                # if verbose: tqdm.write(even_print('gmode_dim2' , '%i'%hf_eas4.gmode_dim2  , s=True))
                # if verbose: tqdm.write(even_print('gmode_dim3' , '%i'%hf_eas4.gmode_dim3  , s=True))
                
                if verbose: tqdm.write(even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1_orig, gmode_dict[hf_eas4.gmode_dim1_orig] ), s=True ))
                if verbose: tqdm.write(even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2_orig, gmode_dict[hf_eas4.gmode_dim2_orig] ), s=True ))
                if verbose: tqdm.write(even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3_orig, gmode_dict[hf_eas4.gmode_dim3_orig] ), s=True ))
                
                if verbose: tqdm.write(even_print('duration'   , '%0.2f'%hf_eas4.duration , s=True))
                
                # === write buffer
                
                # ## 5D [scalar][x,y,z,t] structured array
                # buff = np.zeros(shape=(nxr, nyr, nzr, bt), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
                
                # ===
                
                #domainName = 'DOMAIN_000000' ## only one domain supported
                domainName = hf_eas4.domainName
                
                for ti in range(hf_eas4.nt):
                    tii += 1 ## EAS4 series counter
                    if doRead[tii]:
                        tiii += 1 ## CGD counter
                        for scalar in hf_eas4.scalars:
                            if (scalar in self.scalars):
                                
                                # === collective read
                                
                                dset_path = 'Data/%s/ts_%06d/par_%06d'%(domainName,ti,hf_eas4.scalar_n_map[scalar])
                                dset = hf_eas4[dset_path]
                                
                                if hf_eas4.usingmpi: comm_eas4.Barrier()
                                t_start = timeit.default_timer()
                                if hf_eas4.usingmpi: 
                                    with dset.collective:
                                        data = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                                else:
                                    data = np.copy( dset[()] )
                                if hf_eas4.usingmpi: comm_eas4.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                data_gb       = data.nbytes / 1024**3
                                t_read       += t_delta
                                data_gb_read += data_gb
                                
                                if False:
                                    if verbose:
                                        txt = even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                        tqdm.write(txt)
                                
                                # === reduce precision (e.g. for restart which is usually double precision)
                                
                                if (data.dtype == np.float64):
                                    data = np.copy( data.astype(np.float32) )
                                
                                data_gb = data.nbytes / 1024**3
                                
                                # === collective write
                                
                                dset = self['data/%s'%scalar]
                                
                                if self.usingmpi: self.comm.Barrier()
                                t_start = timeit.default_timer()
                                if self.usingmpi:
                                    with dset.collective:
                                        dset[tiii,rz1:rz2,ry1:ry2,rx1:rx2] = data.T
                                else:
                                    
                                    if self.hasGridFilter:
                                        data = data[self.xfi[:,np.newaxis,np.newaxis],
                                                    self.yfi[np.newaxis,:,np.newaxis],
                                                    self.zfi[np.newaxis,np.newaxis,:]]
                                    
                                    dset[tiii,:,:,:] = data.T
                                
                                if self.usingmpi: self.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if False:
                                    if verbose:
                                        txt = even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                        tqdm.write(txt)
                                
                                if verbose: progress_bar.update()
                if hf_eas4.usingmpi: comm_eas4.Barrier()
        
        if verbose: progress_bar.close()
        
        if hf_eas4.usingmpi: comm_eas4.Barrier()
        if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False, read_grid=False)
        
        ## get read read/write totals all ranks
        if self.usingmpi:
            G = self.comm.gather([data_gb_read, data_gb_write, self.rank], root=0)
            G = self.comm.bcast(G, root=0)
            data_gb_read  = sum([x[0] for x in G])
            data_gb_write = sum([x[1] for x in G])
        
        if verbose: print(72*'-')
        if verbose: even_print('nt',       '%i'%self.nt )
        if verbose: even_print('dt',       '%0.6f'%self.dt )
        if verbose: even_print('duration', '%0.2f'%self.duration )
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        #if verbose: print('\n'+72*'-')
        if verbose: print('total time : cgd.import_eas4() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_csys_vecs_xy(self, fn_dat, xi=None, yi=None, **kwargs):
        '''
        add csys data (from pickled .dat, usually from tgg)
        - csys/vtang : (nx,ny,2)
        - csys/vnorm : (nx,ny,2)
        '''
        
        verbose = kwargs.get('verbose',True, **kwargs)
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        ## check xi
        if (xi is not None):
            if not isinstance(xi, np.ndarray ):
                raise ValueError('xi is not numpy array')
            if (xi.ndim!=1):
                raise ValueError('xi.ndim!=1')
            if (xi.dtype!=np.dtype(np.int32)) and (xi.dtype!=np.dtype(np.int64)):
                raise ValueError('xi.dtype not int32 or int64')
        
        ## check yi
        if (yi is not None):
            if not isinstance(yi, np.ndarray ):
                raise ValueError('yi is not numpy array')
            if (yi.ndim!=1):
                raise ValueError('yi.ndim!=1')
            if (yi.dtype!=np.dtype(np.int32)) and (yi.dtype!=np.dtype(np.int64)):
                raise ValueError('yi.dtype not int32 or int64')
        
        if verbose: print('\n'+'cgd.add_csys_vecs_xy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_dat):
            raise FileNotFoundError('file does not exist: %s'%str(fn_dat))
        
        with open(fn_dat,'rb') as f:
            dd = pickle.load(f)
        
        if ('xy2d' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain xy2d')
        #if ('vtang' not in dd.keys()):
        #    raise ValueError(f'file {fn_dat} does not contain vtang')
        #if ('vnorm' not in dd.keys()):
        #    raise ValueError(f'file {fn_dat} does not contain vnorm')
        
        ## grid from CGD
        x2d = np.copy(self['dims/x'][0,:,:]).T
        y2d = np.copy(self['dims/y'][0,:,:]).T
        
        ## check if consistent with [xy] grid in ref file
        xy2d = np.copy( dd['xy2d'] )
        
        if (xi is None) and (yi is None):
            np.testing.assert_allclose(xy2d[:,:,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,:,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is not None) and (yi is None):
            np.testing.assert_allclose(xy2d[xi,:,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[xi,:,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is None) and (yi is not None):
            np.testing.assert_allclose(xy2d[:,yi,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,yi,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is not None) and (yi is not None):
            xxi,yyi = np.ix_(xi,yi)
            np.testing.assert_allclose(xy2d[xxi,yyi,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[xxi,yyi,1], y2d, rtol=1e-14, atol=1e-14)
        else:
            raise ValueError
        
        ## key names for 'csys' group
        kn_csys = ['vtang','vnorm'] 
        for k in kn_csys:
            if (k in dd.keys()):
                dsn = f'csys/{k}'
                data = np.copy(dd[k])
                data_shape_orig = data.shape
                if (xi is not None):
                    data = np.copy(data[xi,:,:])
                if (yi is not None):
                    data = np.copy(data[:,yi,:])
                if (dsn in self):
                    del self[dsn]
                ds = self.create_dataset(dsn, data=data, chunks=None)
                if verbose: even_print(dsn,'%s --> %s'%(str(data_shape_orig),str(data.shape)))
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_csys_vecs_xy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_geom_data(self, fn_dat, xi=None, yi=None, **kwargs):
        '''
        add geom data (from pickled .dat, usually from tgg)
        - dims/stang : (nx,)
        - dims/snorm : (ny,)
        - dims/crv_R : (nx,)
        '''
        
        verbose = kwargs.get('verbose',True, **kwargs)
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        if verbose: print('\n'+'cgd.add_geom_data()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_dat):
            raise FileNotFoundError('file does not exist: %s'%str(fn_dat))
        
        ## open data file from tgg
        with open(fn_dat,'rb') as f:
            dd = pickle.load(f)
        
        if ('xy2d' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain xy2d')
        #if ('stang' not in dd.keys()):
        #    raise ValueError(f'file {fn_dat} does not contain stang')
        #if ('snorm' not in dd.keys()):
        #    raise ValueError(f'file {fn_dat} does not contain snorm')
        
        ## grid from CGD
        x2d = np.copy(self['dims/x'][0,:,:]).T
        y2d = np.copy(self['dims/y'][0,:,:]).T
        
        ## check if consistent with [xy] grid in ref file
        xy2d = np.copy( dd['xy2d'] )
        
        if (xi is None) and (yi is None):
            np.testing.assert_allclose(xy2d[:,:,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,:,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is not None) and (yi is None):
            np.testing.assert_allclose(xy2d[xi,:,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[xi,:,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is None) and (yi is not None):
            np.testing.assert_allclose(xy2d[:,yi,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,yi,1], y2d, rtol=1e-14, atol=1e-14)
        elif (xi is not None) and (yi is not None):
            xxi,yyi = np.ix_(xi,yi)
            np.testing.assert_allclose(xy2d[xxi,yyi,0], x2d, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[xxi,yyi,1], y2d, rtol=1e-14, atol=1e-14)
        else:
            raise ValueError
        
        ## key names for 'dims' group
        ## these are all 1D with shape either (nx,) or (ny,)
        kn_dims = ['stang', 'snorm', 'crv_R'] 
        for k in kn_dims:
            if (k in dd.keys()):
                dsn = f'dims/{k}'
                #data = np.copy(dd[k])
                data = dd[k]
                if not isinstance(data, np.ndarray):
                    continue
                data_shape_orig = data.shape
                if (xi is not None):
                    if (data.shape[0]==xy2d.shape[0]):
                        data = np.copy(data[xi])
                if (yi is not None):
                    if (data.shape[0]==xy2d.shape[1]):
                        data = np.copy(data[yi])
                if (dsn in self):
                    del self[dsn]
                ds = self.create_dataset(dsn, data=data, chunks=None)
                if verbose: even_print(dsn,'%s --> %s'%(str(data_shape_orig),str(data.shape)))
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_geom_data() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    @staticmethod
    def copy(fn_cgd_src, fn_cgd_tgt, **kwargs):
        '''
        copy header info, selected scalars, and [x,y,z,t] range to new CGD file
        --> this currently does NOT work in serial mode
        '''
        
        #comm    = MPI.COMM_WORLD
        rank    = MPI.COMM_WORLD.Get_rank()
        n_ranks = MPI.COMM_WORLD.Get_size()
        
        if (rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.copy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        rt       = kwargs.get('rt',1)
        force    = kwargs.get('force',False) ## overwrite or raise error if exists
        
        ti_min   = kwargs.get('ti_min',None)
        ti_max   = kwargs.get('ti_max',None)
        scalars  = kwargs.get('scalars',None)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing CGD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        xi_min = kwargs.get('xi_min',None) ## 4D coordinate 
        xi_max = kwargs.get('xi_max',None)
        yi_min = kwargs.get('yi_min',None)
        yi_max = kwargs.get('yi_max',None)
        zi_min = kwargs.get('zi_min',None)
        zi_max = kwargs.get('zi_max',None)
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        
        ct = kwargs.get('ct',1) ## 'chunks' in time
        
        xi_step = kwargs.get('xi_step',1)
        yi_step = kwargs.get('yi_step',1)
        zi_step = kwargs.get('zi_step',1)
        
        prec_coords = kwargs.get('prec_coords',None)
        if (prec_coords is None):
            prec_coords = 'same'
        elif (prec_coords=='single'):
            pass
        elif (prec_coords=='same'):
            pass
        else:
            raise ValueError('prec_coords not set correctly')
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz!=n_ranks):
            raise AssertionError('rx*ry*rz!=n_ranks')
        if not os.path.isfile(fn_cgd_src):
            raise FileNotFoundError('%s not found!'%fn_cgd_src)
        if os.path.isfile(fn_cgd_tgt) and not force:
            raise FileExistsError('%s already exists. delete it or use \'force=True\' kwarg'%fn_cgd_tgt)
        
        # ===
        
        with cgd(fn_cgd_src, 'r', comm=MPI.COMM_WORLD, driver='mpio') as hf_src:
            with cgd(fn_cgd_tgt, 'w', comm=MPI.COMM_WORLD, driver='mpio', force=force, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_tgt:
                
                ## copy over header info
                hf_tgt.init_from_cgd(fn_cgd_src, rx=rx,ry=ry,rz=rz, copy_grid=False)
                
                if (scalars is None):
                    scalars = hf_src.scalars
                
                if verbose:
                    even_print('fn_cgd_src' , fn_cgd_src )
                    even_print('nx' , '%i'%hf_src.nx )
                    even_print('ny' , '%i'%hf_src.ny )
                    even_print('nz' , '%i'%hf_src.nz )
                    even_print('nt' , '%i'%hf_src.nt )
                    if verbose: print(72*'-')
                
                if (rx>hf_src.nx):
                    raise AssertionError('rx>nx')
                if (ry>hf_src.ny):
                    raise AssertionError('ry>ny')
                if (rz>hf_src.nz):
                    raise AssertionError('rz>nz')
                if (rt>hf_src.nt):
                    raise AssertionError('rt>nt')
                
                ## for CGD, CANNOT just load full grid data for CGD on every rank, gets too big
                # x  = np.copy( hf_src.x )
                # y  = np.copy( hf_src.y )
                # z  = np.copy( hf_src.z )
                t  = np.copy( hf_src.t )
                
                # xi  = np.arange(x.shape[0],dtype=np.int64) ## arange index vector, doesnt get touched!
                # yi  = np.arange(y.shape[0],dtype=np.int64)
                # zi  = np.arange(z.shape[0],dtype=np.int64)
                ti  = np.arange(t.shape[0],dtype=np.int64)
                
                # xfi = np.arange(x.shape[0],dtype=np.int64) ## gets clipped depending on x/y/z/t_min/max opts
                # yfi = np.arange(y.shape[0],dtype=np.int64)
                # zfi = np.arange(z.shape[0],dtype=np.int64)
                tfi = np.arange(t.shape[0],dtype=np.int64)
                
                xi  = np.arange(hf_src.nx, dtype=np.int64) ## arange index vector, doesnt get touched!
                yi  = np.arange(hf_src.ny, dtype=np.int64)
                zi  = np.arange(hf_src.nz, dtype=np.int64)
                ti  = np.arange(hf_src.nt, dtype=np.int64)
                
                xfi = np.arange(hf_src.nx, dtype=np.int64) ## gets clipped (and then stepped) depending on x/y/z/t_min/max opts
                yfi = np.arange(hf_src.ny, dtype=np.int64)
                zfi = np.arange(hf_src.nz, dtype=np.int64)
                tfi = np.arange(hf_src.nt, dtype=np.int64)
                
                # === global clip (coordinate index) --> supports negative indexing!
                
                if True: ## code folding
                    
                    if (xi_min is not None):
                        xfi_ = []
                        if verbose:
                            if (xi_min<0):
                                even_print('xi_min', '%i / %i'%(xi_min,xi[xi_min]))
                            else:
                                even_print('xi_min', '%i'%(xi_min,))
                        for c in xfi:
                            if (xi_min<0) and (c>=(hf_src.nx+xi_min)):
                                xfi_.append(c)
                            elif (xi_min>=0) and (c>=xi_min):
                                xfi_.append(c)
                        xfi=np.array(xfi_, dtype=np.int64)
                    else:
                        xi_min = 0
                    
                    if (xi_max is not None):
                        xfi_ = []
                        if verbose:
                            if (xi_max<0):
                                even_print('xi_max', '%i / %i'%(xi_max,xi[xi_max]))
                            else:
                                even_print('xi_max', '%i'%(xi_max,))
                        for c in xfi:
                            if (xi_max<0) and (c<=(hf_src.nx+xi_max)):
                                xfi_.append(c)
                            elif (xi_max>=0) and (c<=xi_max):
                                xfi_.append(c)
                        xfi=np.array(xfi_, dtype=np.int64)
                    else:
                        xi_max = xi[-1]
                    
                    ## check x
                    if ((xi[xi_max]-xi[xi_min]+1)<1):
                        raise ValueError('invalid xi range requested')
                    if (rx>(xi[xi_max]-xi[xi_min]+1)):
                        raise ValueError('more ranks than grid points in x')
                    
                    if (yi_min is not None):
                        yfi_ = []
                        if verbose:
                            if (yi_min<0):
                                even_print('yi_min', '%i / %i'%(yi_min,yi[yi_min]))
                            else:
                                even_print('yi_min', '%i'%(yi_min,))
                        for c in yfi:
                            if (yi_min<0) and (c>=(hf_src.ny+yi_min)):
                                yfi_.append(c)
                            elif (yi_min>=0) and (c>=yi_min):
                                yfi_.append(c)
                        yfi=np.array(yfi_, dtype=np.int64)
                    else:
                        yi_min = 0
                    
                    if (yi_max is not None):
                        yfi_ = []
                        if verbose:
                            if (yi_max<0):
                                even_print('yi_max', '%i / %i'%(yi_max,yi[yi_max]))
                            else:
                                even_print('yi_max', '%i'%(yi_max,))
                        for c in yfi:
                            if (yi_max<0) and (c<=(hf_src.ny+yi_max)):
                                yfi_.append(c)
                            elif (yi_max>=0) and (c<=yi_max):
                                yfi_.append(c)
                        yfi=np.array(yfi_, dtype=np.int64)
                    else:
                        yi_max = yi[-1]
                    
                    ## check y
                    if ((yi[yi_max]-yi[yi_min]+1)<1):
                        raise ValueError('invalid yi range requested')
                    if (ry>(yi[yi_max]-yi[yi_min]+1)):
                        raise ValueError('more ranks than grid points in y')
                    
                    if (zi_min is not None):
                        zfi_ = []
                        if verbose:
                            if (zi_min<0):
                                even_print('zi_min', '%i / %i'%(zi_min,zi[zi_min]))
                            else:
                                even_print('zi_min', '%i'%(zi_min,))
                        for c in zfi:
                            if (zi_min<0) and (c>=(hf_src.nz+zi_min)):
                                zfi_.append(c)
                            elif (zi_min>=0) and (c>=zi_min):
                                zfi_.append(c)
                        zfi=np.array(zfi_, dtype=np.int64)
                    else:
                        zi_min = 0
                    
                    if (zi_max is not None):
                        zfi_ = []
                        if verbose:
                            if (zi_max<0):
                                even_print('zi_max', '%i / %i'%(zi_max,zi[zi_max]))
                            else:
                                even_print('zi_max', '%i'%(zi_max,))
                        for c in zfi:
                            if (zi_max<0) and (c<=(hf_src.nz+zi_max)):
                                zfi_.append(c)
                            elif (zi_max>=0) and (c<=zi_max):
                                zfi_.append(c)
                        zfi=np.array(zfi_, dtype=np.int64)
                    else:
                        zi_max = zi[-1]
                    
                    ## check z
                    if ((zi[zi_max]-zi[zi_min]+1)<1):
                        raise ValueError('invalid zi range requested')
                    if (rz>(zi[zi_max]-zi[zi_min]+1)):
                        raise ValueError('more ranks than grid points in z')
                    
                    if (ti_min is not None):
                        tfi_ = []
                        if verbose:
                            if (ti_min<0):
                                even_print('ti_min', '%i / %i'%(ti_min,ti[ti_min]))
                            else:
                                even_print('ti_min', '%i'%(ti_min,))
                        for c in tfi:
                            if (ti_min<0) and (c>=(hf_src.nt+ti_min)):
                                tfi_.append(c)
                            elif (ti_min>=0) and (c>=ti_min):
                                tfi_.append(c)
                        tfi=np.array(tfi_, dtype=np.int64)
                    else:
                        ti_min = 0
                    
                    if (ti_max is not None):
                        tfi_ = []
                        if verbose:
                            if (ti_max<0):
                                even_print('ti_max', '%i / %i'%(ti_max,ti[ti_max]))
                            else:
                                even_print('ti_max', '%i'%(ti_max,))
                        for c in tfi:
                            if (ti_max<0) and (c<=(hf_src.nt+ti_max)):
                                tfi_.append(c)
                            elif (ti_max>=0) and (c<=ti_max):
                                tfi_.append(c)
                        tfi=np.array(tfi_, dtype=np.int64)
                    else:
                        ti_max = ti[-1]
                    
                    ## check t
                    if ((ti[ti_max]-ti[ti_min]+1)<1):
                        raise ValueError('invalid ti range requested')
                    if (ct>(ti[ti_max]-ti[ti_min]+1)):
                        raise ValueError('more chunks than timesteps')
                
                # === 3D/4D communicator
                
                comm4d = hf_src.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                t4d = comm4d.Get_coords(rank)
                
                rxl_ = np.array_split(xfi,rx)
                ryl_ = np.array_split(yfi,ry)
                rzl_ = np.array_split(zfi,rz)
                #rtl_ = np.array_split(tfi,rt)
                
                rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                
                ## the rank-local bounds for READ --> takes into acct clip but not step!
                rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
                
                ## the global dim sizes for READ
                nx_read = xfi.shape[0]
                ny_read = yfi.shape[0]
                nz_read = zfi.shape[0]
                
                # === global step
                
                ## take every nth index (of the already bounds-clipped) index-to-take vector
                xfi = np.copy(xfi[::xi_step])
                yfi = np.copy(yfi[::yi_step])
                zfi = np.copy(zfi[::zi_step])
                
                ## the global dim sizes for WRITE
                nx = xfi.shape[0]
                ny = yfi.shape[0]
                nz = zfi.shape[0]
                
                # ===
                
                t = np.copy(t[tfi])
                nt = t.shape[0]
                
                # ===
                
                if verbose:
                    even_print('fn_cgd_tgt' , fn_cgd_tgt )
                    even_print('nx' , '%i'%nx )
                    even_print('ny' , '%i'%ny )
                    even_print('nz' , '%i'%nz )
                    even_print('nt' , '%i'%nt )
                    print(72*'-')
                
                if ('dims/x' in hf_tgt):
                    del hf_tgt['dims/x']
                if ('dims/y' in hf_tgt):
                    del hf_tgt['dims/y']
                if ('dims/z' in hf_tgt):
                    del hf_tgt['dims/z']
                if ('dims/t' in hf_tgt):
                    del hf_tgt['dims/t']
                
                dset = hf_src['dims/x']
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                
                if (prec_coords=='same'):
                    pass
                elif (prec_coords=='single'):
                    if (dtype!=np.float32) and (dtype!=np.float64):
                        raise ValueError
                    dtype = np.dtype(np.float32)
                    float_bytes = dtype.itemsize
                else:
                    raise ValueError
                
                shape  = (nz,ny,nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=2, itemsize=float_bytes)
                
                for dd in ['x','y','z']:
                    
                    dsn = f'dims/{dd}'
                    data_gb = float_bytes * nx * ny * nz / 1024**3
                    
                    if verbose:
                        even_print(f'initializing {dsn}','%0.1f [GB]'%(data_gb,))
                    dset = hf_tgt.create_dataset( dsn, 
                                                  shape=shape, 
                                                  dtype=dtype,
                                                  chunks=chunks )
                    
                    chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                    if verbose:
                        even_print('chunk shape (z,y,x)','%s'%str(dset.chunks))
                        even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if verbose: print(72*'-')
                
                # === clip and update [dims/<>] & [csys/<>] dsets
                
                if True:
                    
                    dsn = 'dims/stang'
                    if (dsn in hf_tgt):
                        data = np.copy(hf_tgt[dsn][()])
                        shape_orig = data.shape
                        ni = data.shape[0]
                        if (ni!=hf_src.nx):
                            print(f'cgd.copy() : something is wrong with {dsn}')
                            MPI.COMM_WORLD.Abort(1)
                        del hf_tgt[dsn]
                        data = np.copy(data[xfi])
                        hf_tgt.comm.Barrier()
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,f'{str(shape_orig)} --> {str(data.shape)}')
                    
                    dsn = 'dims/snorm'
                    if (dsn in hf_tgt):
                        data = np.copy(hf_tgt[dsn][()])
                        shape_orig = data.shape
                        ni = data.shape[0]
                        if (ni!=hf_src.ny):
                            print(f'cgd.copy() : something is wrong with {dsn}')
                            MPI.COMM_WORLD.Abort(1)
                        del hf_tgt[dsn]
                        data = np.copy(data[yfi])
                        hf_tgt.comm.Barrier()
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,f'{str(shape_orig)} --> {str(data.shape)}')
                    
                    dsn = 'csys/vtang'
                    if (dsn in hf_tgt):
                        data = np.copy(hf_tgt[dsn][()])
                        shape_orig = data.shape
                        if (data.shape!=(hf_src.nx,hf_src.ny,2)):
                            print(f'cgd.copy() : something is wrong with {dsn}')
                            MPI.COMM_WORLD.Abort(1)
                        del hf_tgt[dsn]
                        data = np.copy( data[ np.ix_(xfi,yfi,np.arange(2,dtype=np.int64)) ] )
                        hf_tgt.comm.Barrier()
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,f'{str(shape_orig)} --> {str(data.shape)}')
                    
                    dsn = 'csys/vnorm'
                    if (dsn in hf_tgt):
                        data = np.copy(hf_tgt[dsn][()])
                        shape_orig = data.shape
                        if (data.shape!=(hf_src.nx,hf_src.ny,2)):
                            print(f'cgd.copy() : something is wrong with {dsn}')
                            MPI.COMM_WORLD.Abort(1)
                        del hf_tgt[dsn]
                        data = np.copy( data[ np.ix_(xfi,yfi,np.arange(2,dtype=np.int64)) ] )
                        hf_tgt.comm.Barrier()
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,f'{str(shape_orig)} --> {str(data.shape)}')
                    
                    # ## copy over [data_dim/<>] dsets if present
                    # if ('data_dim' in hf_tgt):
                    #     for dsn in hf_tgt['data_dim'].keys():
                    #         print(f'WARNING: {dsn} has not been clipped')
                    
                    hf_tgt.flush()
                    if verbose: print(72*'-')
                    
                    hf_tgt.comm.Barrier()
                
                # === bounds for outfile WRITE
                
                xiw     = np.array( [ i for i in xfi if all([(i>=rx1),(i<rx2)]) ], dtype=np.int32 ) ## the global indices in my local rank, taking into acct clip AND step
                nxiw    = xiw.shape[0]
                xiw_off = len([ i for i in xfi if (i<rx1) ]) ## this rank's left offset in the OUTFILE context
                rx1w    = xiw_off
                rx2w    = xiw_off + nxiw
                
                yiw     = np.array( [ i for i in yfi if all([(i>=ry1),(i<ry2)]) ], dtype=np.int32 )
                nyiw    = yiw.shape[0]
                yiw_off = len([ i for i in yfi if (i<ry1) ])
                ry1w    = yiw_off
                ry2w    = yiw_off + nyiw
                
                ziw     = np.array( [ i for i in zfi if all([(i>=rz1),(i<rz2)]) ], dtype=np.int32 )
                nziw    = ziw.shape[0]
                ziw_off = len([ i for i in zfi if (i<rz1) ])
                rz1w    = ziw_off
                rz2w    = ziw_off + nziw
                
                ## xiw,yiw,ziw are used to 'filter' the rank-local data that is read in
                ## xiw,yiw,ziw are currently in the global context, so we need to subtract off the left READ bound
                ## which is NOT just the min xiw
                xiw -= rx1
                yiw -= ry1
                ziw -= rz1
                
                # ===
                
                ## time 'chunks' split (number of timesteps to read / write at a time)
                ctl_ = np.array_split(tfi,ct)
                ctl = [[b[0],b[-1]+1] for b in ctl_ ]
                
                shape  = (nt,nz,ny,nx) ## target
                hf_tgt.scalars = []
                
                ## initialize scalar datasets
                t_start = timeit.default_timer()
                for scalar in hf_src.scalars:
                    
                    dtype = hf_src.scalars_dtypes_dict[scalar]
                    float_bytes = dtype.itemsize
                    
                    chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                    data_gb = float_bytes * nx * ny * nz * nt / 1024**3
                    
                    if (scalar in scalars):
                        if verbose:
                            even_print('initializing data/%s'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        dset = hf_tgt.create_dataset(
                                                    'data/%s'%scalar,
                                                    shape=shape,
                                                    dtype=dtype,
                                                    chunks=chunks,
                                                    )
                        hf_tgt.scalars.append(scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                t_initialize = timeit.default_timer() - t_start
                if verbose:
                    even_print('time initialize',format_time_string(t_initialize))
                    print(72*'-')
                
                # ===
                
                data_gb_read  = 0.
                data_gb_write = 0.
                t_read  = 0.
                t_write = 0.
                
                # === copy over grid (requires collective for CGD)
                
                for scalar in ['x','y','z']:
                    
                    dset_src = hf_src[f'dims/{scalar}']
                    dset_tgt = hf_tgt[f'dims/{scalar}']
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    ## read
                    hf_src.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset_src.collective:
                        data = np.copy( dset_src[rz1:rz2,ry1:ry2,rx1:rx2].T )
                    hf_src.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                    data_gb = float_bytes * nx_read * ny_read * nz_read / 1024**3
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    if verbose:
                        tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    try:
                        data_out = np.copy( data[ np.ix_(xiw,yiw,ziw) ] )
                        #data_out = np.copy( data[ xiw[:,np.newaxis,np.newaxis], yiw[np.newaxis,:,np.newaxis], ziw[np.newaxis,np.newaxis,:] ] )
                    except:
                        print('cgd.copy() : error in xiw,yiw,ziw')
                        MPI.COMM_WORLD.Abort(1)
                    
                    dtype = dset_tgt.dtype
                    float_bytes = dtype.itemsize
                    
                    if ( dset_tgt.dtype != data_out.dtype):
                        data_out = np.copy( data_out.astype(dset_tgt.dtype) )
                    
                    ## write
                    hf_tgt.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset_tgt.collective:
                        dset_tgt[rz1w:rz2w,ry1w:ry2w,rx1w:rx2w] = data_out.T
                    hf_tgt.flush()
                    hf_tgt.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                    data_gb = float_bytes*nx*ny*nz / 1024**3
                    
                    t_write       += t_delta
                    data_gb_write += data_gb
                    
                    if verbose:
                        even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                # === copy over [t] (independent is fine)
                
                hf_tgt.create_dataset('dims/t', data=t, dtype=np.float64, chunks=None)
                
                # ===
                
                hf_tgt.n_scalars = len(hf_tgt.scalars)
                
                # === copy scalar datasets
                
                if verbose:
                    progress_bar = tqdm(total=len(ctl)*hf_tgt.n_scalars, ncols=100, desc='copy', leave=False, file=sys.stdout)
                
                for scalar in hf_tgt.scalars:
                    dset_src = hf_src[f'data/{scalar}']
                    dset_tgt = hf_tgt[f'data/{scalar}']
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    for ctl_ in ctl:
                        
                        ct1, ct2 = ctl_
                        
                        ct1_ = ct1 - ti[ti_min] ## coords in target file
                        ct2_ = ct2 - ti[ti_min]
                        
                        ## read
                        hf_src.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_src.collective:
                            data = dset_src[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                        hf_src.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * nx_read * ny_read * nz_read * (ct2-ct1) / 1024**3
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        if verbose:
                            tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        try:
                            data_out = np.copy( data[ np.ix_(xiw,yiw,ziw) ] )
                            #data_out = np.copy( data[ xiw[:,np.newaxis,np.newaxis], yiw[np.newaxis,:,np.newaxis], ziw[np.newaxis,np.newaxis,:] ] )
                        except:
                            print('cgd.copy() : error in xiw,yiw,ziw')
                            MPI.COMM_WORLD.Abort(1)
                        
                        ## write
                        hf_tgt.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_tgt.collective:
                            dset_tgt[ct1_:ct2_,rz1w:rz2w,ry1w:ry2w,rx1w:rx2w] = data_out.T
                        hf_tgt.flush()
                        hf_tgt.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                        data_gb = float_bytes*nx*ny*nz*(ct2-ct1) / 1024**3
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            tqdm.write(even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        if verbose:
                            progress_bar.update()
                
                if verbose:
                    progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: even_print('time initialize',format_time_string(t_initialize))
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_cgd_src, '%0.2f [GB]'%(os.path.getsize(fn_cgd_src)/1024**3))
        if verbose: even_print(fn_cgd_tgt, '%0.2f [GB]'%(os.path.getsize(fn_cgd_tgt)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.copy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def read(self,**kwargs):
        '''
        read data from file & return structured array
        '''
        
        verbose_master = kwargs.get('verbose',True)
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose:
            verbose = verbose_master
        
        if verbose: print('\n'+'cgd.read()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print(self.fname,'%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: print(72*'-')
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        rt       = kwargs.get('rt',1)
        scalars_to_read = kwargs.get('scalars',None)
        
        if (rx*ry*rz*rt!=self.n_ranks):
            raise AssertionError('rx*ry*rz*rt!=self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if (scalars_to_read is None):
            scalars_to_read = self.scalars
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d    = comm4d.Get_coords(self.rank)
            
            rxl_   = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_   = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_   = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            rtl_   = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            rxl    = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl    = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl    = [[b[0],b[-1]+1] for b in rzl_ ]
            rtl    = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            ntr = self.nt
        
        t_read = 0.
        data_gb_read = 0.
        
        data_gb = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        
        # ===
        
        names   = [ s for s in scalars_to_read if s in self.scalars ]
        formats = [ self.scalars_dtypes_dict[n] for n in names ]
        
        ## 5D [scalar][x,y,z,t] structured array
        data = np.zeros(shape=(nxr,nyr,nzr,ntr), dtype={'names':names, 'formats':formats})
        
        for scalar in names:
            
            # === collective read
            dset = self['data/%s'%scalar]
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi: 
                with dset.collective:
                    data[scalar] = dset[rt1:rt2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                data[scalar] = dset[()].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            
            if verbose:
                even_print( 'read: %s'%scalar , '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)) )
            
            t_read       += t_delta
            data_gb_read += data_gb
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('cgd.read()', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: print(72*'-')
        
        return data
    
    # === test data populators
    
    def make_test_file(self, **kwargs):
        '''
        make a test CGD file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.make_test_file()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        self.nx = nx = kwargs.get('nx',128)
        self.ny = ny = kwargs.get('ny',128)
        self.nz = nz = kwargs.get('nz',128)
        self.nt = nt = kwargs.get('nt',128)
        
        data_gb = 3 * 4*nx*ny*nz*nt / 1024.**3
        #data_gb = 1 * 4*nx*ny*nz*nt / 1024.**3
        
        if verbose: even_print(self.fname, '%0.2f [GB]'%(data_gb,))
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d    = comm4d.Get_coords(self.rank)
            
            rxl_   = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_   = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_   = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            rtl_   = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl    = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl    = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl    = [[b[0],b[-1]+1] for b in rzl_ ]
            rtl    = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]
            ry1, ry2 = ryl[t4d[1]]
            rz1, rz2 = rzl[t4d[2]]
            rt1, rt2 = rtl[t4d[3]]
            
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
            ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            ntr = self.nt
        
        # ===
        
        ## 1D coordinates
        x = np.linspace(0., 2*np.pi, nx, dtype=np.float64)
        y = np.linspace(0., 2*np.pi, ny, dtype=np.float64)
        z = np.linspace(0., 2*np.pi, nz, dtype=np.float64)
        t = 0.1 * np.arange(nt, dtype=np.float64)
        
        ## 3D coordinates
        x,y,z = np.meshgrid(x,y,z, indexing='ij')
        
        ## per-rank dim range
        if self.usingmpi:
            xr = np.copy( x[rx1:rx2,ry1:ry2,rz1:rz2] )
            yr = np.copy( y[rx1:rx2,ry1:ry2,rz1:rz2] )
            zr = np.copy( z[rx1:rx2,ry1:ry2,rz1:rz2] )
            #tr = t[rt1:rt2]
            tr = np.copy(t)
        else:
            xr = np.copy(x)
            yr = np.copy(y)
            zr = np.copy(z)
            tr = np.copy(t)
        
        # === apply deformation to mesh
        
        x_ = np.copy(x)
        y_ = np.copy(y)
        z_ = np.copy(z)
        
        if False: ## distort grid in [x,y] (2D)
            
            ## xy
            x += 0.2*np.sin(1*y_)
            #x += 0.2*np.sin(1*z_)
            
            ## yz
            #y += 0.2*np.sin(1*z_)
            y += 0.2*np.sin(1*x_)
            
            ## zy
            #z += 0.2*np.sin(1*x_)
            #z += 0.2*np.sin(1*y_)
        
        if True: ## distort grid in [x,y,z] (3D)
            
            ## xy
            x += 0.2*np.sin(1*y_)
            x += 0.2*np.sin(1*z_)
            
            ## yz
            y += 0.2*np.sin(1*z_)
            y += 0.2*np.sin(1*x_)
            
            ## zy
            z += 0.2*np.sin(1*x_)
            z += 0.2*np.sin(1*y_)
        
        x_ = None; del x_
        y_ = None; del y_
        z_ = None; del z_
        
        # self.x = x
        # self.y = y
        # self.z = z
        
        ## chunk info for coord arrays
        shape  = (nz,ny,nx)
        chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=2, itemsize=8)
        
        ## initialize coordinate datasets
        data_gb = 4*nx*ny*nz / 1024.**3
        for scalar in ['x','y','z']:
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            if verbose:
                even_print('initializing dims/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            dset = self.create_dataset('dims/%s'%scalar, 
                                        shape=shape,
                                        dtype=np.float64,
                                        chunks=chunks )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === write coordinate datasets (collective)
        
        data_gb = 8*nx*ny*nz / 1024.**3
        
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['dims/x']
        if self.usingmpi:
            with ds.collective:
                #ds[rz1:rz2,ry1:ry2,rx1:rx2] = x[rx1:rx2,ry1:ry2,rz1:rz2].T
                ds[rz1:rz2,ry1:ry2,rx1:rx2] = xr.T
        else:
            ds[:,:,:] = x.T
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: dims/x','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['dims/y']
        if self.usingmpi:
            with ds.collective:
                #ds[rz1:rz2,ry1:ry2,rx1:rx2] = y[rx1:rx2,ry1:ry2,rz1:rz2].T
                ds[rz1:rz2,ry1:ry2,rx1:rx2] = yr.T
        else:
            ds[:,:,:] = y.T
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: dims/y','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['dims/z']
        if self.usingmpi:
            with ds.collective:
                #ds[rz1:rz2,ry1:ry2,rx1:rx2] = z[rx1:rx2,ry1:ry2,rz1:rz2].T
                ds[rz1:rz2,ry1:ry2,rx1:rx2] = zr.T
        else:
            ds[:,:,:] = z.T
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: dims/z','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        if verbose: print(72*'-')
        
        # ===
        
        ## chunk info for scalar arrays
        shape  = (self.nt,self.nz,self.ny,self.nx)
        float_bytes = 4
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        self.scalars = ['u','v','w']
        #self.scalars = ['u']
        self.scalars_dtypes = [ np.float32 for s in self.scalars ]
        
        ## initialize datasets
        data_gb = 4*nx*ny*nz*nt / 1024.**3
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        for scalar in self.scalars:
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            if verbose:
                even_print('initializing data/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            dset = self.create_dataset('data/%s'%scalar, 
                                        shape=shape,
                                        dtype=np.float32,
                                        chunks=chunks )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        if self.usingmpi: self.comm.Barrier()
        t_initialize = timeit.default_timer() - t_start
        
        if verbose: print(72*'-')
        
        # ===
        
        ## 5D [scalar][x,y,z,t] structured array --> data buffer
        data = np.zeros(shape=(nxr,nyr,nzr,ntr), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
        
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        
        if False: ## ABC Flow
            
            A = np.sqrt(3)
            B = np.sqrt(2)
            C = 1.
            na = np.newaxis
            data['u'] = (A + 0.5 * tr[na,na,na,:] * np.sin(np.pi*tr[na,na,na,:])) * np.sin(zr[:,:,:,na]) + \
                        B * np.cos(yr[:,:,:,na]) + \
                        0.*xr[:,:,:,na]
            data['v'] = B * np.sin(xr[:,:,:,na]) + \
                        C * np.cos(zr[:,:,:,na]) + \
                        0.*yr[:,:,:,na] + \
                        0.*tr[na,na,na,:]
            data['w'] = C * np.sin(yr[:,:,:,na]) + \
                        (A + 0.5 * tr[na,na,na,:] * np.sin(np.pi*tr[na,na,na,:])) * np.cos(xr[:,:,:,na]) + \
                        0.*zr[:,:,:,na]
        
        if True: ## random data
            
            rng = np.random.default_rng(seed=self.rank)
            for scalar in self.scalars:
                data[scalar] = rng.uniform(-1, +1, size=(nxr,nyr,nzr,ntr)).astype(np.float32)
        
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('gen data','%0.3f [s]'%(t_delta,))
        
        ## write data
        data_gb_write = 0.
        t_write = 0.
        for scalar in self.scalars:
            ds = self['data/%s'%scalar]
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with ds.collective:
                    ds[rt1:rt2,rz1:rz2,ry1:ry2,rx1:rx2] = data[scalar].T
            else:
                ds[:,:,:,:] = data[scalar].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4*nx*ny*nz*nt / 1024**3
            
            t_write       += t_delta
            data_gb_write += data_gb
            
            if verbose:
                even_print('write: %s'%(scalar,), '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('time initialize',format_time_string(t_initialize))
        if verbose: even_print('write total', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: print(72*'-')
        if verbose: print('total time :  cgd.make_test_file() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing
    
    def get_mean_legacy(self, **kwargs):
        '''
        get mean in [t] --> leaves [x,y,z,1]
        --> save to new CGD file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.get_mean()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        #rt = 1
        
        fn_cgd_mean  = kwargs.get('fn_cgd_mean',None)
        #sfm         = kwargs.get('scalars',None) ## scalars to take (for mean)
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === mean file name (for writing)
        if (fn_cgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_cgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_cgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_cgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if verbose: even_print('fn_cgd'       , self.fname   )
        if verbose: even_print('fn_cgd_mean'  , fn_cgd_mean  )
        #if verbose: even_print('fn_cgd_prime' , fn_cgd_prime )
        if verbose: even_print('do Favre avg' , str(favre)   )
        if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        t_avg_start  = self.t[ti_for_avg[0]]
        t_avg_end    = self.t[ti_for_avg[-1]]
        duration_avg = t_avg_end - t_avg_start
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        if verbose: even_print('Δt','%0.2f [-]'%(dt0,))
        if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        itemsize = dtype.itemsize
        
        #data_gb      = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        data_gb      = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1      / 1024**3
        
        scalars_re = [ 'u','v','w','p','T','rho' ] #, 'utang','unorm', 'vort_x','vort_y','vort_z','vort_tang' ]
        scalars_fv = [ 'u','v','w','p','T','rho' ]
        
        #with cgd(fn_cgd_mean, 'w', force=force, driver='mpio', comm=MPI.COMM_WORLD) as hf_mean:
        with cgd(fn_cgd_mean, 'w', force=force, driver=self.driver, comm=self.comm) as hf_mean:
            
            ## initialize the mean file from the opened unsteady cgd file
            hf_mean.init_from_cgd(self.fname, rx=rx,ry=ry,rz=rz, chunk_kb=chunk_kb)
            
            ## set some top-level attributes
            hf_mean.attrs['duration_avg'] = duration_avg ## duration of mean
            #hf_mean.attrs['duration_avg'] = self.duration
            hf_mean.attrs['dt'] = dt0
            #hf_mean.attrs['fclass'] = 'cgd'
            hf_mean.attrs['fsubtype'] = 'mean'
            
            if verbose: print(72*'-')
            
            # === initialize mean datasets
            for scalar in self.scalars:
                
                shape  = (1,self.nz,self.ny,self.nx)
                #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        if ('data/%s'%scalar in hf_mean):
                            del hf_mean['data/%s'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    
                    if (scalar in scalars_fv):
                        if ('data/%s_fv'%scalar in hf_mean):
                            del hf_mean['data/%s_fv'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}_fv' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}_fv',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s_fv'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            # === read rho
            if favre:
                
                dset = self['data/rho']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi: 
                    with dset.collective:
                        #rho = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                        rho = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    #rho = dset[()].T
                    rho = dset[ti_min:,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                
                t_delta = timeit.default_timer() - t_start
                if (self.rank==0):
                    txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                    tqdm.write(txt)
                
                t_read       += t_delta
                data_gb_read += data_gb
                
                ## mean ρ in [t] --> leave [x,y,z]
                rho_mean = np.mean(rho, axis=-1, keepdims=True, dtype=np.float64).astype(np.float32)
            
            # === read, do mean, write
            for scalar in self.scalars:
                
                # === collective read
                dset = self['data/%s'%scalar]
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        #data = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                        data = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    #data = dset[()].T
                    data = dset[ti_min:,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                if (self.rank==0):
                    txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                    tqdm.write(txt)
                
                t_read       += t_delta
                data_gb_read += data_gb
                
                # === do mean in [t]
                if reynolds:
                    data_mean    = np.mean(data,     axis=-1, keepdims=True, dtype=np.float64).astype(np.float32)
                if favre:
                    data_mean_fv = np.mean(data*rho, axis=-1, keepdims=True, dtype=np.float64).astype(np.float32) / rho_mean
                
                # === write
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        dset = hf_mean['data/%s'%scalar]
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_mean.T
                        else:
                            dset[:,:,:,:] = data_mean.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if (self.rank==0):
                            txt = even_print('write: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
                
                if favre:
                    if scalar in scalars_fv:
                        
                        dset = hf_mean['data/%s_fv'%scalar]
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_mean_fv.T
                        else:
                            dset[:,:,:,:] = data_mean_fv.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if (self.rank==0):
                            txt = even_print('write: %s_fv'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
            
            if self.usingmpi: self.comm.Barrier()
            
            # === replace dims/t array --> take last time of series
            t = np.array([self.t[-1]],dtype=np.float64)
            if ('dims/t' in hf_mean):
                del hf_mean['dims/t']
            hf_mean.create_dataset('dims/t', data=t)
            
            if hasattr(hf_mean, 'duration_avg'):
                if verbose: even_print('duration avg', '%0.2f [-]'%hf_mean.duration_avg)
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_cgd_mean, '%0.2f [GB]'%(os.path.getsize(fn_cgd_mean)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.get_mean() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def get_mean(self, **kwargs):
        '''
        get mean in [t] --> leaves [x,y,z,1]
        --> save to new CGD file
        -----
        - this version uses accumulator buffers and does *(1/n) at end to calculate mean
        - this allows for low RAM usage, as the time dim can be sub-chunked (ct=N)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.get_mean()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_cgd_mean  = kwargs.get('fn_cgd_mean',None)
        #sfm         = kwargs.get('scalars',None) ## scalars to take (for mean)
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        
        ct           = kwargs.get('ct',1)
        
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing mean file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === mean file name (for writing)
        if (fn_cgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_cgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_cgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_cgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if verbose: even_print('fn_cgd'       , self.fname   )
        if verbose: even_print('fn_cgd_mean'  , fn_cgd_mean  )
        #if verbose: even_print('fn_cgd_prime' , fn_cgd_prime )
        if verbose: even_print('do Favre avg' , str(favre)   )
        if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        t_avg_start  = self.t[ti_for_avg[0]]
        t_avg_end    = self.t[ti_for_avg[-1]]
        duration_avg = t_avg_end - t_avg_start
        
        #if not isinstance(ct, (int,np.int32,np.int64)):
        if not isinstance(ct, int):
            raise ValueError
        if (ct<1):
            raise ValueError
        
        ## [t] sub chunk range
        ctl_ = np.array_split( ti_for_avg, min(ct,nt_avg) )
        ctl = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## check that no sub ranges are <=1
        for a_ in [ ctl_[1]-ctl_[0] for ctl_ in ctl ]:
            if (a_ <= 1):
                raise ValueError('at least one rank sub range has zero length')
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        if verbose: even_print('Δt','%0.2f [-]'%(dt0,))
        #if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        # dtype = np.dtype(np.float32)
        # float_bytes = dtype.itemsize
        # itemsize = dtype.itemsize
        
        ##data_gb      = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        #data_gb      = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
        #data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1      / 1024**3
        
        scalars_re = [ 'u','v','w','p','T','rho' ] # 'vort_x','vort_y','vort_z','vort_tang' ]
        scalars_fv = ['u','v','w','T', 'utang','unorm' ] ## 'p','rho'
        
        ## do a loop through to get names
        scalars_mean_names  = []
        scalars_mean_dtypes = []
        for scalar in self.scalars:
            dtype = self.scalars_dtypes_dict[scalar]
            if reynolds:
                if True: ## always
                    sc_name = scalar
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
            if favre:
                if (scalar in scalars_fv):
                    sc_name = f'r_{scalar}'
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
        
        #with cgd(fn_cgd_mean, 'w', force=force, driver='mpio', comm=MPI.COMM_WORLD) as hf_mean:
        with cgd(fn_cgd_mean, 'w', force=force, driver=self.driver, comm=self.comm, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_mean:
            
            ## initialize the mean file from the opened unsteady cgd file
            hf_mean.init_from_cgd(self.fname, rx=rx,ry=ry,rz=rz, chunk_kb=chunk_kb)
            
            ## set some top-level attributes
            hf_mean.attrs['duration_avg'] = duration_avg ## duration of mean
            #hf_mean.attrs['duration_avg'] = self.duration
            hf_mean.attrs['dt'] = dt0
            #hf_mean.attrs['fclass'] = 'cgd'
            hf_mean.attrs['fsubtype'] = 'mean'
            
            if verbose: print(72*'-')
            
            # === initialize mean datasets
            for scalar in self.scalars:
                
                dtype = self.scalars_dtypes_dict[scalar]
                float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                
                data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                
                shape  = (1,self.nz,self.ny,self.nx)
                #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        if ('data/%s'%scalar in hf_mean):
                            del hf_mean['data/%s'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    
                    if (scalar in scalars_fv):
                        if ('data/%s_fv'%scalar in hf_mean):
                            del hf_mean['data/%s_fv'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}_fv' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}_fv',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s_fv'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            ## accumulator array for local rank --> initialize
            data_sum = np.zeros(shape=(nxr,nyr,nzr,1), dtype={'names':scalars_mean_names, 'formats':[np.float64 for _ in scalars_mean_names]})
            
            # === main loop
            
            if verbose:
                progress_bar = tqdm(total=ct, ncols=100, desc='mean', leave=False, file=sys.stdout)
            
            ct_counter=0
            for ctl_ in ctl:
                ct_counter += 1
                ct1, ct2 = ctl_
                ntc = ct2 - ct1
                
                if (ct>1):
                    if verbose:
                        mesg = f'[t] sub chunk {ct_counter:d}/{ct:d}'
                        tqdm.write( mesg )
                        tqdm.write( '-'*len(mesg) )
                
                # === read rho
                if favre:
                    
                    dset = self['data/rho']
                    
                    dtype = self.scalars_dtypes_dict['rho']
                    if (dtype!=dset.dtype):
                        raise ValueError
                    float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if self.usingmpi: 
                        with dset.collective:
                            ##rho = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            #rho = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            rho = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                    else:
                        #rho = dset[()].T
                        #rho = dset[ti_min:,:,:,:].T
                        rho = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                    data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                    
                    if verbose:
                        txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    # ## mean ρ in [t] --> leave [x,y,z]
                    # rho_mean = np.mean(rho, axis=-1, keepdims=True, dtype=np.float64).astype(np.float32)
                
                # === read data, sum
                for scalar in self.scalars:
                    
                    dset = self[f'data/{scalar}']
                    dtype = self.scalars_dtypes_dict[scalar]
                    if (dtype!=dset.dtype):
                        raise ValueError
                    float_bytes = dtype.itemsize
                    
                    # === collective read
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if self.usingmpi:
                        with dset.collective:
                            ##data = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            #data = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            data = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                    else:
                        ##data = dset[()].T
                        #data = dset[ti_min:,:,:,:].T
                        data = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                    data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                    
                    if verbose:
                        txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    # === do sum, add to accumulator
                    if reynolds:
                        sc_name = scalar
                        data_sum[sc_name] += np.sum(data, axis=-1, dtype=np.float64, keepdims=True)
                    if favre:
                        if (scalar in scalars_fv):
                            sc_name = f'r_{scalar}'
                            data_sum[sc_name] += np.sum(data*rho, axis=-1, dtype=np.float64, keepdims=True)
                    
                    if self.usingmpi: self.comm.Barrier()
                
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                
                if verbose: progress_bar.update()
                if verbose: tqdm.write(72*'-')
            if verbose: progress_bar.close()
            
            # ==========================================================
            # multiply accumulators by (1/n)
            # ==========================================================
            
            for scalar in self.scalars:
                if reynolds:
                    sc_name = scalar
                    data_sum[sc_name] *= (1/nt_avg)
                if favre:
                    if (scalar in scalars_fv):
                        sc_name = f'r_{scalar}'
                        data_sum[sc_name] *= (1/nt_avg)
            
            # ==========================================================
            # 'data_sum' now contains averages, not sums!
            # ==========================================================
            
            ## Favre avg : φ_tilde = avg[ρ·φ]/avg[ρ]
            rho_mean = np.copy( data_sum['rho'] )
            
            # === write
            for scalar in self.scalars:
                
                if reynolds:
                    
                    dset = hf_mean[f'data/{scalar}']
                    dtype = dset.dtype
                    float_bytes = dtype.itemsize
                    
                    if (dtype==np.float32):
                        data_out = np.copy( data_sum[scalar].astype(np.float32) )
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                    else:
                        dset[:,:,:,:] = data_out.T
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                    
                    if verbose:
                        txt = even_print(f'write: {scalar}', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_write       += t_delta
                    data_gb_write += data_gb_mean
                
                if favre:
                    if (scalar in scalars_fv):
                        
                        dset = hf_mean[f'data/{scalar}_fv']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        
                        ## φ_tilde = avg[ρ·φ]/avg[ρ]
                        data_out = np.copy( data_sum[f'r_{scalar}'] / rho_mean )
                        
                        if (dtype==np.float32):
                            data_out = np.copy( data_out.astype(np.float32) )
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                        else:
                            dset[:,:,:,:] = data_out.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                        
                        if verbose:
                            txt = even_print(f'write: {scalar}_fv', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
            
            # === replace dims/t array --> take last time of series
            t = np.array([self.t[-1]],dtype=np.float64)
            if ('dims/t' in hf_mean):
                del hf_mean['dims/t']
            hf_mean.create_dataset('dims/t', data=t)
            
            if hasattr(hf_mean, 'duration_avg'):
                if verbose: even_print('duration avg', '%0.2f [-]'%hf_mean.duration_avg)
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_cgd_mean, '%0.2f [GB]'%(os.path.getsize(fn_cgd_mean)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.get_mean() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def add_mean_dimensional_data_xpln(self, **kwargs):
        '''
        get dimensionalized mean data for [x]/[s1] plane
        --> save to existing CGD file with fsubtype=mean
        - assumes volume which is thin in [x]/[s1] direction
        - a CGD which is the output of cgd.get_mean() should be opened here
        - not parallel
        '''
        
        verbose = kwargs.get('verbose',True)
        epsilon = kwargs.get('epsilon',5e-5)
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.add_mean_dimensional_data_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## this func is not parallel
        if self.usingmpi:
            raise NotImplementedError('cgd.add_mean_dimensional_data_xpln() is not a parallel function')
        
        ## 'r' and 'w' open modes are not allowed
        if not (self.open_mode=='a') or (self.open_mode=='r+'):
            raise ValueError(f'open mode is {self.open_mode}')
        
        ## assert that this is a mean flow file ( i.e. output from cgd.get_mean() )
        if (self.fsubtype!='mean'):
            print(self.fsubtype)
            raise ValueError
        
        ## assert that 'data/utang' and 'data/unorm' are present
        if not ('data/utang' in self):
            raise ValueError
        if not ('data/unorm' in self):
            raise ValueError
        
        ## assert that 'dims/stang' and 'data/snorm' are present
        if not ('dims/stang' in self):
            raise ValueError
        if not ('dims/snorm' in self):
            raise ValueError
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## read in 3D coordinate arrays, then dimensionalize [m]
        x = np.copy( self['dims/x'][()].T * self.lchar )
        y = np.copy( self['dims/y'][()].T * self.lchar )
        z = np.copy( self['dims/z'][()].T * self.lchar )
        nx = self.nx
        ny = self.ny
        nz = self.nz
        
        ## read in 1D coordinate arrays, then dimensinoalize [m]
        stang_ = np.copy(self['dims/stang'])
        stang  = np.copy( stang_ * self.lchar ) ## dimensional [m]
        
        snorm_ = np.copy(self['dims/snorm'])
        snorm  = np.copy( snorm_ * self.lchar ) ## dimensional [m]
        
        ## assert [z] is same over all [x,y]
        if (z.ndim!=3):
            raise ValueError
        z1d = np.copy(z[0,0,:])
        for i in range(nx):
            for j in range(ny):
                np.testing.assert_allclose(z1d, z[i,j,:], rtol=1e-14, atol=1e-14)
        
        ## assert [x,y] is same over all [z]
        x2d  = np.copy(x[:,:,0])
        y2d  = np.copy(y[:,:,0])
        xy2d = np.stack((x2d,y2d), axis=-1)
        for k in range(nz):
            x2d_  = np.copy(x[:,:,k])
            y2d_  = np.copy(y[:,:,k])
            xy2d_ = np.stack((x2d_,y2d_), axis=-1)
            np.testing.assert_allclose(xy2d, xy2d_, rtol=1e-14, atol=1e-14)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## check if mean cgd has attr 'dt'
        if ('dt' in self.attrs.keys()):
            dt = self.attrs['dt']
            if (dt is not None):
                dt *= self.tchar
        else:
            raise ValueError
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration_avg,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration_avg*self.tchar,))
        if verbose: print(72*'-')
        
        ## re-dimensionalize
        u   =  self.U_inf                     * np.copy( self['data/u'][()].T   )
        v   =  self.U_inf                     * np.copy( self['data/v'][()].T   )
        w   =  self.U_inf                     * np.copy( self['data/w'][()].T   )
        rho =  self.rho_inf                   * np.copy( self['data/rho'][()].T )
        p   =  (self.rho_inf * self.U_inf**2) * np.copy( self['data/p'][()].T   )
        T   =  self.T_inf                     * np.copy( self['data/T'][()].T   )
        
        utang = self.U_inf * np.copy( self['data/utang'][()].T )
        unorm = self.U_inf * np.copy( self['data/unorm'][()].T )
        
        # mu1 = np.copy( self.C_Suth * T**(3/2) / (T + self.S_Suth) )
        # mu2 = np.copy( self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth)) )
        # np.testing.assert_allclose(mu1, mu2, rtol=2e-7, atol=2e-7)
        # mu = np.copy(mu2)
        
        mu = np.copy( self.C_Suth * T**(3/2) / (T + self.S_Suth) )
        nu = np.copy( mu / rho )
        
        # === average in [z]/[s3] --> leave 2D [x,y]/[s1,s2]
        
        u     = np.squeeze( np.mean( u     , axis=2, dtype=np.float64).astype(np.float32) )
        v     = np.squeeze( np.mean( v     , axis=2, dtype=np.float64).astype(np.float32) )
        w     = np.squeeze( np.mean( w     , axis=2, dtype=np.float64).astype(np.float32) )
        rho   = np.squeeze( np.mean( rho   , axis=2, dtype=np.float64).astype(np.float32) )
        p     = np.squeeze( np.mean( p     , axis=2, dtype=np.float64).astype(np.float32) )
        T     = np.squeeze( np.mean( T     , axis=2, dtype=np.float64).astype(np.float32) )
        utang = np.squeeze( np.mean( utang , axis=2, dtype=np.float64).astype(np.float32) )
        unorm = np.squeeze( np.mean( unorm , axis=2, dtype=np.float64).astype(np.float32) )
        mu    = np.squeeze( np.mean( mu    , axis=2, dtype=np.float64).astype(np.float32) )
        nu    = np.squeeze( np.mean( nu    , axis=2, dtype=np.float64).astype(np.float32) )
        
        # === get 2D metric tensor
        
        if (nx<3):
            raise ValueError('dds1[] not possible because nx<3')
        elif (nx>=3) and (nx<5):
            acc = 2
        elif (nx>=5) and (nx<7):
            acc = 4
        elif (nx>=7):
            acc = 6
        else:
            raise ValueError('this should never happen')
        
        edge_stencil = 'half'
        if verbose: even_print('acc','%i'%acc)
        if verbose: even_print('edge_stencil',edge_stencil)
        
        M = get_metric_tensor_2d(x2d, y2d, acc=acc, edge_stencil=edge_stencil, no_warn=True, verbose=verbose)
        
        ddx_q1 = np.copy( M[:,:,0,0] ) ## ξ_x
        ddx_q2 = np.copy( M[:,:,1,0] ) ## η_x
        ddy_q1 = np.copy( M[:,:,0,1] ) ## ξ_y
        ddy_q2 = np.copy( M[:,:,1,1] ) ## η_y
        
        if verbose: even_print('ξ_x','%s'%str(ddx_q1.shape))
        if verbose: even_print('η_x','%s'%str(ddx_q2.shape))
        if verbose: even_print('ξ_y','%s'%str(ddy_q1.shape))
        if verbose: even_print('η_y','%s'%str(ddy_q2.shape))
        
        M = None; del M
        
        ## the 'computational' grid (unit Cartesian)
        #x_comp = np.arange(nx, dtype=np.float64)
        #y_comp = np.arange(ny, dtype=np.float64)
        x_comp = 1.
        y_comp = 1.
        
        # === get gradients & vort_z
        
        ddx_u_comp = gradient(u, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddy_u_comp = gradient(u, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddx_u      = ddx_u_comp*ddx_q1 + ddy_u_comp*ddx_q2
        ddy_u      = ddx_u_comp*ddy_q1 + ddy_u_comp*ddy_q2
        
        ddx_v_comp = gradient(v, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddy_v_comp = gradient(v, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
        ddx_v      = ddx_v_comp*ddx_q1 + ddy_v_comp*ddx_q2
        ddy_v      = ddx_v_comp*ddy_q1 + ddy_v_comp*ddy_q2
        
        ## dimensional [1/s]
        vort_z = ddx_v - ddy_u
        
        # ===
        
        s1 = np.copy( stang )
        s2 = np.copy( snorm )
        
        ## [s2] gradients --> yields 1D in [y]/[s2]
        dds2_u     = gradient(u     , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_v     = gradient(v     , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_T     = gradient(T     , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_p     = gradient(p     , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_rho   = gradient(rho   , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_utang = gradient(utang , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        dds2_unorm = gradient(unorm , s2, axis=1, acc=acc, edge_stencil=edge_stencil)
        
        ## wall quantities --> mean over [x]/[s1] --> leave 0D float
        dds2_u1_wall = float( np.mean( dds2_utang[:,0] ) )
        dds2_T       = float( np.mean( dds2_T[:,0]     ) )
        rho_wall     = float( np.mean( rho[:,0]        ) )
        nu_wall      = float( np.mean( nu[:,0]         ) )
        mu_wall      = float( np.mean( mu[:,0]         ) )
        T_wall       = float( np.mean( T[:,0]          ) )
        tau_wall     = mu_wall * dds2_u1_wall
        q_wall       = self.cp * mu_wall / self.Pr * dds2_T ### wall heat flux
        
        ## friction velocity
        u_tau  = np.sqrt(tau_wall/rho_wall)
        #y_plus = np.copy( s2 * u_tau / nu_wall )
        
        # === populate 1D & 2D arrays using calc_profile_edge_1d(), calc_d99_1d()
        
        psvel = np.zeros((nx,ny), dtype=np.float64)
        
        psvel_edge = np.zeros((nx,), dtype=np.float64)
        u1_edge    = np.zeros((nx,), dtype=np.float64)
        rho_edge   = np.zeros((nx,), dtype=np.float64)
        mu_edge    = np.zeros((nx,), dtype=np.float64)
        nu_edge    = np.zeros((nx,), dtype=np.float64)
        T_edge     = np.zeros((nx,), dtype=np.float64)
        
        s2_edge = np.zeros((nx,), dtype=np.float64)
        d99     = np.zeros((nx,), dtype=np.float64)
        
        u1_99  = np.zeros((nx,), dtype=np.float64)
        
        for i in range(nx):
            
            vort_z_ = np.copy( vort_z[i,:] )
            psvel_  = sp.integrate.cumulative_trapezoid(-1*vort_z_, s2, initial=0.)
            psvel[i,:] = psvel_
            
            ## get edge
            s2_edge_ = calc_profile_edge_1d( y=s2, u=psvel_, lchar=self.lchar, acc=acc, edge_stencil=edge_stencil, epsilon=epsilon )
            #aa = ( s2_edge_ - s2.min() ) / ( s2.max() - s2.min() )
            #tqdm.write(f'{aa:0.9f}')
            s2_edge[i] = s2_edge_
            
            ## interpolate at edge
            psvel_edge_ = sp.interpolate.interp1d(s2, psvel_ , kind='cubic', bounds_error=True)(s2_edge_)
            psvel_edge_ = float(psvel_edge_)
            psvel_edge[i] = psvel_edge_
            
            u1_edge[i]  = sp.interpolate.interp1d(s2, utang[i,:] , kind='cubic', bounds_error=True)(s2_edge_)
            rho_edge[i] = sp.interpolate.interp1d(s2, rho[i,:]   , kind='cubic', bounds_error=True)(s2_edge_)
            mu_edge[i]  = sp.interpolate.interp1d(s2, mu[i,:]    , kind='cubic', bounds_error=True)(s2_edge_)
            nu_edge[i]  = sp.interpolate.interp1d(s2, nu[i,:]    , kind='cubic', bounds_error=True)(s2_edge_)
            T_edge[i]   = sp.interpolate.interp1d(s2, T[i,:]     , kind='cubic', bounds_error=True)(s2_edge_)
            
            ## get δ99
            d99_ = calc_d99_1d( y=s2, u=psvel_, y_edge=s2_edge_, u_edge=psvel_edge_ )
            d99_ = float(d99_)
            d99[i] = d99_
            
            u1_99[i] = sp.interpolate.interp1d(s2, utang[i,:], kind='cubic', bounds_error=True)(d99_)
        
        # === avg in [x]/[s1] --> leave [y]/[s2]
        
        psvel = np.mean( psvel , axis=0 )
        utang = np.mean( utang , axis=0 )
        rho   = np.mean( rho   , axis=0 )
        
        psvel_edge  = np.mean( psvel_edge , axis=0 )
        u1_edge     = np.mean( u1_edge    , axis=0 )
        rho_edge    = np.mean( rho_edge   , axis=0 )
        mu_edge     = np.mean( mu_edge    , axis=0 )
        nu_edge     = np.mean( nu_edge    , axis=0 )
        T_edge      = np.mean( T_edge     , axis=0 )
        
        s2_edge     = np.mean( s2_edge  , axis=0 )
        d99         = np.mean( d99      , axis=0 )
        
        u1_99       = np.mean( u1_99 , axis=0 )
        
        sc_l_out = d99
        sc_u_out = u1_99
        sc_t_out = d99/u1_99
        np.testing.assert_allclose(sc_t_out, sc_l_out/sc_u_out, rtol=1e-14, atol=1e-14)
        
        sc_u_in = u_tau
        sc_l_in = nu_wall / u_tau
        sc_t_in = nu_wall / u_tau**2
        np.testing.assert_allclose(sc_t_in, sc_l_in/sc_u_in, rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration_avg * (self.lchar / self.U_inf)
        t_eddy = t_meas / (d99/u_tau)
        
        # === get 0D BL integral quantities
        
        dd = calc_bl_integral_quantities_1d( y=s2,
                                             u=utang,
                                             rho=rho,
                                             u_tau=u_tau,
                                             d99=d99,
                                             y_edge=s2_edge,
                                             rho_edge=rho_edge,
                                             nu_edge=nu_edge,
                                             u_edge=u1_edge,
                                             nu_wall=nu_wall,
                                            )
        
        # === add to file
        
        gn = 'data_dim'
        
        ## if group already exists in file, delete it entirely
        if (gn in self):
            del self[gn]
        
        ## 1D
        self.create_dataset(f'{gn}/psvel' , data=psvel, chunks=None)
        self.create_dataset(f'{gn}/utang' , data=utang, chunks=None)
        self.create_dataset(f'{gn}/rho'   , data=rho,   chunks=None)
        
        self.create_dataset(f'{gn}/z1d'   , data=z1d,   chunks=None)
        
        ## 0D
        self.create_dataset(f'{gn}/dz0'        , data=dz0, chunks=None)
        self.create_dataset(f'{gn}/dt'         , data=dt, chunks=None)
        
        self.create_dataset(f'{gn}/s2_edge'    , data=s2_edge    , chunks=None)
        self.create_dataset(f'{gn}/d99'        , data=d99        , chunks=None)
        self.create_dataset(f'{gn}/u1_99'      , data=u1_99      , chunks=None)
        
        self.create_dataset(f'{gn}/psvel_edge' , data=psvel_edge , chunks=None)
        self.create_dataset(f'{gn}/u1_edge'    , data=u1_edge    , chunks=None)
        self.create_dataset(f'{gn}/rho_edge'   , data=rho_edge   , chunks=None)
        self.create_dataset(f'{gn}/mu_edge'    , data=mu_edge    , chunks=None)
        self.create_dataset(f'{gn}/nu_edge'    , data=nu_edge    , chunks=None)
        self.create_dataset(f'{gn}/T_edge'     , data=T_edge     , chunks=None)
        
        self.create_dataset(f'{gn}/u_tau'      , data=u_tau      , chunks=None)
        
        self.create_dataset(f'{gn}/dds2_u1_wall' , data=dds2_u1_wall , chunks=None )
        self.create_dataset(f'{gn}/dds2_T'       , data=dds2_T       , chunks=None )
        self.create_dataset(f'{gn}/rho_wall'     , data=rho_wall     , chunks=None )
        self.create_dataset(f'{gn}/nu_wall'      , data=nu_wall      , chunks=None )
        self.create_dataset(f'{gn}/mu_wall'      , data=mu_wall      , chunks=None )
        self.create_dataset(f'{gn}/T_wall'       , data=T_wall       , chunks=None )
        self.create_dataset(f'{gn}/tau_wall'     , data=tau_wall     , chunks=None )
        self.create_dataset(f'{gn}/q_wall'       , data=q_wall       , chunks=None )
        
        self.create_dataset(f'{gn}/sc_u_in'    , data=sc_u_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_l_in'    , data=sc_l_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_t_in'    , data=sc_t_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_u_out'   , data=sc_u_out   , chunks=None)
        self.create_dataset(f'{gn}/sc_l_out'   , data=sc_l_out   , chunks=None)
        self.create_dataset(f'{gn}/sc_t_out'   , data=sc_t_out   , chunks=None)
        
        ## add integrated quantities (all 0D)
        for key,val in dd.items():
            self.create_dataset(f'{gn}/{key}', data=val, chunks=None)
        
        ## report
        if verbose:
            print(72*'-')
            even_print('Re_τ'                      , '%0.1f'%dd['Re_tau']            )
            even_print('Re_θ'                      , '%0.1f'%dd['Re_theta']          )
            even_print('θ'                         , '%0.5e [m]'%dd['theta_cmp']     )
            even_print('δ*'                        , '%0.5e [m]'%dd['dstar_cmp']     )
            even_print('H12'                       , '%0.5f'%dd['H12']               )
            ##
            even_print('δ99'                       , '%0.5e [m]'%d99                 )
            even_print('θ/δ99'                     , '%0.5f'%(dd['theta_cmp']/d99)   )
            even_print('δ*/δ99'                    , '%0.5f'%(dd['dstar_cmp']/d99)   )
            even_print('u_τ'                       , '%0.3f [m/s]'%u_tau             )
            even_print('ν_wall'                    , '%0.5e [m²/s]'%nu_wall          )
            even_print('τ_wall'                    , '%0.5e [Pa]'%tau_wall           )
            even_print('τ_wall/q_inf'              , '%0.5e'%(tau_wall/(self.rho_inf*self.U_inf**2)) )
            even_print('cf = 2·τ_wall/q_edge'      , '%0.5e'%(2*tau_wall/(rho_edge*u1_edge**2)) )
            even_print('t_meas'                    , '%0.5e [s]'%t_meas              )
            even_print('t_meas/tchar'              , '%0.1f'%(t_meas/self.tchar)     )
            even_print('t_eddy = t_meas/(δ99/u_τ)' , '%0.2f'%t_eddy                  )
            even_print('t_meas/(δ99/u1_99)'        , '%0.2f'%(t_meas/(d99/u1_99))    )
            even_print('t_meas/(20·δ99/u1_99)'     , '%0.2f'%(t_meas/(20*d99/u1_99)) )
            print(72*'-')
            even_print('sc_u_in = u_τ'               , '%0.5e [m/s]'%(sc_u_in,)  )
            even_print('sc_l_in = δ_ν = ν_wall/u_τ'  , '%0.5e [m]'%(sc_l_in,)    )
            even_print('sc_t_in = t_ν = ν_wall/u_τ²' , '%0.5e [s]'%(sc_t_in,)    )
            even_print('sc_u_out = u1_99'            , '%0.5e [m/s]'%(sc_u_out,) )
            even_print('sc_l_out = δ99'              , '%0.5e [m]'%(sc_l_out,)   )
            even_print('sc_t_out = δ99/u1_99'        , '%0.5e [s]'%(sc_t_out,)   )
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_mean_dimensional_data_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_mean_dimensional_data_ypln(self, **kwargs):
        '''
        get dimensionalized mean data for [y] plane
        --> save to existing CGD file with fsubtype=mean
        - assumes volume which is thin in [y] direction
        - an CGD which is the output of cgd.get_mean() should be opened here
        - not parallel
        '''
        
        pass
        raise NotImplementedError
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_mean_dimensional_data_ypln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_mean_dimensional_data_zpln(self, **kwargs):
        '''
        get dimensionalized mean data for [z] plane
        --> save to existing CGD file with fsubtype=mean
        - assumes volume which is thin in [z] direction
        - an CGD which is the output of cgd.get_mean() should be opened here
        - not parallel
        '''
        
        pass
        raise NotImplementedError
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.add_mean_dimensional_data_zpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing (unsteady)
    
    def get_prime(self, **kwargs):
        '''
        calc mean-removed (prime) variables in [t]
        --> save to new CGD file
        -----
        XI  : Reynolds primes : mean(XI)=0
        XII : Favre primes    : mean(ρ·XII)=0 --> mean(XII)≠0 !!
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        ## assert that the opened CGD has fsubtype 'unsteady'
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        if verbose: print('\n'+'cgd.get_prime()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        fn_cgd_mean  = kwargs.get('fn_cgd_mean',None)
        fn_cgd_prime = kwargs.get('fn_cgd_prime',None)
        sfp          = kwargs.get('scalars',None) ## scalars (for prime)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing prime file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        ## start timestep index
        ti_min = kwargs.get('ti_min',None)
        
        ## if writing Favre primes, copy over ρ --> mean(ρ·XII)=0 / mean(XII)≠0 !!
        if favre:
            copy_rho = True
        else:
            copy_rho = False
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if (sfp is None):
            sfp = self.scalars
        
        # === ranks
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === get times to take for prime
        if (ti_min is not None):
            ti_for_prime = np.copy( self.ti[ti_min:] )
        else:
            ti_for_prime = np.copy( self.ti )
        
        nt_prime       = ti_for_prime.shape[0]
        t_prime_start  = self.t[ti_for_prime[0]]
        t_prime_end    = self.t[ti_for_prime[-1]]
        duration_prime = t_prime_end - t_prime_start
        
        # === chunks
        #ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl_ = np.array_split(ti_for_prime,min(ct,nt_prime))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # === mean file name (for reading)
        if (fn_cgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_cgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_cgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_cgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if not os.path.isfile(fn_cgd_mean):
            raise FileNotFoundError('%s not found!'%fn_cgd_mean)
        
        # === prime file name (for writing)
        if (fn_cgd_prime is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_prime_h5_base = fname_root+'_prime.h5'
            #fn_cgd_prime = os.path.join(fname_path, fname_prime_h5_base)
            fn_cgd_prime = str(PurePosixPath(fname_path, fname_prime_h5_base))
            #fn_cgd_prime = Path(fname_path, fname_prime_h5_base)
        
        if verbose: even_print('fn_cgd'          , self.fname    )
        if verbose: even_print('fn_cgd_mean'     , fn_cgd_mean   )
        if verbose: even_print('fn_cgd_prime'    , fn_cgd_prime  )
        if verbose: even_print('do Favre avg'    , str(favre)    )
        if verbose: even_print('do Reynolds avg' , str(reynolds) )
        if verbose: even_print('copy rho'        , str(copy_rho) )
        if verbose: even_print('ct'              , '%i'%ct       )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('n timesteps prime','%i/%i'%(nt_prime,self.nt))
        if verbose: even_print('t index prime start','%i'%(ti_for_prime[0],))
        if verbose: even_print('t index prime end','%i'%(ti_for_prime[-1],))
        if verbose: even_print('t prime start','%0.2f [-]'%(t_prime_start,))
        if verbose: even_print('t prime end','%0.2f [-]'%(t_prime_end,))
        if verbose: even_print('duration prime','%0.2f [-]'%(duration_prime,))
        if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        itemsize = dtype.itemsize
        
        #data_gb      = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        data_gb      = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1       / 1024**3
        
        scalars_re = [ 'u','v','w','T','p','rho', 'utang','unorm', 'vort_x','vort_y','vort_z','vort_tang' ]
        scalars_fv = [ 'u','v','w','T',           'utang','unorm' ] ## p'' and ρ'' are never really needed
        
        scalars_re_ = []
        for scalar in scalars_re:
            if (scalar in self.scalars) and (scalar in sfp):
                scalars_re_.append(scalar)
        scalars_re = scalars_re_
        
        scalars_fv_ = []
        for scalar in scalars_fv:
            if (scalar in self.scalars) and (scalar in sfp):
                scalars_fv_.append(scalar)
        scalars_fv = scalars_fv_
        
        # ===
        
        comm_cgd_prime = MPI.COMM_WORLD
        
        with cgd(fn_cgd_prime, 'w', force=force, driver=self.driver, comm=self.comm, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_prime:
            
            ## initialize prime cgd from cgd
            hf_prime.init_from_cgd(self.fname, rx=rx,ry=ry,rz=rz, chunk_kb=chunk_kb)
            
            ## add top-level attributes
            #hf_prime.attrs['fclass'] = 'cgd'
            hf_prime.attrs['fsubtype'] = 'prime'
            
            #shape = (self.nt,self.nz,self.ny,self.nx)
            shape = (nt_prime,self.nz,self.ny,self.nx)
            
            ## determine dtypes for prime file
            for scalar in self.scalars:
                dset = self[f'data/{scalar}']
                dtype = dset.dtype
                if reynolds and (scalar in scalars_re):
                    hf_prime.scalars_dtypes_dict[f'{scalar}I'] = dtype
                if favre and (scalar in scalars_fv):
                    hf_prime.scalars_dtypes_dict[f'{scalar}II'] = dtype
            
            ##chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
            #chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
            
            # === initialize prime datasets + rho
            
            if copy_rho:
                
                dtype = self.scalars_dtypes_dict['rho']
                float_bytes = dtype.itemsize
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                
                if verbose:
                    even_print('initializing data/rho','%0.1f [GB]'%(data_gb,))
                
                dset = hf_prime.create_dataset('data/rho',
                                               shape=shape,
                                               dtype=dtype,
                                               chunks=chunks)
                
                chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                if verbose:
                    even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            for scalar in self.scalars:
                
                if reynolds:
                    if (scalar in scalars_re):
                        
                        dtype = hf_prime.scalars_dtypes_dict[f'{scalar}I']
                        float_bytes = dtype.itemsize
                        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                        
                        ## if ('data/%sI'%scalar in hf_prime):
                        ##     del hf_prime['data/%sI'%scalar]
                        if verbose:
                            even_print('initializing data/%sI'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        
                        dset = hf_prime.create_dataset(f'data/{scalar}I',
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_prime.scalars.append(f'{scalar}I')
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    if (scalar in scalars_fv):
                        
                        dtype = hf_prime.scalars_dtypes_dict[f'{scalar}II']
                        float_bytes = dtype.itemsize
                        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                        
                        ## if ('data/%sII'%scalar in hf_prime):
                        ##     del hf_prime['data/%sII'%scalar]
                        if verbose:
                            even_print('initializing data/%sII'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        
                        dset = hf_prime.create_dataset(f'data/{scalar}II',
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_prime.scalars.append(f'{scalar}II')
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if hf_prime.usingmpi: comm_cgd_prime.Barrier()
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            # === read unsteady + mean, do difference, write
            
            n_pbar = 0
            if favre or copy_rho:
                n_pbar += 1
            for scalar in self.scalars:
                if (scalar in scalars_re) and reynolds:
                    n_pbar += 1
                if (scalar in scalars_fv) and favre:
                    n_pbar += 1
            
            comm_cgd_mean = MPI.COMM_WORLD
            
            with cgd(fn_cgd_mean, 'r', driver=self.driver, comm=self.comm) as hf_mean:
                
                ## copy over 'data_dim' from mean file
                if ('data_dim' in hf_mean):
                    grp = hf_mean['data_dim']
                    for dsn in grp.keys():
                        ds = hf_mean[f'data_dim/{dsn}']
                        data = np.copy(ds[()])
                        #if (f'data_dim/{dsn}' in hf_prime):
                        #    del hf_prime[f'data_dim/{dsn}']
                        
                        #if self.usingmpi:
                        #    with dset.collective:
                        #        hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                        #else:
                        #    hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                        
                        hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                    
                    if verbose:
                        even_print('data_dim copied to prime',str(True))
                else:
                    if verbose:
                        even_print('data_dim copied to prime',str(False))
                
                if verbose:
                    progress_bar = tqdm(total=ct*n_pbar, ncols=100, desc='prime', leave=False, file=sys.stdout)
                
                for ctl_ in ctl:
                    ct1, ct2 = ctl_
                    ntc = ct2 - ct1
                    
                    #if verbose: tqdm.write(f'ct1,ct2 = {ct1:d},{ct2:d}')
                    #if verbose: tqdm.write(f'ntc = {ntc:d}')
                    
                    ## chunk range for writing to file (offset from read if using ti_min)
                    if (ti_min is not None):
                        #ct1w,ct2w = ct1-ti_min, ct2-ti_min ## doesnt work for (-) ti_min
                        ct1w,ct2w = ct1-ti_for_prime[0], ct2-ti_for_prime[0]
                    else:
                        ct1w,ct2w = ct1,ct2
                    
                    # ## debug report
                    # if verbose: tqdm.write('ct1,ct2 = %i,%i'%(ct1,ct2))
                    # if verbose: tqdm.write('ct1w,ct2w = %i,%i'%(ct1w,ct2w))
                    
                    if favre or copy_rho:
                        
                        ## read rho
                        dset = self['data/rho']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                rho = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                        else:
                            rho = dset[ct1:ct2,:,:,:].T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if verbose:
                            txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        ## write a copy of rho to the prime file
                        dset = hf_prime['data/rho']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if hf_prime.usingmpi: hf_prime.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = rho.T
                                dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = rho.T
                        else:
                            #dset[ct1:ct2,:,:,:] = rho.T
                            dset[ct1w:ct2w,:,:,:] = rho.T
                        if hf_prime.usingmpi: hf_prime.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            txt = even_print('write: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        if verbose: progress_bar.update()
                    
                    for scalar in self.scalars:
                        
                        if (scalar in scalars_re) or (scalar in scalars_fv):
                            
                            ## read CGD data
                            dset = self['data/%s'%scalar]
                            dtype = dset.dtype
                            float_bytes = dtype.itemsize
                            data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                            
                            if self.usingmpi: self.comm.Barrier()
                            t_start = timeit.default_timer()
                            if self.usingmpi:
                                with dset.collective:
                                    data = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                            else:
                                data = dset[ct1:ct2,:,:,:].T
                            if self.usingmpi: self.comm.Barrier()
                            t_delta = timeit.default_timer() - t_start
                            
                            if verbose:
                                txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                tqdm.write(txt)
                            
                            t_read       += t_delta
                            data_gb_read += data_gb
                            
                            # === do prime Reynolds
                            
                            if (scalar in scalars_re) and reynolds:
                                
                                ## read Reynolds avg from mean file
                                dset = hf_mean['data/%s'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_mean.nx * hf_mean.ny * hf_mean.nz * 1 / 1024**3
                                
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_mean.usingmpi:
                                    with dset.collective:
                                        data_mean_re = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                                else:
                                    data_mean_re = dset[()].T
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                ## if verbose:
                                ##     txt = even_print('read: %s (Re avg)'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                ##     tqdm.write(txt)
                                
                                ## calc mean-removed Reynolds
                                data_prime_re = data - data_mean_re
                                
                                ## if False:
                                ##     data_prime_re_mean = np.mean(data_prime_re, axis=-1, dtype=np.float64, keepdims=True).astype(np.float32)
                                ##     
                                ##     ## normalize [mean(prime)] by mean
                                ##     data_prime_re_mean = np.abs(np.divide(data_prime_re_mean,
                                ##                                           data_mean_re, 
                                ##                                           out=np.zeros_like(data_prime_re_mean), 
                                ##                                           where=data_mean_re!=0))
                                ##     
                                ##     # np.testing.assert_allclose( data_prime_re_mean , 
                                ##     #                             np.zeros_like(data_prime_re_mean, dtype=np.float32), atol=1e-4)
                                ##     if verbose:
                                ##         tqdm.write('max(abs(mean(%sI)/mean(%s)))=%0.4e'%(scalar,scalar,data_prime_re_mean.max()))
                                
                                ## write Reynolds prime
                                dset = hf_prime['data/%sI'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_prime.nx * hf_prime.ny * hf_prime.nz * ntc / 1024**3
                                
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_prime.usingmpi:
                                    with dset.collective:
                                        #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_re.T
                                        dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_re.T
                                else:
                                    #dset[ct1:ct2,:,:,:] = data_prime_re.T
                                    dset[ct1w:ct2w,:,:,:] = data_prime_re.T
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if verbose:
                                    txt = even_print('write: %sI'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                    tqdm.write(txt)
                                
                                if verbose: progress_bar.update()
                            
                            # === do prime Favre
                            
                            if (scalar in scalars_fv) and favre:
                                
                                ## read Favre avg from mean file
                                dset = hf_mean['data/%s_fv'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_mean.nx * hf_mean.ny * hf_mean.nz * 1 / 1024**3
                                
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_mean.usingmpi:
                                    with dset.collective:
                                        data_mean_fv = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                                else:
                                    data_mean_fv = dset[()].T
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                ## if verbose:
                                ##     txt = even_print('read: %s (Fv avg)'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                ##     tqdm.write(txt)
                                
                                ## calc mean-removed Favre
                                ## data_prime_fv = ( data - data_mean_fv ) * rho ## pre-multiply with ρ (has zero mean) --> better to not do this here
                                data_prime_fv = data - data_mean_fv
                                
                                ## write Favre prime
                                dset = hf_prime['data/%sII'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_prime.nx * hf_prime.ny * hf_prime.nz * ntc / 1024**3
                                
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_prime.usingmpi:
                                    with dset.collective:
                                        #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_fv.T
                                        dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_fv.T
                                else:
                                    #dset[ct1:ct2,:,:,:] = data_prime_fv.T
                                    dset[ct1w:ct2w,:,:,:] = data_prime_fv.T
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if verbose:
                                    txt = even_print('write: %sII'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                    tqdm.write(txt)
                                
                                pass
                                
                                if verbose: progress_bar.update()
                        
                        if self.usingmpi: self.comm.Barrier()
                        if hf_prime.usingmpi: comm_cgd_prime.Barrier()
                        if hf_mean.usingmpi: comm_cgd_mean.Barrier()
                    
                    ## report mem free
                    mem_avail_gb = psutil.virtual_memory().available/1024**3
                    mem_free_gb  = psutil.virtual_memory().free/1024**3
                    if verbose:
                        tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                
                if verbose:
                    progress_bar.close()
            
            # === replace dims/t array in prime file (if ti_min was given)
            if (ti_min is not None):
                t = np.copy( self.t[ti_min:] )
                if ('dims/t' in hf_prime):
                    del hf_prime['dims/t']
                hf_prime.create_dataset('dims/t', data=t)
            
            if hf_mean.usingmpi: comm_cgd_mean.Barrier()
        if hf_prime.usingmpi: comm_cgd_prime.Barrier()
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_cgd_prime, '%0.2f [GB]'%(os.path.getsize(fn_cgd_prime)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.get_prime() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_lambda2(self, **kwargs):
        '''
        calculate λ₂ & Q
        Jeong & Hussain (1996) : https://doi.org/10.1017/S0022112095000462
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx           = kwargs.get('rx',1)
        ry           = kwargs.get('ry',1)
        rz           = kwargs.get('rz',1)
        
        save_Q       = kwargs.get('save_Q',True)
        save_lambda2 = kwargs.get('save_lambda2',True)
        
        acc          = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','half')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        d = 1 ## derivative order
        stencil_npts = 2*math.floor((d+1)/2) - 1 + acc
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        
        # ===
        
        if verbose: print('\n'+'turbx.cgd.calc_lambda2()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## checks
        if all([(save_Q is False),(save_lambda2 is False)]):
            raise AssertionError('neither λ-2 nor Q set to be solved')
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('data/w' in self):
            raise ValueError('data/w not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('save_Q','%s'%save_Q)
        if verbose: even_print('save_lambda2','%s'%save_lambda2)
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        t_q_crit = 0.
        t_l2     = 0.
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # === extend the rank ranges (spatial range overlap)
        
        if self.usingmpi:
            
            n_overlap = stencil_npts_one_side + 3
            
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            zA = 0
            zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## overlap in [y]
            if (t4d[1]!=0):
                ry1, ry2 = ry1-n_overlap, ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                ry1, ry2 = ry1, ry2+n_overlap
            
            ## overlap in [z]
            if (t4d[2]!=0):
                rz1, rz2 = rz1-n_overlap, rz2
                zA += n_overlap
                zB += n_overlap
            if (t4d[2]!=rz-1):
                rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        dtype = self['data/u'].dtype
        itemsize = dtype.itemsize
        float_bytes = dtype.itemsize
        
        data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        # === initialize 4D arrays in HDF5
        
        if save_lambda2:
            
            self.scalars_dtypes_dict['lambda2'] = dtype
            
            if verbose:
                even_print('initializing data/lambda2','%0.2f [GB]'%(data_gb,))
            if ('data/lambda2' in self):
                del self['data/lambda2']
            dset = self.create_dataset('data/lambda2', 
                                        shape=shape, 
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if save_Q:
            
            self.scalars_dtypes_dict['Q'] = dtype
            
            if verbose:
                even_print('initializing data/Q','%0.2f [GB]'%(data_gb,))
            if ('data/Q' in self):
                del self['data/Q']
            dset = self.create_dataset('data/Q', 
                                        shape=shape, 
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === read grid collectively (CGD)
        
        dset = self['dims/x']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                x_ = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            x_ = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read x', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/y']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                y_ = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            y_ = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read y', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/z']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                z_ = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            z_ = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        # === get metric tensor (CGD / curvilinear)
        
        t_start = timeit.default_timer()
        
        M = get_metric_tensor_3d(x_, y_, z_, acc=acc, edge_stencil=edge_stencil, verbose=verbose)
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get metric tensor','%0.3f [s]'%(t_delta,), s=True) )
        
        # === split out the metric tensor components
        
        ddx_q1 = np.copy( M[:,:,:,0,0] ) ## ξ_x
        ddx_q2 = np.copy( M[:,:,:,1,0] ) ## η_x
        ddx_q3 = np.copy( M[:,:,:,2,0] ) ## ζ_x
        ##
        ddy_q1 = np.copy( M[:,:,:,0,1] ) ## ξ_y
        ddy_q2 = np.copy( M[:,:,:,1,1] ) ## η_y
        ddy_q3 = np.copy( M[:,:,:,2,1] ) ## ζ_y
        ##
        ddz_q1 = np.copy( M[:,:,:,0,2] ) ## ξ_z
        ddz_q2 = np.copy( M[:,:,:,1,2] ) ## η_z
        ddz_q3 = np.copy( M[:,:,:,2,2] ) ## ζ_z
        
        M = None; del M ## free memory
        
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        #if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## the 'computational' grid (unit Cartesian)
        #x_comp = np.arange(nx, dtype=np.float64)
        #y_comp = np.arange(ny, dtype=np.float64)
        #z_comp = np.arange(nz, dtype=np.float64)
        x_comp = 1.
        y_comp = 1.
        z_comp = 1.
        
        # === main loop
        
        if verbose:
            progress_bar = tqdm(total=self.nt, ncols=100, desc='calc λ2', leave=False, file=sys.stdout)
        
        for ti in self.ti:
            
            # === read u,v,w
            
            dset = self['data/u']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    u_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                u_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read u', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/v']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    v_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                v_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read v', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/w']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    w_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                w_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # ===
            
            # if self.usingmpi:
            #     x_ = np.copy(self.x[rx1:rx2])
            #     y_ = np.copy(self.y[ry1:ry2])
            #     z_ = np.copy(self.z[rz1:rz2])
            # else:
            #     x_ = np.copy(self.x)
            #     y_ = np.copy(self.y)
            #     z_ = np.copy(self.z)
            
            # ==========================================================
            # get velocity gradient (strain) tensor ∂(u,v,w)/∂(x,y,z)
            # ==========================================================
            
            t_start = timeit.default_timer()
            
            # === ∂(u)/∂(x,y,z)
            
            ddx_u_comp = gradient(u_, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_u_comp = gradient(u_, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_u_comp = gradient(u_, z_comp, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            u_ = None; del u_ ## free memory
            
            ## get physical grid gradient elements by taking
            ##  inner product with metric tensor
            ddx_u = np.copy( ddx_u_comp * ddx_q1 +
                             ddy_u_comp * ddx_q2 +
                             ddz_u_comp * ddx_q3 )
            
            ddy_u = np.copy( ddx_u_comp * ddy_q1 +
                             ddy_u_comp * ddy_q2 +
                             ddz_u_comp * ddy_q3 )
            
            ddz_u = np.copy( ddx_u_comp * ddz_q1 +
                             ddy_u_comp * ddz_q2 +
                             ddz_u_comp * ddz_q3 )
            
            ddx_u_comp = None; del ddx_u_comp ## free memory
            ddy_u_comp = None; del ddy_u_comp ## free memory
            ddz_u_comp = None; del ddz_u_comp ## free memory
            
            # === ∂(v)/∂(x,y,z)
            
            ddx_v_comp = gradient(v_, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_v_comp = gradient(v_, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_v_comp = gradient(v_, z_comp, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            v_ = None; del v_ ## free memory
            
            ddx_v = np.copy( ddx_v_comp * ddx_q1 + \
                             ddy_v_comp * ddx_q2 + \
                             ddz_v_comp * ddx_q3 )
            
            ddy_v = np.copy( ddx_v_comp * ddy_q1 + \
                             ddy_v_comp * ddy_q2 + \
                             ddz_v_comp * ddy_q3 )
            
            ddz_v = np.copy( ddx_v_comp * ddz_q1 + \
                             ddy_v_comp * ddz_q2 + \
                             ddz_v_comp * ddz_q3 )
            
            ddx_v_comp = None; del ddx_v_comp ## free memory
            ddy_v_comp = None; del ddy_v_comp ## free memory
            ddz_v_comp = None; del ddz_v_comp ## free memory
            
            # === ∂(w)/∂(x,y,z)
            
            ddx_w_comp = gradient(w_, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_w_comp = gradient(w_, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_w_comp = gradient(w_, z_comp, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            w_ = None; del w_ ## free memory
            
            ddx_w = np.copy( ddx_w_comp * ddx_q1 + \
                             ddy_w_comp * ddx_q2 + \
                             ddz_w_comp * ddx_q3 )
            
            ddy_w = np.copy( ddx_w_comp * ddy_q1 + \
                             ddy_w_comp * ddy_q2 + \
                             ddz_w_comp * ddy_q3 )
            
            ddz_w = np.copy( ddx_w_comp * ddz_q1 + \
                             ddy_w_comp * ddz_q2 + \
                             ddz_w_comp * ddz_q3 )
            
            ddx_w_comp = None; del ddx_w_comp ## free memory
            ddy_w_comp = None; del ddy_w_comp ## free memory
            ddz_w_comp = None; del ddz_w_comp ## free memory
            
            # ===
            
            ## free memory
            u_ = None; del u_
            v_ = None; del v_
            w_ = None; del w_
            gc.collect()
            
            strain = np.copy( np.stack((np.stack((ddx_u, ddy_u, ddz_u), axis=3),
                                        np.stack((ddx_v, ddy_v, ddz_v), axis=3),
                                        np.stack((ddx_w, ddy_w, ddz_w), axis=3)), axis=4) )
            
            t_delta = timeit.default_timer() - t_start
            if verbose: tqdm.write( even_print('get strain ∂(u,v,v)/∂(x,y,z)' , '%0.3f [s]'%(t_delta,), s=True) )
            
            ## free memory
            ddx_u = None; del ddx_u
            ddy_u = None; del ddy_u
            ddz_u = None; del ddz_u
            ddx_v = None; del ddx_v
            ddy_v = None; del ddy_v
            ddz_v = None; del ddz_v
            ddx_w = None; del ddx_w
            ddy_w = None; del ddy_w
            ddz_w = None; del ddz_w
            gc.collect()
            
            # === get the rate-of-strain & vorticity tensors
            
            S = np.copy( 0.5*(strain + np.transpose(strain, axes=(0,1,2,4,3))) ) ## strain rate tensor (symmetric)
            O = np.copy( 0.5*(strain - np.transpose(strain, axes=(0,1,2,4,3))) ) ## rotation rate tensor (anti-symmetric)
            # np.testing.assert_allclose(S+O, strain, atol=1.e-6)
            
            ## free memory
            strain = None; del strain
            gc.collect()
            
            # === Q : second invariant of characteristics equation: λ³ + Pλ² + Qλ + R = 0
            
            if save_Q:
                
                t_start = timeit.default_timer()
                
                O_norm  = np.linalg.norm(O, ord='fro', axis=(3,4)) ## Frobenius norm
                S_norm  = np.linalg.norm(S, ord='fro', axis=(3,4))
                Q       = 0.5*(O_norm**2 - S_norm**2)
                
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print('calc Q','%s'%format_time_string(t_delta), s=True))
                
                dset = self['data/Q']
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ti,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = Q[xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ti,:,:,:] = Q.T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = float_bytes * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print('write Q','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                # === second invariant : Q --> an equivalent formulation using eigenvalues (but much slower)
                if False:
                    Q_bak = np.copy(Q)
                    t_start = timeit.default_timer()
                    eigvals = np.linalg.eigvals(strain)
                    P       = -1*np.sum(eigvals, axis=-1) ## first invariant : P
                    SijSji  = np.einsum('xyzij,xyzji->xyz', S, S)
                    OijOji  = np.einsum('xyzij,xyzji->xyz', O, O)
                    Q       = 0.5*(P**2 - SijSji - OijOji)
                    t_delta = timeit.default_timer() - t_start
                    if verbose: tqdm.write(even_print('calc Q','%s'%format_time_string(t_delta), s=True))
                    np.testing.assert_allclose(Q.imag, np.zeros_like(Q.imag, dtype=np.float32), atol=1e-6)
                    Q = np.copy(Q.real)
                    np.testing.assert_allclose(Q, Q_bak, rtol=1e-2, atol=1e-5)
                
                ## free memory
                O_norm = None; del O_norm
                S_norm = None; del S_norm
                Q = None; del Q
                gc.collect()
            
            # === λ₂
            
            if save_lambda2:
                
                t_start = timeit.default_timer()
                
                # === S² and Ω²
                SikSkj = np.einsum('xyzik,xyzkj->xyzij', S, S)
                OikOkj = np.einsum('xyzik,xyzkj->xyzij', O, O)
                #np.testing.assert_allclose(np.matmul(S,S), SikSkj, atol=1e-6)
                #np.testing.assert_allclose(np.matmul(O,O), OikOkj, atol=1e-6)
                
                ## free memory
                S = None; del S
                O = None; del O
                gc.collect()
                
                # === Eigenvalues of (S²+Ω²) --> a real symmetric (Hermitian) matrix
                eigvals            = np.linalg.eigvalsh(SikSkj+OikOkj, UPLO='L')
                #eigvals_sort_order = np.argsort(np.abs(eigvals), axis=3) ## sort order of λ --> magnitude (wrong)
                eigvals_sort_order = np.argsort(eigvals, axis=3) ## sort order of λ
                eigvals_sorted     = np.take_along_axis(eigvals, eigvals_sort_order, axis=3) ## do λ sort
                lambda2            = np.squeeze(eigvals_sorted[:,:,:,1]) ## λ2 is the second eigenvalue (index=1)
                t_delta            = timeit.default_timer() - t_start
                
                if verbose: tqdm.write(even_print('calc λ2','%s'%format_time_string(t_delta), s=True))
                
                dset = self['data/lambda2']
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ti,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = lambda2[xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ti,:,:,:] = lambda2.T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = float_bytes * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print('write λ2','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## free memory
                lambda2 = None; del lambda2
                eigvals = None; del eigvals
                eigvals_sort_order = None; del eigvals_sort_order
                eigvals_sorted = None; del eigvals_sorted
                gc.collect()
            
            if verbose: progress_bar.update()
            if verbose and (ti<self.nt-1): tqdm.write( '---' )
        if verbose: progress_bar.close()
        if verbose: print(72*'-')
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.cgd.calc_lambda2() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vel_tangnorm(self, **kwargs):
        '''
        calculate velocity in wall-tangent & wall-normal coordinates [utang,unorm]=[uξ,uη]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        if verbose: print('\n'+'cgd.calc_vel_tangnorm()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('csys/vtang' in self):
            raise ValueError('csys/vtang not in hdf5')
        if not ('csys/vnorm' in self):
            raise ValueError('csys/vnorm not in hdf5')
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ct>self.nt):
            raise AssertionError('ct>self.nt')
        
        if ( self['csys/vnorm'].shape != (self.nx,self.ny,2)):
            raise ValueError(f'csys/vnorm shape != (nx,ny,2)')
        if ( self['csys/vtang'].shape != (self.nx,self.ny,2)):
            raise ValueError(f'csys/vtang shape != (nx,ny,2)')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        # === ranks
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            rx1 = 0
            rx2 = self.nx
            ry1 = 0
            ry2 = self.ny
            rz1 = 0
            rz2 = self.nz
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # ===
        
        ## read rank-local tang/norm basis vectors vtang & vnorm
        dset = self['csys/vtang']
        vtang = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        dset = self['csys/vnorm']
        vnorm = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        
        #vtang = np.broadcast_to(vtang[:,:,np.newaxis,np.newaxis,:], (nxr,nyr,nzr,1,2))
        #vnorm = np.broadcast_to(vnorm[:,:,np.newaxis,np.newaxis,:], (nxr,nyr,nzr,1,2))
        
        # ===
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        
        data_gb = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        for scalar in ['utang','unorm']:
            
            self.scalars_dtypes_dict[scalar] = dtype
            
            if verbose:
                even_print(f'initializing data/{scalar}','%0.1f [GB]'%(data_gb,))
            if (f'data/{scalar}' in self):
                del self[f'data/{scalar}']
            dset = self.create_dataset(f'data/{scalar}',
                                        shape=shape,
                                        dtype=dtype,
                                        chunks=chunks )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === main loop
        
        if verbose:
            #progress_bar = tqdm(total=self.nt, ncols=100, desc='calc unorm,utang', leave=False, file=sys.stdout)
            progress_bar = tqdm(total=len(ctl), ncols=100, desc='calc unorm,utang', leave=False, file=sys.stdout)
        
        #for ti in self.ti:
        for ctl_ in ctl:
            ct1, ct2 = ctl_
            ntc = ct2 - ct1
            
            # === read u,v
            
            dset = self['data/u']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    u = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
            else:
                u = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read u', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/v']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    v = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
            else:
                v = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read v', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # === perform csys transformation
            
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            
            uv = np.stack((u,v), axis=-1)
            
            if (uv.ndim!=4+1):
                raise ValueError
            
            ## inner product of velocity vector and basis vector (csys transform)
            utang = np.einsum('xyi,xyzti->xyzt', vtang, uv)
            unorm = np.einsum('xyi,xyzti->xyzt', vnorm, uv)
            
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            if verbose: tqdm.write(even_print('csys trafo','%0.3f [s]'%(t_delta,), s=True))
            
            # === convert back to float32 before write
            
            utang = np.copy( utang.astype(np.float32) )
            unorm = np.copy( unorm.astype(np.float32) )
            
            # === write
            
            dset = self['data/utang']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = utang.T
            else:
                dset[ct1:ct2,:,:,:] = utang.T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz / 1024**3
            
            if verbose: tqdm.write(even_print('write utang','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            dset = self['data/unorm']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = unorm.T
            else:
                dset[ct1:ct2,:,:,:] = unorm.T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz / 1024**3
            
            if verbose: tqdm.write(even_print('write unorm','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_vel_tangnorm() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vorticity(self, **kwargs):
        '''
        calculate vorticity vector [ω_x,ω_y,ω_z]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx           = kwargs.get('rx',1)
        ry           = kwargs.get('ry',1)
        rz           = kwargs.get('rz',1)
        ct           = kwargs.get('ct',1) ## n chunks [t]
        
        acc          = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','half')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        d = 1 ## derivative order
        stencil_npts = 2*math.floor((d+1)/2) - 1 + acc
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        
        # ===
        
        if verbose: print('\n'+'cgd.calc_vorticity()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('data/w' in self):
            raise ValueError('data/w not in hdf5')
        #if not ('csys/vtang' in self):
        #    raise ValueError('csys/vtang not in hdf5')
        #if not ('csys/vnorm' in self):
        #    raise ValueError('csys/vnorm not in hdf5')
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ct>self.nt):
            raise AssertionError('ct>self.nt')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## time chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        # === extend the rank ranges (spatial range overlap)
        
        if self.usingmpi:
            
            n_overlap = stencil_npts_one_side + 3
            
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            zA = 0
            zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## ## overlap in [y]
            ## if (t4d[1]!=0):
            ##     ry1, ry2 = ry1-n_overlap, ry2
            ##     yA += n_overlap
            ##     yB += n_overlap
            ## if (t4d[1]!=ry-1):
            ##     ry1, ry2 = ry1, ry2+n_overlap
            
            ## overlap in [y] --> TODO: needs checking
            if (t4d[1]!=0):
                ry1 = ry1-n_overlap
                #ry2 = ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                #ry1 = ry1
                ry2 = min(ry2+n_overlap,self.ny)
            
            if (ry2>self.ny):
                #raise IndexError
                print('ry2>self.ny')
                self.comm.Abort(1)
            if (ry1<0):
                #raise IndexError
                print('ry1<0')
                self.comm.Abort(1)
            
            ## overlap in [z]
            if (t4d[2]!=0):
                rz1, rz2 = rz1-n_overlap, rz2
                zA += n_overlap
                zB += n_overlap
            if (t4d[2]!=rz-1):
                rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        # ## read rank-local tang/norm basis vectos vtang & vnorm
        # dset = self['csys/vtang']
        # vtang = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        # dset = self['csys/vnorm']
        # vnorm = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        # 
        # vtang = np.broadcast_to(vtang[:,:,np.newaxis,:], (nxr,nyr,nzr,2))
        # vnorm = np.broadcast_to(vnorm[:,:,np.newaxis,:], (nxr,nyr,nzr,2))
        
        # ===
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        
        data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        # === initialize 4D arrays in HDF5
        
        for dsn in ['vort_x','vort_y','vort_z']:
            
            self.scalars_dtypes_dict[dsn] = dtype
            
            if verbose:
                even_print(f'initializing data/{dsn}','%0.2f [GB]'%(data_gb,))
            if (f'data/{dsn}' in self):
                del self[f'data/{dsn}']
            dset = self.create_dataset( f'data/{dsn}',
                                        shape=shape,
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === read grid collectively (CGD)
        
        dset = self['dims/x']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                x = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            x = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read x', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/y']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                y = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            y = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read y', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/z']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                z = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            z = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        # === get metric tensor (CGD / curvilinear)
        
        t_start = timeit.default_timer()
        
        M = get_metric_tensor_3d(x, y, z, acc=acc, edge_stencil=edge_stencil, verbose=verbose, no_warn=True)
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get metric tensor','%0.3f [s]'%(t_delta,), s=True) )
        
        # === split out the metric tensor components
        
        ddx_q1 = np.copy( M[:,:,:,0,0] ) ## ξ_x
        ddx_q2 = np.copy( M[:,:,:,1,0] ) ## η_x
        ddx_q3 = np.copy( M[:,:,:,2,0] ) ## ζ_x
        ##
        ddy_q1 = np.copy( M[:,:,:,0,1] ) ## ξ_y
        ddy_q2 = np.copy( M[:,:,:,1,1] ) ## η_y
        ddy_q3 = np.copy( M[:,:,:,2,1] ) ## ζ_y
        ##
        ddz_q1 = np.copy( M[:,:,:,0,2] ) ## ξ_z
        ddz_q2 = np.copy( M[:,:,:,1,2] ) ## η_z
        ddz_q3 = np.copy( M[:,:,:,2,2] ) ## ζ_z
        
        M = None; del M ## free memory
        
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        #if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## the 'computational' grid (unit Cartesian)
        #x_comp = np.arange(nx, dtype=np.float64)
        #y_comp = np.arange(ny, dtype=np.float64)
        #z_comp = np.arange(nz, dtype=np.float64)
        x_comp = 1.
        y_comp = 1.
        z_comp = 1.
        
        # === main loop
        
        if verbose:
            #progress_bar = tqdm(total=self.nt, ncols=100, desc='calc ω', leave=False, file=sys.stdout)
            progress_bar = tqdm(total=len(ctl), ncols=100, desc='calc ω', leave=False, file=sys.stdout)
        
        #for ti in self.ti:
        for ctl_ in ctl:
            ct1, ct2 = ctl_
            ntc = ct2 - ct1
            
            # === read u,v,w
            
            dset = self['data/u']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    u = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                u = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read u', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/v']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    v = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                v = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read v', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/w']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    w = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                w = dset[ct1:ct2,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # ==========================================================
            # get velocity gradient (strain) tensor ∂(u,v,w)/∂(x,y,z)
            # ==========================================================
            
            t_start = timeit.default_timer()
            
            # === ∂(u)/∂(x,y,z)
            
            ## rectilinear case
            # ddx_u = gradient(u, x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddy_u = gradient(u, y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddz_u = gradient(u, z, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            ## get computational grid gradient elements
            ddx_u_comp = gradient(u, x_comp, axis=0, acc=acc, d=1, no_warn=True)
            ddy_u_comp = gradient(u, y_comp, axis=1, acc=acc, d=1, no_warn=True)
            ddz_u_comp = gradient(u, z_comp, axis=2, acc=acc, d=1, no_warn=True)
            u = None; del u ## free memory
            
            ## get physical grid gradient elements by taking
            ##  inner product with metric tensor
            ddx_u = np.copy( ddx_u_comp * ddx_q1[:,:,:,np.newaxis] +
                             ddy_u_comp * ddx_q2[:,:,:,np.newaxis] +
                             ddz_u_comp * ddx_q3[:,:,:,np.newaxis] )
            
            ddy_u = np.copy( ddx_u_comp * ddy_q1[:,:,:,np.newaxis] +
                             ddy_u_comp * ddy_q2[:,:,:,np.newaxis] +
                             ddz_u_comp * ddy_q3[:,:,:,np.newaxis] )
            
            ddz_u = np.copy( ddx_u_comp * ddz_q1[:,:,:,np.newaxis] +
                             ddy_u_comp * ddz_q2[:,:,:,np.newaxis] +
                             ddz_u_comp * ddz_q3[:,:,:,np.newaxis] )
            
            ddx_u_comp = None; del ddx_u_comp ## free memory
            ddy_u_comp = None; del ddy_u_comp ## free memory
            ddz_u_comp = None; del ddz_u_comp ## free memory
            
            # === ∂(v)/∂(x,y,z)
            
            ## rectilinear case
            # ddx_v = gradient(v, x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddy_v = gradient(v, y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddz_v = gradient(v, z, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            ddx_v_comp = gradient(v, x_comp, axis=0, acc=acc, d=1, no_warn=True)
            ddy_v_comp = gradient(v, y_comp, axis=1, acc=acc, d=1, no_warn=True)
            ddz_v_comp = gradient(v, z_comp, axis=2, acc=acc, d=1, no_warn=True)
            v = None; del v ## free memory
            
            ddx_v = np.copy( ddx_v_comp * ddx_q1[:,:,:,np.newaxis] + \
                             ddy_v_comp * ddx_q2[:,:,:,np.newaxis] + \
                             ddz_v_comp * ddx_q3[:,:,:,np.newaxis] )
            
            ddy_v = np.copy( ddx_v_comp * ddy_q1[:,:,:,np.newaxis] + \
                             ddy_v_comp * ddy_q2[:,:,:,np.newaxis] + \
                             ddz_v_comp * ddy_q3[:,:,:,np.newaxis] )
            
            ddz_v = np.copy( ddx_v_comp * ddz_q1[:,:,:,np.newaxis] + \
                             ddy_v_comp * ddz_q2[:,:,:,np.newaxis] + \
                             ddz_v_comp * ddz_q3[:,:,:,np.newaxis] )
            
            ddx_v_comp = None; del ddx_v_comp ## free memory
            ddy_v_comp = None; del ddy_v_comp ## free memory
            ddz_v_comp = None; del ddz_v_comp ## free memory
            
            # === ∂(w)/∂(x,y,z)
            
            ## rectilinear case
            # ddx_w = gradient(w, x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddy_w = gradient(w, y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            # ddz_w = gradient(w, z, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            ddx_w_comp = gradient(w, x_comp, axis=0, acc=acc, d=1, no_warn=True)
            ddy_w_comp = gradient(w, y_comp, axis=1, acc=acc, d=1, no_warn=True)
            ddz_w_comp = gradient(w, z_comp, axis=2, acc=acc, d=1, no_warn=True)
            w = None; del w ## free memory
            
            ddx_w = np.copy( ddx_w_comp * ddx_q1[:,:,:,np.newaxis] + \
                             ddy_w_comp * ddx_q2[:,:,:,np.newaxis] + \
                             ddz_w_comp * ddx_q3[:,:,:,np.newaxis] )
            
            ddy_w = np.copy( ddx_w_comp * ddy_q1[:,:,:,np.newaxis] + \
                             ddy_w_comp * ddy_q2[:,:,:,np.newaxis] + \
                             ddz_w_comp * ddy_q3[:,:,:,np.newaxis] )
            
            ddz_w = np.copy( ddx_w_comp * ddz_q1[:,:,:,np.newaxis] + \
                             ddy_w_comp * ddz_q2[:,:,:,np.newaxis] + \
                             ddz_w_comp * ddz_q3[:,:,:,np.newaxis] )
            
            ddx_w_comp = None; del ddx_w_comp ## free memory
            ddy_w_comp = None; del ddy_w_comp ## free memory
            ddz_w_comp = None; del ddz_w_comp ## free memory
            
            # ===
            
            ## free memory
            u_ = None; del u_
            v_ = None; del v_
            w_ = None; del w_
            gc.collect()
            
            # ===
            
            ## 4D [scalar][x,y,z] structured array
            names = ['vort_x', 'vort_y', 'vort_z']
            formats  = [ np.float32 for s in names ]
            vort = np.zeros(shape=ddx_u.shape, dtype={'names':names, 'formats':formats})
            
            vort['vort_x'] = ddy_w - ddz_v
            vort['vort_y'] = ddz_u - ddx_w
            vort['vort_z'] = ddx_v - ddy_u
            
            ## free memory
            ddx_u = None; del ddx_u
            ddy_u = None; del ddy_u
            ddz_u = None; del ddz_u
            ddx_v = None; del ddx_v
            ddy_v = None; del ddy_v
            ddz_v = None; del ddz_v
            ddx_w = None; del ddx_w
            ddy_w = None; del ddy_w
            ddz_w = None; del ddz_w
            gc.collect()
            
            for scalar in vort.dtype.names:
                dset = self[f'data/{scalar}']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ct1:ct2,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = vort[scalar][xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ct1:ct2,:,:,:] = vort[scalar].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print(f'write {scalar}','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            if verbose: progress_bar.update()
            #if verbose and (ti<self.nt-1): tqdm.write( '---' )
        if verbose: progress_bar.close()
        if verbose: print(72*'-')
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_vorticity() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vorticity_tangnorm(self, **kwargs):
        '''
        calculate vorticity vector [ω_ξ,ω_η]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx           = kwargs.get('rx',1)
        ry           = kwargs.get('ry',1)
        rz           = kwargs.get('rz',1)
        ct           = kwargs.get('ct',1) ## n chunks [t]
        
        acc          = kwargs.get('acc',6)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        d = 1 ## derivative order
        stencil_npts = 2*int(np.floor((d+1)/2)) - 1 + acc
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        
        # ===
        
        if verbose: print('\n'+'cgd.calc_vorticity_tangnorm()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('csys/vtang' in self):
            raise ValueError('csys/vtang not in hdf5')
        if not ('csys/vnorm' in self):
            raise ValueError('csys/vnorm not in hdf5')
        if not ('data/utang' in self):
            raise ValueError('data/utang not in hdf5')
        if not ('data/unorm' in self):
            raise ValueError('data/unorm not in hdf5')
        if not ('dims/snorm' in self):
            raise ValueError('dims/snorm not in hdf5')
        if not ('dims/z' in self):
            raise ValueError('dims/z not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ct>self.nt):
            raise AssertionError('ct>self.nt')
        
        if ( self['csys/vnorm'].shape != (self.nx,self.ny,2)):
            raise ValueError(f'csys/vnorm shape != (nx,ny,2)')
        if ( self['csys/vtang'].shape != (self.nx,self.ny,2)):
            raise ValueError(f'csys/vtang shape != (nx,ny,2)')
        if ( self['dims/snorm'].shape != (self.ny,)):
            raise ValueError(f'dims/snorm shape != (ny,)')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## time chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # === extend the rank ranges (spatial range overlap)
        
        if self.usingmpi:
            
            n_overlap = stencil_npts_one_side + 3
            
            ## default start:end indices for local-->global write
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            zA = 0
            zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## ## overlap in [y]
            ## if (t4d[1]!=0):
            ##     ry1, ry2 = ry1-n_overlap, ry2
            ##     yA += n_overlap
            ##     yB += n_overlap
            ## if (t4d[1]!=ry-1):
            ##     ry1, ry2 = ry1, ry2+n_overlap
            
            ## overlap in [y] --> TODO: needs checking
            if (t4d[1]!=0):
                ry1 = ry1-n_overlap
                #ry2 = ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                #ry1 = ry1
                ry2 = min(ry2+n_overlap,self.ny)
            
            if (ry2>self.ny):
                #raise IndexError
                print('ry2>self.ny')
                self.comm.Abort(1)
            if (ry1<0):
                #raise IndexError
                print('ry1<0')
                self.comm.Abort(1)
            
            ## overlap in [z]
            if (t4d[2]!=0):
                rz1, rz2 = rz1-n_overlap, rz2
                zA += n_overlap
                zB += n_overlap
            if (t4d[2]!=rz-1):
                rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        #time.sleep(1)
        #if verbose: print('')
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        dset = self['dims/snorm']
        snorm = np.copy( dset[ry1:ry2] )
        
        ## read rank-local tang/norm basis vectors vtang & vnorm
        dset = self['csys/vtang']
        vtang = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        dset = self['csys/vnorm']
        vnorm = np.copy( dset[rx1:rx2,ry1:ry2,:] )
        
        ## convert vtang,vnorm (nx,ny,2) --> (nx,ny,nz,3)
        vtang3d = np.zeros((nxr,nyr,nzr,3), dtype=np.float64)
        vnorm3d = np.zeros((nxr,nyr,nzr,3), dtype=np.float64)
        vspan3d = np.zeros((nxr,nyr,nzr,3), dtype=np.float64)
        
        vtang3d[:,:,:,:2] = vtang[:,:,np.newaxis,:]
        vnorm3d[:,:,:,:2] = vnorm[:,:,np.newaxis,:]
        vspan3d[:,:,:,-1] = 1.
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        dtype = np.dtype(np.float32)
        float_bytes = dtype.itemsize
        
        data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        #chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        # === initialize 4D arrays in HDF5
        
        for dsn in ['vort_tang']:
            
            self.scalars_dtypes_dict[dsn] = dtype
            
            if verbose:
                even_print(f'initializing data/{dsn}','%0.2f [GB]'%(data_gb,))
            if (f'data/{dsn}' in self):
                del self[f'data/{dsn}']
            dset = self.create_dataset( f'data/{dsn}',
                                        shape=shape,
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === read grid collectively (CGD)
        
        dset = self['dims/x']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                x = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            x = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read x', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/y']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                y = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            y = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read y', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        dset = self['dims/z']
        float_bytes = dset.dtype.itemsize
        if self.usingmpi: self.comm.Barrier()
        t_start = timeit.default_timer()
        if self.usingmpi:
            with dset.collective:
                z = np.copy( dset[rz1:rz2,ry1:ry2,rx1:rx2].T )
        else:
            z = np.copy( dset[()].T )
        if self.usingmpi: self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
        if verbose:
            tqdm.write( even_print('read z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
        
        # ===
        
        ## assuming dim [z] does not require curvilinear description
        np.testing.assert_allclose(z[0,0,:], z[-1,-1,:], rtol=1e-14, atol=1e-14)
        z1d = np.copy(z[0,0,:])
        
        # === get metric tensor (CGD / curvilinear)
        
        ### t_start = timeit.default_timer()
        ### 
        ### M = get_metric_tensor_3d(x, y, z, acc=acc, edge_stencil=edge_stencil, verbose=verbose, no_warn=True)
        ### 
        ### t_delta = timeit.default_timer() - t_start
        ### if verbose: tqdm.write( even_print('get metric tensor','%0.3f [s]'%(t_delta,), s=True) )
        ### 
        ### # === split out the metric tensor components
        ### 
        ### ddx_q1 = np.copy( M[:,:,:,0,0] ) ## ξ_x
        ### ddx_q2 = np.copy( M[:,:,:,1,0] ) ## η_x
        ### ddx_q3 = np.copy( M[:,:,:,2,0] ) ## ζ_x
        ### ##
        ### ddy_q1 = np.copy( M[:,:,:,0,1] ) ## ξ_y
        ### ddy_q2 = np.copy( M[:,:,:,1,1] ) ## η_y
        ### ddy_q3 = np.copy( M[:,:,:,2,1] ) ## ζ_y
        ### ##
        ### ddz_q1 = np.copy( M[:,:,:,0,2] ) ## ξ_z
        ### ddz_q2 = np.copy( M[:,:,:,1,2] ) ## η_z
        ### ddz_q3 = np.copy( M[:,:,:,2,2] ) ## ζ_z
        ### 
        ### M = None; del M ## free memory
        ### 
        ### mem_total_gb = psutil.virtual_memory().total/1024**3
        ### mem_avail_gb = psutil.virtual_memory().available/1024**3
        ### mem_free_gb  = psutil.virtual_memory().free/1024**3
        ### if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        ### #if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        ### if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        ### if verbose: print(72*'-')
        
        ### ## the 'computational' grid (unit Cartesian)
        ### #x_comp = np.arange(nx, dtype=np.float64)
        ### #y_comp = np.arange(ny, dtype=np.float64)
        ### #z_comp = np.arange(nz, dtype=np.float64)
        ### x_comp = 1.
        ### y_comp = 1.
        ### z_comp = 1.
        
        # === main loop
        
        vort_tang_calc_method = 1 ## 1,2
        
        # option 1: calculate vort_s1 directly using [w/u3] & [unorm/u2] over coord vecs [snorm/s2] & [z/s3]
        #   - in Cartesian : vort_x = ddy_w - ddz_v
        #   - in s1,s2,s3  : vort_s1 = dds2_u3 - dds3_u2
        #
        # option 2: rotate [ω_x,ω_y,ω_z] --> [ω_s1,ω_s2,ω_s3]
        # 
        # this has been tested, the results are visually, although not numerically, identical.
        # method 1 is probably a little bit more accurate, especially for x-planes, because 
        # one doesn't have to rely on the accuracy of a 1D gradient over a very short dimension i.e. ddx_w
        # which is in vort_y = ddz_u - ddx_w
        # 
        # vort_x = ddy_w - ddz_v
        # vort_y = ddz_u - ddx_w
        # vort_z = ddx_v - ddy_u
        #
        # --> vort_s1 = vort_tang = ntang[0]·vort_x + ntang[1]·vort_y + ...
        #                                                      ------      
        
        if verbose:
            progress_bar = tqdm(total=len(ctl), ncols=100, desc='calc vort_tang', leave=False, file=sys.stdout)
        
        #for ti in self.ti:
        for ctl_ in ctl:
            ct1, ct2 = ctl_
            ntc = ct2 - ct1
            
            # === read utang, unorm, w
            
            if (vort_tang_calc_method == 1):
                
                dset = self['data/utang']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        utang = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    utang = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read utang', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                dset = self['data/unorm']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        unorm = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    unorm = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read unorm', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                dset = self['data/w']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        w = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    w = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # ===
            
            if (vort_tang_calc_method == 2):
                
                dset = self['data/vort_x']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        vort_x = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    vort_x = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read vort_x', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                dset = self['data/vort_y']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        vort_y = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    vort_y = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read vort_y', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                dset = self['data/vort_z']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        vort_z = np.copy(dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2]).T
                else:
                    vort_z = dset[ct1:ct2,:,:,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * ntc * self.nx * self.ny * self.nz * 1 / 1024**3
                if verbose:
                    tqdm.write( even_print('read vort_z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # === calculate (1)
            
            if (vort_tang_calc_method == 1):
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## in Cartesian: vort_x = ddy_w - ddz_v
                ## in s1,s2,s3:  vort_s1 = dds2_u3 - dds3_u2
                dds2_w    = gradient(w,     snorm, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
                ddz_unorm = gradient(unorm, z1d,   axis=2, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
                
                vort_tang = dds2_w - ddz_unorm
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print('calc vort_tang','%0.3f [s]'%(t_delta,), s=True))
            
            # === calculate (2)
            
            if (vort_tang_calc_method == 2):
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## stack vorticities into vector
                vort = np.stack((vort_x,vort_y,vort_z), axis=-1)
                if (vort.ndim != 5):
                    raise ValueError
                
                ## rotate vorticity vector
                vort_tang = np.einsum('xyzi,xyzti->xyzt', vtang3d, vort)
                #vort_norm = np.einsum('xyzi,xyzti->xyzt', vnorm3d, vort)
                #vort_span = np.einsum('xyzi,xyzti->xyzt', vspan3d, vort)
                
                ## this fails, but understandably so
                ## rtol will never match well because the variable is centered (logarithmically) around 0
                ## atol does better but high spatial gradients make it tough to get a perfect match
                ## 
                # np.testing.assert_allclose(vort_tang, vort_tang_bak, atol=1e-3)
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print('calc vort_tang','%0.3f [s]'%(t_delta,), s=True))
            
            # === write
            
            dset = self['data/vort_tang']
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    dset[ct1:ct2,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = vort_tang[xA:xB,yA:yB,zA:zB].T
            else:
                dset[ct1:ct2,:,:,:] = vort_tang.T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = ntc * 4 * self.nx * self.ny * self.nz / 1024**3
            
            if verbose: tqdm.write(even_print(f'write vort_tang','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            # dset = self['data/vort_tang_2']
            # if self.usingmpi: self.comm.Barrier()
            # t_start = timeit.default_timer()
            # if self.usingmpi:
            #     with dset.collective:
            #         dset[ct1:ct2,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = vort_tang_2[xA:xB,yA:yB,zA:zB].T
            # else:
            #     dset[ct1:ct2,:,:,:] = vort_tang_2.T
            # if self.usingmpi: self.comm.Barrier()
            # t_delta = timeit.default_timer() - t_start
            # data_gb = 4 * self.nx * self.ny * self.nz / 1024**3
            # 
            # if verbose: tqdm.write(even_print(f'write vort_tang_2','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_vorticity_tangnorm() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_triple_products(self, **kwargs):
        '''
        calculate triple products e.g. <u'v'v'>
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.calc_triple_products()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        #fn_cgd_prime = kwargs.get('fn_cgd_prime',None)
        #fn_cgd_mean  = kwargs.get('fn_cgd_mean',None)
        fn_out = kwargs.get('fn_out',None)
        
        force        = kwargs.get('force',False)
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing prime file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        ## assert that a CGD with (fsubtype=='prime') was opened
        if (self.fsubtype!='prime'):
            raise ValueError("fsubtype!='prime'")
        
        ## for now only distribute data in [y] --> allows [x,t] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        # === distribute 4D data over ranks --> for now only in [y]
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        nxr = self.nx
        nzr = self.nz
        
        # ===
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        t_avg_start  = self.t[ti_for_avg[0]]
        t_avg_end    = self.t[ti_for_avg[-1]]
        duration_avg = t_avg_end - t_avg_start
        
        #if not isinstance(ct, (int,np.int32,np.int64)):
        if not isinstance(ct, int):
            raise ValueError
        if (ct<1):
            raise ValueError
        
        ## [t] sub chunk range
        ctl_ = np.array_split( ti_for_avg, min(ct,nt_avg) )
        ctl = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## check that no sub ranges are <=1
        for a_ in [ ctl_[1]-ctl_[0] for ctl_ in ctl ]:
            if (a_ <= 1):
                raise ValueError('at least one rank sub range has zero length')
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        ## 'triple products' file name (for writing) --> .dat
        if (fn_out is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_dat_base = fname_root+'_triple_products.dat'
            #fn_out = os.path.join(fname_path, fname_dat_base)
            fn_out = str(PurePosixPath(fname_path, fname_dat_base))
            #fn_out = Path(fname_path, fname_dat_base)
        
        if verbose: even_print('fn_cgd', self.fname   )
        if verbose: even_print('fn out', fn_out )
        #if verbose: even_print('fn_cgd_prime' , fn_cgd_prime )
        #if verbose: even_print('do Favre avg' , str(favre)   )
        #if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        # ===
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            #if verbose: print(f'data_dim --> {dsn}')
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## add 'snorm' (dimensional)
        snorm = np.copy( self['dims/snorm'][()] )
        snorm *= self.lchar
        data['snorm'] = snorm
        
        ## add avg 'stang' (dimensional)
        stang = np.copy( self['dims/stang'][()] )
        stang *= self.lchar
        #data['stang'] = stang
        ns = stang.shape[0]
        if not np.isclose(stang[ns//2], np.mean(stang), rtol=1e-7):
            raise ValueError
        data['stang'] = np.array( [stang[ns//2],] , dtype=np.float64 )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        u1_99    = float( self['data_dim/u1_99'][()]    )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        ## the list of product combinations : [A,B,C,<name>]
        prod_combis = [
                    [ 'utangI' , 'utangI' , 'utangI' , 'utIutIutI' ],
                    [ 'unormI' , 'unormI' , 'unormI' , 'unIunIunI' ],
                    [ 'wI'     , 'wI'     , 'wI'     , 'wIwIwI'    ],
                    [ 'utangI' , 'utangI' , 'unormI' , 'utIutIunI' ],
                    [ 'utangI' , 'utangI' , 'wI'     , 'utIutIwI'  ],
                    [ 'utangI' , 'unormI' , 'unormI' , 'utIunIunI' ],
                    [ 'utangI' , 'wI'     , 'wI'     , 'utIwIwI'   ],
                    ]
        
        ## confirm that the scalars exist in the file
        for combi in prod_combis:
            for scalar in combi[:3]:
                if (f'data/{scalar}' not in self):
                    raise ValueError
        
        ## accumulator array for local rank --> initialize
        product_names = [ combi[-1] for combi in prod_combis ]
        data_sum = np.zeros(shape=(nxr,nyr,nzr,1), dtype={'names':product_names, 'formats':[np.float64 for _ in product_names]})
        
        # === main loop
        
        scalar = None ## clearing to be sure
        
        if verbose:
            progress_bar = tqdm(total=ct, ncols=100, desc='triple products', leave=False, file=sys.stdout)
        
        ct_counter=0
        for ctl_ in ctl: ## iterating through time chunks
            ct_counter += 1
            ct1, ct2 = ctl_
            ntc = ct2 - ct1
            
            if (ct>1):
                if verbose:
                    mesg = f'[t] sub chunk {ct_counter:d}/{ct:d}'
                    tqdm.write( mesg )
                    tqdm.write( '-'*len(mesg) )
            
            for prod_combi in prod_combis: ## iterate through list of product combinations (e.g. <u'u'v'>)
                
                compA = prod_combi[0]
                compB = prod_combi[1]
                compC = prod_combi[2]
                
                scalar = prod_combi[-1]
                
                # === collective read --> this could be optimized to prevent reading same data twice
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi:
                    dset = self[f'data/{compA}']
                    with dset.collective:
                        data_compA = np.copy( dset[ct1:ct2,:,ry1:ry2,:].T ).astype(np.float64)
                    dset = self[f'data/{compB}']
                    with dset.collective:
                        data_compB = np.copy( dset[ct1:ct2,:,ry1:ry2,:].T ).astype(np.float64)
                    dset = self[f'data/{compC}']
                    with dset.collective:
                        data_compC = np.copy( dset[ct1:ct2,:,ry1:ry2,:].T ).astype(np.float64)
                else:
                    dset       = self[f'data/{compA}']
                    data_compA = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    dset       = self[f'data/{compB}']
                    data_compB = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    dset       = self[f'data/{compC}']
                    data_compC = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                dtype = data_compA.dtype
                float_bytes = dtype.itemsize
                data_gb = 3 * float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                
                if verbose:
                    txt = even_print(f'read: {prod_combi[0]},{prod_combi[1]},{prod_combi[2]}', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                    tqdm.write(txt)
                
                t_read       += t_delta
                data_gb_read += data_gb
                
                # ===
                
                ## 4D triple product --> also re-dimensionalize!
                product_ = np.copy( data_compA * data_compB * data_compC   *   self.U_inf**3 )
                
                ## 3D [t] sum to accumulator
                data_sum[scalar] += np.sum(product_, axis=-1, dtype=np.float64, keepdims=True)
                
                #if self.usingmpi: self.comm.Barrier()
            
            if verbose: progress_bar.update()
        #if verbose: tqdm.write(72*'-')
        if verbose: progress_bar.close()
        
        # ==============================================================
        # multiply accumulators by (1/n)
        # ==============================================================
        
        for scalar in product_names:
            data_sum[scalar] *= (1/nt_avg)
        
        # ==============================================================
        # 'data_sum' now contains averages, not sums!
        # ==============================================================
        
        ## perform avg in [x,z,(t)] --> leave [y]
        data_sum_ = np.zeros(shape=(nyr,), dtype={'names':product_names, 'formats':[np.float64 for _ in product_names]})
        for scalar in product_names:
            data_sum_[scalar] = np.mean( data_sum[scalar], axis=(0,2,3), dtype=np.float64 )
        data_sum = np.copy( data_sum_ )
        
        if self.usingmpi: self.comm.Barrier()
        
        G = self.comm.gather([self.rank, np.copy(data_sum) ], root=0)
        G = self.comm.bcast(G, root=0)
        
        data_sum = np.zeros(shape=(self.ny,), dtype={'names':product_names, 'formats':[np.float64 for _ in product_names]})
        
        for ri in range(self.n_ranks):
            j = ri
            for GG in G:
                if (GG[0]==ri):
                    for tag in product_names:
                        data_sum[tag][ryl[j][0]:ryl[j][1]] = np.copy( GG[1][tag] )
                else:
                    pass
        #if verbose: print(72*'-')
        
        ## write into data dict for .dat output
        for scalar in product_names:
            data[f'data/{scalar}'] = data_sum[scalar]
        
        if self.usingmpi: self.comm.Barrier()
        
        # === save results
        
        if (self.rank==0):
            with open(fn_out,'wb') as f:
                pickle.dump(data, f, protocol=4)
            #print('--w-> %s : %0.2f [MB]'%(fn_out,os.path.getsize(fn_out)/1024**2))
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if os.path.isfile(fn_out):
            if verbose: even_print(fn_out, '%0.2f [KB]'%(os.path.getsize(fn_out)/1024**1))
        if (t_read>0.):
            if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        #if (t_write>0.):
        #    if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_triple_products() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # ===
    
    def calc_turb_spectrum_span_xpln(self, **kwargs):
        '''
        calculate FFT in [z] (wavenumber) at every [x,y,t], avg in [x,t]
        - designed for analyzing unsteady, thin planes in [x]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.calc_turb_spectrum_span_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_dat_fft  = kwargs.get('fn_dat_fft',None)
        fn_cgd_mean = kwargs.get('fn_cgd_mean',None)
        
        ## assert that a CGD with (fsubtype=='prime') was opened
        if (self.fsubtype!='prime'):
            raise ValueError("fsubtype!='prime'")
        
        ## for now only distribute data in [y] --> allows [x,t] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # === distribute 4D data over ranks
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # # === mean file name (for reading)
        # if (fn_cgd_mean is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     if ('_prime' in fname_root):
        #         fname_root = fname_root.replace('_prime','')
        #     fname_mean_h5_base = fname_root+'_mean.h5'
        #     #fn_cgd_mean = os.path.join(fname_path, fname_mean_h5_base)
        #     fn_cgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
        #     #fn_cgd_mean = Path(fname_path, fname_mean_h5_base)
        
        # === fft file name (for writing) : dat
        if (fn_dat_fft is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_dat_base = fname_root+'_turb_spec_span.dat'
            fn_dat_fft = str(PurePosixPath(fname_path, fname_fft_dat_base))
        
        if verbose: even_print('fn_cgd_prime'    , self.fname   )
        #if verbose: even_print('fn_cgd_mean'     , fn_cgd_mean  )
        if verbose: even_print('fn_dat_fft'      , fn_dat_fft   )
        if verbose: print(72*'-')
        
        #if not os.path.isfile(fn_cgd_mean):
        #    raise FileNotFoundError('%s not found'%fn_cgd_mean)
        
        #with cgd(fn_cgd_mean, 'r', driver=self.driver, comm=self.comm) as hf_mean:
        #    if (self.fsubtype!='mean'):
        #        raise ValueError("fsubtype!='mean'")
        
        # ===
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho   = np.copy( self['data_dim/rho'][()]   )
        utang = np.copy( self['data_dim/utang'][()] )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        u1_99    = float( self['data_dim/u1_99'][()]    )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## these are recalculated and checked in next step
        z1d_ = np.copy( self['data_dim/z1d'][()] )
        dz0_ = np.copy( self['data_dim/dz0'][()] )
        dt_  = np.copy( self['data_dim/dt'][()]  )
        
        # ===
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 3D coordinate arrays, then dimensionalize [m]
        ## every ranks gets full grid
        x = np.copy( self['dims/x'][()].T * self.lchar )
        y = np.copy( self['dims/y'][()].T * self.lchar )
        z = np.copy( self['dims/z'][()].T * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        ## read in 1D coordinate arrays, then dimensionalize [m]
        stang_ = np.copy(self['dims/stang'])
        stang  = np.copy( stang_ * self.lchar ) ## dimensional [m]
        data['stang'] = stang
        
        snorm_ = np.copy(self['dims/snorm'])
        snorm  = np.copy( snorm_ * self.lchar ) ## dimensional [m]
        data['snorm'] = snorm
        
        ## assert [z] is same over all [x,y]
        if (z.ndim!=3):
            raise ValueError
        z1d = np.copy(z[0,0,:])
        for i in range(nx):
            for j in range(ny):
                np.testing.assert_allclose(z1d, z[i,j,:], rtol=1e-14, atol=1e-14)
        
        ## assert [x,y] is same over all [z]
        x2d  = np.copy(x[:,:,0])
        y2d  = np.copy(y[:,:,0])
        xy2d = np.stack((x2d,y2d), axis=-1)
        for k in range(nz):
            x2d_  = np.copy(x[:,:,k])
            y2d_  = np.copy(y[:,:,k])
            xy2d_ = np.stack((x2d_,y2d_), axis=-1)
            np.testing.assert_allclose(xy2d, xy2d_, rtol=1e-14, atol=1e-14)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z1d.max() - z1d.min()
        
        #data['x'] = x
        #data['y'] = y
        #data['z'] = z
        data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        # === report
        if verbose:
            #even_print('nx'     , '%i'        %nx     )
            #even_print('ny'     , '%i'        %ny     )
            #even_print('nz'     , '%i'        %nz     )
            #even_print('nt'     , '%i'        %nt     )
            #even_print('dt'     , '%0.5e [s]' %dt     )
            #even_print('t_meas' , '%0.5e [s]' %t_meas )
            even_print('dz0'    , '%0.5e [m]' %dz0     )
            even_print('zrange' , '%0.5e [m]' %zrange  )
            print(72*'-')
        
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+' , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'Δt+' , '%0.3f'%(dt/sc_t_in) )
            print(72*'-')
        
        ## get spanwise wavenumber [kz] vector
        kz_full = sp.fft.fftfreq(n=nz, d=dz0) * ( 2 * np.pi )
        kzp     = np.where(kz_full>0)
        kz      = np.copy(kz_full[kzp])
        dkz     = kz[1]-kz[0]
        nkz     = kz.size
        
        data['kz']  = kz
        data['dkz'] = dkz
        data['nkz'] = nkz
        
        if verbose:
            even_print('kz min','%0.1f [1/m]'%kz.min())
            even_print('kz max','%0.1f [1/m]'%kz.max())
            even_print('dkz','%0.1f [1/m]'%dkz)
            even_print('nkz','%i'%nkz)
            
            kz_inner = np.copy( kz * sc_l_in  )
            kz_outer = np.copy( kz * sc_l_out )
            
            even_print('(kz·δ_ν) min' , '%0.5e [-]'%kz_inner.min())
            even_print('(kz·δ_ν) max' , '%0.5f [-]'%kz_inner.max())
            even_print('(kz·δ99) min' , '%0.5f [-]'%kz_outer.min())
            even_print('(kz·δ99) max' , '%0.5f [-]'%kz_outer.max())
            
            print(72*'-')
        
        # === read in data (prime) --> still dimless (char)
        
        #scalars = [ 'uI','vI','wI' , 'rho','uII','vII','wII' ]
        scalars = [ 'uI','vI','wI', 'utangI', 'unormI' ]
        scalars_dtypes = [self.scalars_dtypes_dict[s] for s in scalars]
        
        ## [var1, var2, density_scaling]
        fft_combis = [
                     # [ 'uI'  , 'uI'  , False ],
                     # [ 'vI'  , 'vI'  , False ],
                     # [ 'wI'  , 'wI'  , False ],
                     # [ 'uI'  , 'vI'  , False ],
                     # [ 'uII' , 'uII' , True  ],
                     # [ 'vII' , 'vII' , True  ],
                     # [ 'wII' , 'wII' , True  ],
                     # [ 'uII' , 'vII' , True  ],
                     ##
                     [ 'utangI' , 'utangI' , False ],
                     [ 'unormI' , 'unormI' , False ],
                     [ 'wI'     , 'wI'     , False ],
                     [ 'utangI' , 'unormI' , False ],
                     [ 'utangI' , 'wI'     , False ],
                     ]
        
        scalars_dtypes = [ self.scalars_dtypes_dict[s] for s in scalars ]
        
        ## dtype of prime data (currently must be all same dtype)
        if np.all( [ (dtp==np.float32) for dtp in scalars_dtypes ] ):
            dtype_primes = np.float32
        elif np.all( [ (dtp==np.float64) for dtp in scalars_dtypes ] ):
            dtype_primes = np.float64
        else:
            raise NotImplementedError
        
        ## 5D [scalar][x,y,z,t] structured array
        data_prime = np.zeros(shape=(self.nx, nyr, self.nz, self.nt), dtype={'names':scalars, 'formats':scalars_dtypes})
        
        for scalar in scalars:
            
            dset = self[f'data/{scalar}']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            
            self.comm.Barrier()
            t_start = timeit.default_timer()
            with dset.collective:
                data_prime[scalar] = np.copy( dset[:,:,ry1:ry2,:].T )
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            
            data_gb = float_bytes * self.nx * self.ny * self.nz * self.nt / 1024**3
            
            if verbose:
                even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        # === redimensionalize prime data
        
        for var in data_prime.dtype.names:
            if var in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII', 'utangI','unormI', 'utangII','unormII']:
                data_prime[var] *= U_inf
            elif var in ['r_uII','r_vII','r_wII']:
                data_prime[var] *= (U_inf*rho_inf)
            elif var in ['T','TI','TII']:
                data_prime[var] *= T_inf
            elif var in ['r_TII']:
                data_prime[var] *= (T_inf*rho_inf)
            elif var in ['rho','rhoI']:
                data_prime[var] *= rho_inf
            elif var in ['p','pI','pII']:
                data_prime[var] *= (rho_inf * U_inf**2)
            else:
                raise ValueError('condition needed for redimensionalizing \'%s\''%var)
        
        ## force the ∫PSD == (co)variance
        ## this usually represents about a 1-2% power correction which comes about due to 
        ##   non-stationarity of windowed data
        normalize_psd_by_cov = False
        
        ## initialize buffers
        Euu_scalars         = [ '%s%s'%(cc[0],cc[1]) for cc in fft_combis ]
        Euu_scalars_dtypes  = [ dtype_primes for s in Euu_scalars ]
        Euu                 = np.zeros(shape=(self.nx, nyr, self.nt, nkz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg            = np.zeros(shape=(self.nx, nyr, self.nt)      , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        if normalize_psd_by_cov:
            energy_norm_fac_arr = np.zeros(shape=(self.nx, nyr, self.nt) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes}) ## just for monitoring
        
        ## check memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose:
            even_print('mem total',     '%0.1f [GB]'%(mem_total_gb,))
            even_print('mem available', '%0.1f [GB] / %0.1f[%%]'%(mem_avail_gb,(100*mem_avail_gb/mem_total_gb)))
            even_print('mem free',      '%0.1f [GB] / %0.1f[%%]'%(mem_free_gb,(100*mem_free_gb/mem_total_gb)))
        
        ## for spanwise mean, no overlapping windows are applied, so win_len = [z] vec length
        win_len = nz
        
        ## the window function
        window_type = 'tukey'
        if (window_type=='tukey'):
            window = sp.signal.windows.tukey(win_len, alpha=0.1)
        elif (window_type is None):
            window = np.ones(win_len, dtype=np.float64)
        if verbose:
            even_print('window type', '\'%s\''%str(window_type))
        
        ## sum of sqrt of window: needed for power normalization
        sum_sqrt_win = np.sum(np.sqrt(window))
        
        # === main loop
        
        self.comm.Barrier()
        if verbose: progress_bar = tqdm(total=self.nx*nyr*self.nt, ncols=100, desc='fft', leave=False)
        for xi in range(self.nx):
            for yi in range(nyr):
                for ti in range(self.nt):
                    for cci,cc in enumerate(fft_combis):
                        
                        tag = Euu_scalars[cci]
                        ccL,ccR,density_scaling = cc
                        
                        uL = np.copy( data_prime[ccL][xi,yi,:,ti] )
                        uR = np.copy( data_prime[ccR][xi,yi,:,ti] )
                        
                        if ('rho' in data_prime.dtype.names):
                            rho     = np.copy( data_prime['rho'][xi,yi,:,ti] )
                            rho_avg = np.mean( rho, dtype=np.float64 )
                        else:
                            rho     = None
                            rho_avg = None
                        
                        # ===
                        
                        if density_scaling:
                            uIuI_avg_ijk = np.mean(uL*uR*rho, dtype=np.float64) / rho_avg
                        else:
                            uIuI_avg_ijk = np.mean(uL*uR,     dtype=np.float64)
                        
                        uIuI_avg[tag][xi,yi,ti] = uIuI_avg_ijk
                        
                        if density_scaling:
                            ui = np.copy( uL * rho )
                            uj = np.copy( uR * rho )
                        else:
                            ui = np.copy( uL )
                            uj = np.copy( uR )
                        
                        n     = ui.size
                        #A_ui = sp.fft.fft(ui)[fp] / n
                        #A_uj = sp.fft.fft(uj)[fp] / n
                        ui   *= window
                        uj   *= window
                        #ui  -= np.mean(ui) ## de-trend
                        #uj  -= np.mean(uj)
                        A_ui    = sp.fft.fft(ui)[kzp] / sum_sqrt_win
                        A_uj    = sp.fft.fft(uj)[kzp] / sum_sqrt_win
                        Euu_ijk = 2 * np.real(A_ui*np.conj(A_uj)) / dkz
                        
                        ## divide off mean mass density
                        if density_scaling:
                            Euu_ijk /= rho_avg**2
                        
                        ## normalize such that ∫PSD=(co)variance
                        if normalize_psd_by_cov:
                            if (uIuI_avg_ijk!=0.):
                                energy_norm_fac = np.sum(dkz*Euu_ijk) / uIuI_avg_ijk
                            else:
                                energy_norm_fac = 1.
                            Euu_ijk /= energy_norm_fac
                            energy_norm_fac_arr[tag][xi,yi,ti] = energy_norm_fac
                        
                        ## write
                        Euu[tag][xi,yi,ti,:] = Euu_ijk
                    
                    if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        ## report energy normalization factors --> tmp,only rank 0 currently!
        if normalize_psd_by_cov:
            if verbose: print(72*'-')
            for tag in Euu_scalars:
                energy_norm_fac_min = energy_norm_fac_arr[tag].min()
                energy_norm_fac_max = energy_norm_fac_arr[tag].max()
                energy_norm_fac_avg = np.mean(energy_norm_fac_arr[tag], axis=(0,1,2), dtype=np.float64)
                if verbose:
                    even_print('energy norm min/max/avg : %s'%tag, '%0.4f / %0.4f / %0.4f'%(energy_norm_fac_min,energy_norm_fac_max,energy_norm_fac_avg))
            energy_norm_fac_arr = None ; del energy_norm_fac_arr
        
        # === average in [x,t] --> leave [y,kz]
        
        Euu_      = np.zeros(shape=(nyr,nkz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg_ = np.zeros(shape=(nyr,)    , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        self.comm.Barrier()
        
        for tag in Euu_scalars:
            Euu_[tag]      = np.mean( Euu[tag]      , axis=(0,2) , dtype=np.float64) #.astype(np.float32) ## avg in [x,t] --> leave [y,kz]
            uIuI_avg_[tag] = np.mean( uIuI_avg[tag] , axis=(0,2) , dtype=np.float64) #.astype(np.float32) ## avg in [x,t] --> leave [y]
        Euu      = np.copy( Euu_ )
        uIuI_avg = np.copy( uIuI_avg_ )
        self.comm.Barrier()
        
        # === gather all results
        # --> arrays are very small at this point, just do 'lazy' gather/bcast, dont worry about buffers etc
        
        G = self.comm.gather([self.rank, 
                              Euu, uIuI_avg ], root=0)
        G = self.comm.bcast(G, root=0)
        
        Euu      = np.zeros( (ny,nkz) , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        uIuI_avg = np.zeros( (ny,)    , dtype={'names':Euu_scalars, 'formats':Euu_scalars_dtypes})
        
        for ri in range(self.n_ranks):
            j = ri
            for GG in G:
                if (GG[0]==ri):
                    for tag in Euu_scalars:
                        Euu[tag][ryl[j][0]:ryl[j][1],:]    = GG[1][tag]
                        uIuI_avg[tag][ryl[j][0]:ryl[j][1]] = GG[2][tag]
                else:
                    pass
        if verbose: print(72*'-')
        
        # === save results
        if (self.rank==0):
            
            data['Euu']      = Euu
            data['uIuI_avg'] = uIuI_avg
            
            with open(fn_dat_fft,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_fft,os.path.getsize(fn_dat_fft)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_turb_spectrum_span_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_turb_spectrum_time_xpln(self, **kwargs):
        raise NotImplementedError
        return
    
    def calc_ccor_span(self, **kwargs):
        '''
        calculate cross-correlation in [z] and avg in [x,t] --> leave [y,Δz]
        - designed for analyzing unsteady, thin planes in [x]
        '''
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'cgd.calc_ccor_span()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_dat_ccor_span = kwargs.get('fn_dat_ccor_span',None)
        #fn_dat_mean_dim  = kwargs.get('fn_dat_mean_dim',None)
        
        ## assert that a CGD with (fsubtype=='prime') was opened
        if (self.fsubtype!='prime'):
            raise ValueError("fsubtype!='prime'")
        
        ## for now only distribute data in [y]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # ===
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # # === mean dimensional file name (for reading) : .dat
        # if (fn_dat_mean_dim is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_dat_mean_base = fname_root+'_mean_dim.dat'
        #     fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
        
        # === cross-correlation file name (for writing) : dat
        if (fn_dat_ccor_span is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_ccor_span_dat_base = fname_root+'_ccor_span.dat'
            fn_dat_ccor_span = str(PurePosixPath(fname_path, fname_ccor_span_dat_base))
        
        if verbose: even_print('fn_cgd_prime'     , self.fname       )
        #if verbose: even_print('fn_dat_mean_dim'  , fn_dat_mean_dim  )
        if verbose: even_print('fn_dat_ccor_span' , fn_dat_ccor_span )
        if verbose: print(72*'-')
        
        # if not os.path.isfile(fn_dat_mean_dim):
        #     raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        # # === read in data (mean dim) --> every rank gets full [x,z]
        # with open(fn_dat_mean_dim,'rb') as f:
        #     data_mean_dim = pickle.load(f)
        # fmd = type('foo', (object,), data_mean_dim)
        
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        if ('data_dim' not in self):
            raise ValueError('group data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in self['data_dim'].keys():
            d_ = np.copy( self[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        rho_avg = np.copy( self['data_dim/rho'][()] )
        #u_avg   = np.copy( self['data_dim/u'][()]   )
        utang   = np.copy( self['data_dim/utang'][()] )
        
        ## 0D
        u_tau    = float( self['data_dim/u_tau'][()]    )
        nu_wall  = float( self['data_dim/nu_wall'][()]  )
        rho_wall = float( self['data_dim/rho_wall'][()] )
        d99      = float( self['data_dim/d99'][()]      )
        #u_99     = float( self['data_dim/u_99'][()]     )
        u1_99     = float( self['data_dim/u1_99'][()]     )
        Re_tau   = float( self['data_dim/Re_tau'][()]   )
        Re_theta = float( self['data_dim/Re_theta'][()] )
        sc_u_in  = float( self['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( self['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( self['data_dim/sc_t_in'][()]  )
        sc_u_out = float( self['data_dim/sc_u_out'][()] )
        sc_l_out = float( self['data_dim/sc_l_out'][()] )
        sc_t_out = float( self['data_dim/sc_t_out'][()] )
        
        ## these are recalculated and checked in next step
        z1d_ = np.copy( self['data_dim/z1d'][()] )
        dz0_ = np.copy( self['data_dim/dz0'][()] )
        dt_  = np.copy( self['data_dim/dt'][()]  )
        
        # ===
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 3D coordinate arrays, then dimensionalize [m]
        ## every ranks gets full grid
        x = np.copy( self['dims/x'][()].T * self.lchar )
        y = np.copy( self['dims/y'][()].T * self.lchar )
        z = np.copy( self['dims/z'][()].T * self.lchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        ## read in 1D coordinate arrays, then dimensionalize [m]
        stang_ = np.copy(self['dims/stang'])
        stang  = np.copy( stang_ * self.lchar ) ## dimensional [m]
        data['stang'] = stang
        
        snorm_ = np.copy(self['dims/snorm'])
        snorm  = np.copy( snorm_ * self.lchar ) ## dimensional [m]
        data['snorm'] = snorm
        
        ## assert [z] is same over all [x,y]
        if (z.ndim!=3):
            raise ValueError
        z1d = np.copy(z[0,0,:])
        for i in range(nx):
            for j in range(ny):
                np.testing.assert_allclose(z1d, z[i,j,:], rtol=1e-14, atol=1e-14)
        
        ## assert [x,y] is same over all [z]
        x2d  = np.copy(x[:,:,0])
        y2d  = np.copy(y[:,:,0])
        xy2d = np.stack((x2d,y2d), axis=-1)
        for k in range(nz):
            x2d_  = np.copy(x[:,:,k])
            y2d_  = np.copy(y[:,:,k])
            xy2d_ = np.stack((x2d_,y2d_), axis=-1)
            np.testing.assert_allclose(xy2d, xy2d_, rtol=1e-14, atol=1e-14)
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z1d)[0]
        if not np.all(np.isclose(np.diff(z1d), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        ## check against values in 'data_dim' (imported above)
        np.testing.assert_allclose(dt  , dt_  , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(dz0 , dz0_ , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(z1d , z1d_ , rtol=1e-14, atol=1e-14)
        
        zrange = z1d.max() - z1d.min()
        
        #data['x'] = x
        #data['y'] = y
        #data['z'] = z
        data['z1d'] = z1d
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            #even_print('nx'     , '%i'        %nx     )
            #even_print('ny'     , '%i'        %ny     )
            #even_print('nz'     , '%i'        %nz     )
            #even_print('nt'     , '%i'        %nt     )
            #even_print('dt'     , '%0.5e [s]' % dt      )
            #even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        t_eddy = t_meas / ( d99 / u_tau )
        
        if verbose:
            even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
            #even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
            even_print('t_meas/(δ99/u1_99)'          , '%0.2f'%(t_meas/(d99/u1_99)))
            #even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
            even_print('t_meas/(20·δ99/u1_99)'       , '%0.2f'%(t_meas/(20*d99/u1_99)))
        
        ## get lags
        lags,_  = ccor( np.ones(nz,dtype=np.float32) , np.ones(nz,dtype=np.float32), get_lags=True )
        n_lags_ = nz*2-1
        n_lags  = lags.shape[0]
        if (n_lags!=n_lags_):
            raise AssertionError('possible problem with lags calc --> check!')
        
        if verbose:
            even_print('n lags (Δz)' , '%i'%(n_lags,))
        
        ## [var1, var2, density_scaling]
        R_combis = [
                   # [ 'uI'  , 'uI'  , False ],
                   # [ 'vI'  , 'vI'  , False ],
                   # [ 'wI'  , 'wI'  , False ],
                   # [ 'uI'  , 'vI'  , False ],
                   # [ 'uI'  , 'TI'  , False ],
                   # [ 'vI'  , 'TI'  , False ],
                   # [ 'TI'  , 'TI'  , False ],
                   # [ 'uII' , 'uII' , True  ],
                   # [ 'vII' , 'vII' , True  ],
                   # [ 'wII' , 'wII' , True  ],
                   # [ 'uII' , 'vII' , True  ],
                   # [ 'uII' , 'TII' , True  ],
                   # [ 'vII' , 'TII' , True  ],
                   # [ 'TII' , 'TII' , True  ],
                   ##
                   [ 'utangI' , 'utangI' , False ],
                   [ 'unormI' , 'unormI' , False ],
                   [ 'wI'     , 'wI'     , False ],
                   [ 'utangI' , 'unormI' , False ],
                   [ 'utangI' , 'wI'     , False ],
                   ]
        
        if verbose:
            even_print('n ccor (Δz) scalar combinations' , '%i'%(len(R_combis),))
            print(72*'-')
        
        ## decide if density will be needed
        read_density = False
        for cc in R_combis:
            scalar_L, scalar_R, density_scaling = cc
            if density_scaling:
                read_density = True
                break
        
        if verbose:
            even_print('read ρ', str(read_density))
        
        ## confirm 'rho' is not in locals
        if read_density and ('rho' in locals()):
            raise ValueError('rho alread in locals... check')
        
        if read_density:
            
            ## buffer for rho (NOT rhoI) --> this is read from 'prime' file
            rho = np.zeros(shape=(nx, nyr, nz, nt), dtype=np.float32)
            
            dset = self['data/rho']
            self.comm.Barrier()
            t_start = timeit.default_timer()
            with dset.collective:
                rho[:,:,:,:] = dset[:,:,ry1:ry2,:].T
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
            if verbose:
                even_print( 'read: rho','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
            
            ## re-dimensionalize rho
            rho *= rho_inf
        
        scalars_R        = [ '%s%s'%(cc[0],cc[1]) for cc in R_combis ]
        scalars_dtypes_R = [ np.float32 for s in scalars_R ]
        
        ## averaged cross-correlation data buffer
        ## 3D [scalar][y,Δz] structured array
        data_R_avg = np.zeros(shape=(nyr, n_lags), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
        
        ## main loop --> cross-correlation in [Δz] at every [x,y,t]
        if verbose:
            progress_bar = tqdm(total=len(R_combis)*nx*nyr*nt, ncols=100, desc='ccor_span()', leave=False, file=sys.stdout)
        
        for cci,cc in enumerate(R_combis):
            
            scalar_L, scalar_R, density_scaling = cc
            tag = '%s%s'%(scalar_L, scalar_R)
            
            ## check if autocorrelation
            if (scalar_L==scalar_R):
                scalars = [ scalar_L ]
            else:
                scalars = [ scalar_L, scalar_R ]
            
            scalars_dtypes = [self.scalars_dtypes_dict[s] for s in scalars]
            
            ## prime data buffer
            ## 5D [scalar][x,y,z,t] structured array
            data_prime = np.zeros(shape=(nx, nyr, nz, nt), dtype={'names':scalars, 'formats':scalars_dtypes})
            
            ## cross-correlation data buffer
            ## 5D [scalar][x,y,z,t] structured array
            #scalars_R        = [ tag ]
            #scalars_dtypes_R = [ np.float32 for s in scalars_R ]
            #data_R           = np.zeros(shape=(nx, nyr, n_lags, nt), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
            
            ## cross-correlation data buffer
            ## 4D numpy array
            data_R = np.zeros(shape=(nx, nyr, n_lags, nt), dtype=np.float32)
            
            ## read prime data
            for scalar in scalars:
                dset = self['data/%s'%scalar]
                self.comm.Barrier()
                t_start = timeit.default_timer()
                with dset.collective:
                    data_prime[scalar][:,:,:,:] = np.copy( dset[:,:,ry1:ry2,:].T )
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = 4 * self.nx * self.ny * self.nz * self.nt / 1024**3
                if verbose:
                    tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
            
            ## redimensionalize prime data
            for var in data_prime.dtype.names:
                if var in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII', 'utangI','unormI', 'utangII','unormII']:
                    data_prime[var] *= U_inf
                elif var in ['r_uII','r_vII','r_wII']:
                    data_prime[var] *= (U_inf*rho_inf)
                elif var in ['T','TI','TII']:
                    data_prime[var] *= T_inf
                elif var in ['r_TII']:
                    data_prime[var] *= (T_inf*rho_inf)
                elif var in ['rho','rhoI']:
                    data_prime[var] *= rho_inf
                elif var in ['p','pI','pII']:
                    data_prime[var] *= (rho_inf * U_inf**2)
                else:
                    raise ValueError('condition needed for redimensionalizing \'%s\''%var)
            
            # ===
            
            ## print names of components being cross-correlated
            if verbose:
                if density_scaling:
                    fancy_tag = '<ρ·%s,ρ·%s>'%(scalar_L,scalar_R)
                else:
                    fancy_tag = '<%s,%s>'%(scalar_L,scalar_R)
                tqdm.write(even_print('running Δz cross-correlation calc', fancy_tag, s=True))
            
            for xi in range(nx):
                for yi in range(nyr):
                    for ti in range(nt):
                        
                        uL = np.copy( data_prime[scalar_L][xi,yi,:,ti]   )
                        uR = np.copy( data_prime[scalar_R][xi,yi,:,ti]   )
                        
                        if density_scaling:
                            rho1d = np.copy( rho[xi,yi,:,ti] )
                        else:
                            rho1d = None
                        
                        if density_scaling:
                            data_R[xi,yi,:,ti] = ccor( rho1d*uL , rho1d*uR )
                        else:
                            data_R[xi,yi,:,ti] = ccor( uL , uR )
                        
                        if verbose: progress_bar.update()
            
            # ===
            
            ## average in [x,t] --> leave [y,lag] (where lag is Δz)
            data_R_avg[tag] = np.mean(data_R, axis=(0,3), dtype=np.float64).astype(np.float32)
            data_R = None; del data_R
            
            ## manually delete the prime data from memory
            data_prime = None; del data_prime
            self.comm.Barrier()
        
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        # === gather all (pre-averaged) results
        
        self.comm.Barrier()
        data_R_all = None
        if (self.rank==0):
            
            j=0
            data_R_all = np.zeros(shape=(ny,n_lags), dtype={'names':scalars_R, 'formats':scalars_dtypes_R})
            
            ## data this rank
            for scalar_R in scalars_R:
                data_R_all[scalar_R][ryl[j][0]:ryl[j][1],:] = data_R_avg[scalar_R]
        
        for scalar_R in scalars_R:
            for ri in range(1,self.n_ranks):
                j = ri
                self.comm.Barrier()
                if (self.rank==ri):
                    sendbuf = np.copy(data_R_avg[scalar_R])
                    self.comm.Send(sendbuf, dest=0, tag=ri)
                    #print('rank %i sending %s'%(ri,scalar_R))
                elif (self.rank==0):
                    #print('rank %i receiving %s'%(self.rank,scalar_R))
                    nyri = ryl[j][1] - ryl[j][0]
                    
                    recvbuf = np.zeros((nyri,n_lags), dtype=data_R_avg[scalar_R].dtype)
                    
                    #print('rank %i : recvbuf.shape=%s'%(rank,str(recvbuf.shape)))
                    self.comm.Recv(recvbuf, source=ri, tag=ri)
                    data_R_all[scalar_R][ryl[j][0]:ryl[j][1],:] = recvbuf
                else:
                    pass
        
        ## overwrite
        if (self.rank==0):
            R = np.copy(data_R_all)
        
        # === save results
        
        if (self.rank==0):
            
            data['R']    = R ## the main cross-correlation data array
            data['lags'] = lags
            
            with open(fn_dat_ccor_span,'wb') as f:
                pickle.dump(data, f, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_ccor_span,os.path.getsize(fn_dat_ccor_span)/1024**2))
        
        # ===
        
        self.comm.Barrier()
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.calc_ccor_span() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === polydata (turbx.spd() file)
    
    def export_polydata_wall(self, fn_spd=None, **kwargs):
        '''
        get 2D [x,z] wall quantities, export structured polydata (SPD) file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        force    = kwargs.get('force',False)
        
        acc = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(None,None,1)) ## the 'constraint' parameter for sizing h5 chunks (i,j,t)
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing SPD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        if verbose: print('\n'+'cgd.export_polydata_wall()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/utang' in self) and self.curvilinear:
            raise ValueError('data/utang not in hdf5')
        if not ('data/unorm' in self) and self.curvilinear:
            raise ValueError('data/unorm not in hdf5')
        if not ('dims/snorm' in self) and self.curvilinear:
            raise ValueError('dims/snorm not in hdf5')
        if not ('dims/stang' in self) and self.curvilinear:
            raise ValueError('dims/stang not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        #if (ry>self.ny):
        #    raise AssertionError('ry>self.ny')
        if (ry!=1):
            raise AssertionError('ry!=1')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## polydata file name (for writing)
        if (fn_spd is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_spd_h5_base = fname_root+'_polydata_wall.h5'
            #fn_spd = os.path.join(fname_path, fname_spd_h5_base)
            fn_spd = str(PurePosixPath(fname_path, fname_spd_h5_base))
            #fn_spd = Path(fname_path, fname_spd_h5_base)
        
        # === ranks
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            rx1 = 0
            rx2 = self.nx
            ry1 = 0
            ry2 = self.ny
            rz1 = 0
            rz2 = self.nz
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## [t] chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # ===
        
        ## only take first N wall-normal points
        nyr = min(20,self.ny)
        ry1, ry2 = 0, 0+nyr
        
        data_gb = nyr * (self.nx * self.nz) * 4 / 1024**3
        
        ## get [snorm] or [y]
        if ('dims/snorm' in self):
            dset = self['dims/snorm']
            snorm = np.copy( dset[ry1:ry2] )
        else:
            if self.rectilinear:
                dset = self['dims/y']
                snorm = np.copy( dset[0,ry1:ry2,0].T )
            else:
                raise ValueError
        
        ## 3D polydata grid coordinates : shape (nx,nz,3)
        xyz_wall = np.zeros((self.nx, self.nz, 3))
        xyz_wall[:,:,0] = np.copy(self['dims/x'][:,0,:]).T
        xyz_wall[:,:,1] = np.copy(self['dims/y'][:,0,:]).T
        xyz_wall[:,:,2] = np.copy(self['dims/z'][:,0,:]).T
        
        ## open & initialize polydata (SPD) file
        if verbose: even_print('surface polydata (spd) .h5', fn_spd)
        with spd(fn_spd, 'w',
                 force=force,
                 driver=self.driver,
                 comm=MPI.COMM_WORLD,
                 stripe_count=stripe_count,
                 stripe_size_mb=stripe_size_mb) as hfspd:
            
            ## add attributes from CGD to SPD
            header_attr_str_list = ['Ma','Re','Pr','kappa','R','p_inf','T_inf','S_Suth','mu_Suth_ref','T_Suth_ref']
            for key in header_attr_str_list: #for key in self.udef:
                hfspd.attrs[key] = self.udef[key]
            #for key in self.udef_deriv:
            #    hfspd.attrs[key] = self.udef_deriv[key]
            
            ## add time vector from CGD to SPD
            dsn = f'dims/t'
            if (dsn in hfspd):
                del hfspd[dsn]
            data = np.copy(self[dsn][()])
            ds = hfspd.create_dataset(dsn, data=data, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## add 3D polydata grid coordinates : shape (nx,nz,3)
            dsn = f'dims/xyz'
            if (dsn in hfspd):
                del hfspd[dsn]
            ds = hfspd.create_dataset(dsn, data=xyz_wall, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            hfspd.attrs['ni'] = self.nx
            hfspd.attrs['nj'] = self.nz
            hfspd.attrs['n_quads'] = ( self.nx - 1 ) * ( self.nz - 1 )
            hfspd.attrs['n_pts'] = self.nx * self.nz
            hfspd.attrs['nt'] = self.nt
            
            hfspd.get_header(verbose=verbose)
            
            ## add additional [dims/<>] dsets
            for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][()])
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            ## add additional [csys/<>] dsets
            for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][:,0,:]) ## take only vectors at wall [i,j==0,:]
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            if verbose: print(72*'-')
            
            ## initialize datasets
            dtype    = np.dtype(np.float32)
            itemsize = dtype.itemsize
            shape    = (self.nx,self.nz,self.nt)
            data_gb  = np.prod(shape) * itemsize / 1024**3
            
            chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=4)
            
            scalars_spd = [ 'tau_u1_s2', 'tau_u3_s2', 'T','rho','mu','nu','u_tau','p' ]
            
            for scalar in scalars_spd:
                
                if (f'data/{scalar}' in hfspd):
                    del hfspd[f'data/{scalar}']
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if verbose:
                    even_print(f'initializing data/{scalar}','%0.1f [GB]'%(data_gb,))
                
                dset = hfspd.create_dataset(f'data/{scalar}',
                                            shape=shape,
                                            dtype=dtype,
                                            chunks=chunks)
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]')
                
                chunk_kb_ = np.prod(dset.chunks)*itemsize / 1024. ## actual
                if verbose:
                    #even_print('chunk shape (t,z,x)','%s'%str(dset.chunks))
                    even_print('chunk shape (x,z,t)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === main loop
        
        ## re-dimensionalize snorm
        snorm *= self.lchar
        
        with spd(fn_spd, 'a', driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
            
            if verbose:
                progress_bar = tqdm(total=ct, ncols=100, desc='calc τ_w', leave=False, file=sys.stdout)
            
            for ctl_ in ctl:
                ct1, ct2 = ctl_
                ntc = ct2 - ct1
                
                if ('data/utang' in self):
                    dset = self['data/utang']
                else:
                    if self.rectilinear:
                        dset = self['data/u']
                    else:
                        raise ValueError
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        utang = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    utang = dset[ct1:ct2,:,ry1:ry2,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read utang', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                
                dset = self['data/w']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        w = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    w = dset[ct1:ct2,:,ry1:ry2,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                
                dset = self['data/rho']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        rho = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    rho = dset[ct1:ct2,:,ry1:ry2,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read rho', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                
                dset = self['data/T']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        T = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    T = dset[ct1:ct2,:,ry1:ry2,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read T', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                
                dset = self['data/p']
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        p = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                else:
                    p = dset[ct1:ct2,:,ry1:ry2,:].T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ntc * nyr * (self.nx * self.nz) * 4 / 1024**3
                if verbose:
                    tqdm.write( even_print('read p', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
                
                # === re-dimensionalize
                
                T     *= self.T_inf
                utang *= self.U_inf
                w     *= self.U_inf
                rho   *= self.rho_inf
                p     *= self.rho_inf * self.U_inf**2
                
                # ===
                
                # mu1 = (14.58e-7 * T**1.5) / ( T + 110.4 )
                # mu2 = self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth))
                # mu3 = self.C_Suth * T**(3/2) / (T + self.S_Suth)
                # np.testing.assert_allclose(mu1, mu2, rtol=1e-14, atol=1e-14)
                # np.testing.assert_allclose(mu2, mu3, rtol=1e-14, atol=1e-14)
                # mu = np.copy(mu3)
                
                mu = self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth))
                nu = mu / rho
                
                # ===
                
                ## ddy[u] equivalent : ddn[utang]
                ddn_utang = gradient(utang, snorm, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
                ddn_utang_wall = np.copy( ddn_utang[:,0,:,:] )
                
                ## ddy[w] equivalent : ddn[w]
                ddn_w = gradient(w, snorm, axis=1, acc=acc, edge_stencil=edge_stencil, d=1, no_warn=True)
                ddn_w_wall = np.copy( ddn_w[:,0,:,:] )
                
                #tau = np.copy( mu * ddn_utang )
                #tau_wall = np.copy( tau[:,0,:] )
                
                ## dimensional!
                rho_wall = np.copy( rho[:,0,:,:] )
                mu_wall  = np.copy( mu[:,0,:,:]  )
                nu_wall  = np.copy( nu[:,0,:,:]  )
                T_wall   = np.copy( T[:,0,:,:]   )
                p_wall   = np.copy( p[:,0,:,:]   )
                
                tau_wall = np.copy( mu_wall * ddn_utang_wall )
                
                #u_tau = np.sqrt(tau_wall/rho_wall)
                arg = np.copy(tau_wall/rho_wall)
                u_tau = np.sqrt(np.abs(arg)) * np.sign(arg)
                
                tau_u3_s2 = np.copy( mu_wall * ddn_w_wall )
                
                # ===
                
                ## 3D [scalar][x,z] structured array
                dtypes_spd  = [ np.float32 for s in scalars_spd ]
                data4spd = np.zeros(shape=(nxr,nzr,ntc), dtype={'names':scalars_spd, 'formats':dtypes_spd})
                data4spd['tau_u1_s2'][:,:,:] = tau_wall  / ( self.rho_inf * self.U_inf**2 )
                data4spd['tau_u3_s2'][:,:,:] = tau_u3_s2 / ( self.rho_inf * self.U_inf**2 )
                data4spd['T'][:,:,:]         = T_wall    / self.T_inf
                data4spd['rho'][:,:,:]       = rho_wall  / self.rho_inf
                data4spd['p'][:,:,:]         = p_wall    / ( self.rho_inf * self.U_inf**2 )
                data4spd['mu'][:,:,:]        = mu_wall   / self.mu_inf
                data4spd['nu'][:,:,:]        = nu_wall   / self.nu_inf
                data4spd['u_tau'][:,:,:]     = u_tau     / self.U_inf
                
                for scalar in data4spd.dtype.names:
                    
                    dset = hfspd[f'data/{scalar}']
                    if hfspd.usingmpi: hfspd.comm.Barrier()
                    t_start = timeit.default_timer()
                    if hfspd.usingmpi:
                        with dset.collective:
                            #dset[ct1:ct2,rz1:rz2,rx1:rx2] = data4spd[scalar]
                            dset[rx1:rx2,rz1:rz2,ct1:ct2] = data4spd[scalar]
                    else:
                        dset[:,:,ct1:ct2] = data4spd[scalar]
                    if hfspd.usingmpi: hfspd.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = 4 * ntc * 1 * self.nx * self.nz / 1024**3
                    
                    if verbose: tqdm.write(even_print(f'write {scalar}','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.export_polydata_wall() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def export_polydata_xpln(self, fn_spd=None, **kwargs):
        '''
        get 2D [y,z] quantities of a reduced I/O volume which is thin in the [x]/[s1] direction
        and export structured polydata (SPD) file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        force    = kwargs.get('force',False)
        
        acc = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        if verbose: print('\n'+'cgd.export_polydata_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/utang' in self) and self.curvilinear:
            raise ValueError('data/utang not in hdf5')
        if not ('data/unorm' in self) and self.curvilinear:
            raise ValueError('data/unorm not in hdf5')
        if not ('dims/snorm' in self) and self.curvilinear:
            raise ValueError('dims/snorm not in hdf5')
        if not ('dims/stang' in self) and self.curvilinear:
            raise ValueError('dims/stang not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## polydata file name (for writing)
        if (fn_spd is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_spd_h5_base = fname_root+'_polydata.h5'
            #fn_spd = os.path.join(fname_path, fname_spd_h5_base)
            fn_spd = str(PurePosixPath(fname_path, fname_spd_h5_base))
            #fn_spd = Path(fname_path, fname_spd_h5_base)
        
        # === ranks
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ri1 = ry1
        ri2 = ry2
        rj1 = rz1
        rj2 = rz2
        
        ## [t] chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        data_gb = (self.ny * self.nz) * 4 / 1024**3
        
        ## get [snorm] or [y]
        if ('dims/snorm' in self):
            dset = self['dims/snorm']
            snorm = np.copy( dset[ry1:ry2] )
        else:
            if self.rectilinear:
                dset = self['dims/y']
                snorm = np.copy( dset[0,ry1:ry2,0].T )
            else:
                raise ValueError
        
        ## take middle-most x index
        xi = self.nx // 2
        
        ## 3D polydata grid coordinates : shape (ny,nz,3)
        xyz_wall = np.zeros((self.ny, self.nz, 3))
        xyz_wall[:,:,0] = np.copy(self['dims/x'][:,:,xi]).T
        xyz_wall[:,:,1] = np.copy(self['dims/y'][:,:,xi]).T
        xyz_wall[:,:,2] = np.copy(self['dims/z'][:,:,xi]).T
        
        ## open & initialize polydata (SPD) file
        if verbose: even_print('surface polydata (spd) .h5', fn_spd)
        with spd(fn_spd, 'w', force=force, driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
            
            ## add attributes from CGD to SPD
            ## this doesnt work because CGD does not use attributes for its metadata as of writing this func
            # for key,val in self.attrs.items():
            #     hfspd.attrs[key] = val
            
            ## add attributes from CGD to SPD
            for key in self.udef:
                hfspd.attrs[key] = self.udef[key]
            #for key in self.udef_deriv:
            #    hfspd.attrs[key] = self.udef_deriv[key]
            
            ## add time vector from CGD to SPD
            dsn = f'dims/t'
            if (dsn in hfspd):
                del hfspd[dsn]
            data = np.copy(self[dsn][()])
            ds = hfspd.create_dataset(dsn, data=data, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## add 3D polydata grid coordinates : shape (nx,nz,3)
            dsn = f'dims/xyz'
            if (dsn in hfspd):
                del hfspd[dsn]
            ds = hfspd.create_dataset(dsn, data=xyz_wall, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ni = self.ny
            nj = self.nz
            nt = self.nt
            hfspd.attrs['ni'] = ni
            hfspd.attrs['nj'] = nj
            hfspd.attrs['n_quads'] = ( ni - 1 ) * ( nj - 1 )
            hfspd.attrs['n_pts'] = ni * nj
            hfspd.attrs['nt'] = nt
            
            if verbose: print(72*'-')
            hfspd.get_header(verbose=verbose)
            if verbose: print(72*'-')
            
            ## add additional [dims/<>] dsets
            for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][()])
                    
                    if (data.shape==(self.nx,)):
                        data = np.array( [data[xi] ], dtype=data.dtype )
                        if (data.ndim!=1):
                            raise ValueError
                    
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            ## add additional [csys/<>] dsets
            for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][xi,:,:])
                    
                    if (data.ndim!=3):
                        data = data[np.newaxis,:,:]
                    if (data.ndim!=3):
                        raise ValueError
                    
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            if verbose: print(72*'-')
            
            ## initialize datasets
            for scalar in self.scalars:
                
                dsn = f'data/{scalar}'
                dset = self[dsn]
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                
                #shape  = (self.ny,self.nz,self.nt)
                shape  = (ni,nj,nt)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,1), size_kb=chunk_kb, base=4, itemsize=float_bytes)
                
                data_gb = np.prod(shape) * float_bytes / 1024**3
                
                if verbose:
                    even_print(f'initializing {dsn}','%0.1f [GB]'%(data_gb,))
                if (dsn in hfspd):
                    del hfspd[dsn]
                dset = hfspd.create_dataset( dsn,
                                             shape=shape,
                                             dtype=dtype,
                                             chunks=chunks )
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (i,j,t)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === main loop
        
        data_gb_read  = 0.
        data_gb_write = 0.
        t_read  = 0.
        t_write = 0.
        
        with spd(fn_spd, 'a', driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
            
            if verbose:
                progress_bar = tqdm(total=ct*self.n_scalars, ncols=100, desc='export spd', leave=False, file=sys.stdout)
            
            for scalar in self.scalars:
                
                dsn = f'data/{scalar}'
                
                dset_src = self[dsn]
                dset_tgt = hfspd[dsn]
                
                dtype = dset_src.dtype
                float_bytes = dtype.itemsize
                
                for ctl_ in ctl:
                    ct1, ct2 = ctl_
                    ntc = ct2 - ct1
                    
                    ## read
                    self.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset_src.collective:
                        data = np.copy( dset_src[ct1:ct2,rz1:rz2,ry1:ry2,xi].T )
                    self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = float_bytes * ni * nj * ntc / 1024**3
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    if verbose:
                        tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    ## write
                    hfspd.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset_tgt.collective:
                        dset_tgt[ri1:ri2,rj1:rj2,ct1:ct2] = data
                    hfspd.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = float_bytes * ni * nj * ntc / 1024**3
                    
                    t_write       += t_delta
                    data_gb_write += data_gb
                    
                    if verbose:
                        tqdm.write(even_print(f'write: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    if verbose: progress_bar.update()
            
            if verbose: progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.export_polydata_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def export_polydata_zpln(self, fn_spd=None, **kwargs):
        '''
        get 2D [x,y] quantities of a reduced I/O volume which is thin in the [z]/[s3] direction
        and export structured polydata (SPD) file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        force    = kwargs.get('force',False)
        
        acc = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(None,None,1)) ## the 'constraint' parameter for sizing h5 chunks (i,j,t)
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing SPD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        d = 1 ## derivative order
        stencil_npts = 2*math.floor((d+1)/2) - 1 + acc ## central explicit FD kernel width
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        n_overlap             = stencil_npts_one_side + 3
        
        if verbose: print('\n'+'cgd.export_polydata_zpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if not ('data/utang' in self) and self.curvilinear:
            raise ValueError('data/utang not in hdf5')
        if not ('data/unorm' in self) and self.curvilinear:
            raise ValueError('data/unorm not in hdf5')
        if not ('dims/snorm' in self) and self.curvilinear:
            raise ValueError('dims/snorm not in hdf5')
        if not ('dims/stang' in self) and self.curvilinear:
            raise ValueError('dims/stang not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## polydata file name (for writing)
        if (fn_spd is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_spd_h5_base = fname_root+'_polydata.h5'
            #fn_spd = os.path.join(fname_path, fname_spd_h5_base)
            fn_spd = str(PurePosixPath(fname_path, fname_spd_h5_base))
            #fn_spd = Path(fname_path, fname_spd_h5_base)
        
        # === ranks
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # ===
        
        ri1 = rx1
        ri2 = rx2
        rj1 = ry1
        rj2 = ry2
        
        ## [t] chunks
        ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## get [snorm] or [y]
        if ('dims/snorm' in self):
            dset = self['dims/snorm']
            snorm = np.copy( dset[ry1:ry2] )
        else:
            if self.rectilinear:
                dset = self['dims/y']
                snorm = np.copy( dset[0,ry1:ry2,0].T )
            else:
                raise ValueError
        
        ## get [stang] or [x]
        if ('dims/stang' in self):
            dset = self['dims/stang']
            stang = np.copy( dset[rx1:rx2] )
        else:
            if self.rectilinear:
                dset = self['dims/x']
                stang = np.copy( dset[rx1:rx2,ry1:ry2,0].T )
            else:
                raise ValueError
        
        ## get middle-most [z] index
        zi = self.nz // 2
        
        ## 3D polydata grid coordinates : shape (nx,ny,3)
        xyz = np.zeros((self.nx, self.ny, 3))
        xyz[:,:,0] = np.copy(self['dims/x'][zi,:,:]).T
        xyz[:,:,1] = np.copy(self['dims/y'][zi,:,:]).T
        xyz[:,:,2] = np.copy(self['dims/z'][zi,:,:]).T
        
        ## open & initialize polydata (SPD) file
        if verbose: even_print('surface polydata (spd) .h5', fn_spd)
        with spd(fn_spd, 'w',
                 force=force,
                 driver=self.driver,
                 comm=MPI.COMM_WORLD,
                 stripe_count=stripe_count,
                 stripe_size_mb=stripe_size_mb) as hfspd:
            
            ## add attributes from CGD to SPD
            ## this doesnt work because CGD does not use attributes for its metadata as of writing this func
            # for key,val in self.attrs.items():
            #     hfspd.attrs[key] = val
            
            ## add attributes from CGD to SPD
            for key in self.udef:
                hfspd.attrs[key] = self.udef[key]
            #for key in self.udef_deriv:
            #    hfspd.attrs[key] = self.udef_deriv[key]
            
            ## add time vector from CGD to SPD
            dsn = f'dims/t'
            if (dsn in hfspd):
                del hfspd[dsn]
            data = np.copy(self[dsn][()])
            ds = hfspd.create_dataset(dsn, data=data, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ## add 3D polydata grid coordinates : shape (nx,nz,3)
            dsn = f'dims/xyz'
            if (dsn in hfspd):
                del hfspd[dsn]
            ds = hfspd.create_dataset(dsn, data=xyz, chunks=None) ## no transpose!
            if verbose: even_print(dsn,'%s'%str(ds.shape))
            
            ni = self.nx
            nj = self.ny
            nt = self.nt
            hfspd.attrs['ni'] = ni
            hfspd.attrs['nj'] = nj
            hfspd.attrs['n_quads'] = ( ni - 1 ) * ( nj - 1 )
            hfspd.attrs['n_pts'] = ni * nj
            hfspd.attrs['nt'] = nt
            
            if verbose: print(72*'-')
            hfspd.get_header(verbose=verbose)
            if verbose: print(72*'-')
            
            ## add additional [dims/<>] dsets
            for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    data = np.copy(self[dsn][()])
                    
                    ## this would clip the additional datasets in [x], but doing nothing for now
                    if (data.shape==(self.nx,)):
                        #data = np.array( [data[xi]], dtype=data.dtype )
                        if (data.ndim!=1):
                            raise ValueError
                    
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            ## add additional [csys/<>] dsets
            for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                if (dsn in self):
                    if (dsn in hfspd):
                        del hfspd[dsn]
                    
                    #data = np.copy(self[dsn][xi,:,:]) ## clip in [x]
                    data = np.copy(self[dsn][()])
                    
                    if (data.ndim!=3):
                        data = data[np.newaxis,:,:]
                    if (data.ndim!=3):
                        raise ValueError
                    
                    ds = hfspd.create_dataset(dsn, data=data, chunks=None)
                    if verbose: even_print(dsn,'%s'%str(ds.shape))
                else:
                    if verbose: even_print(dsn,'not found')
            
            if verbose: print(72*'-')
            
            ## initialize datasets
            #shape   = (self.nx,self.ny,self.nt)
            shape  = (ni,nj,nt)
            
            scalars_spd = self.scalars
            
            # scalars_spd += ['umag',
            #                 'ddx_u', 'ddy_u', 'ddz_u',
            #                 'ddx_v', 'ddy_v', 'ddz_v',
            #                 'ddx_w', 'ddy_w', 'ddz_w',
            #                 'mag_grad_vel',
            #                 'ddx_rho', 'ddy_rho', 'ddz_rho',
            #                 'mag_grad_rho',
            #                 'ddx_p', 'ddy_p', 'ddz_p',
            #                 'mag_grad_p',
            #                 'ddx_T', 'ddy_T', 'ddz_T',
            #                 'mag_grad_T',
            #                 ]
            
            for scalar in scalars_spd:
                
                dsn = f'data/{scalar}'
                
                if (dsn in self):
                    dset = self[dsn]
                    dtype = dset.dtype
                    float_bytes = dtype.itemsize
                else:
                    dset = None
                    dtype = np.dtype(np.float32)
                    float_bytes = dtype.itemsize
                
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                data_gb = np.prod(shape) * float_bytes / 1024**3
                
                if verbose:
                    even_print(f'initializing {dsn}','%0.1f [GB]'%(data_gb,))
                if (dsn in hfspd):
                    del hfspd[dsn]
                dset = hfspd.create_dataset( dsn,
                                             shape=shape,
                                             dtype=dtype,
                                             chunks=chunks,
                                             )
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (i,j,t)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === main loop : copy over [u,v,w,ρ,T,p]
        
        data_gb_read  = 0.
        data_gb_write = 0.
        t_read  = 0.
        t_write = 0.
        
        with spd(fn_spd, 'a', driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
            
            if verbose:
                progress_bar = tqdm(total=ct*self.n_scalars, ncols=100, desc='export spd', leave=False, file=sys.stdout)
            
            for scalar in scalars_spd:
                
                dsn = f'data/{scalar}'
                
                if (dsn in self):
                    
                    dset_src = self[dsn]
                    dset_tgt = hfspd[dsn]
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    for ctl_ in ctl:
                        ct1, ct2 = ctl_
                        ntc = ct2 - ct1
                        
                        ## read
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_src.collective:
                            data = np.copy( dset_src[ct1:ct2,zi,ry1:ry2,rx1:rx2].T )
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * ni * nj * ntc / 1024**3
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        if verbose:
                            tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        ## write
                        hfspd.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_tgt.collective:
                            dset_tgt[ri1:ri2,rj1:rj2,ct1:ct2] = data
                        hfspd.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * ni * nj * ntc / 1024**3
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            tqdm.write(even_print(f'write: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        if verbose: progress_bar.update()
            
            if verbose: progress_bar.close()
        
        ## === [u,v,w,ρ,T,p] have been copied, now calculate extra variables 
        
        # === extend the rank ranges (spatial range overlap)
        
        ## backup non-overlapped bounds
        rx1_orig, rx2_orig = rx1, rx2
        ry1_orig, ry2_orig = ry1, ry2
        #rz1_orig, rz2_orig = rz1, rz2
        
        if self.usingmpi: 
            
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            #zA = 0
            #zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            #rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## overlap in [y]
            if (t4d[1]!=0):
                ry1, ry2 = ry1-n_overlap, ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                ry1, ry2 = ry1, ry2+n_overlap
            
            # ## overlap in [z]
            # if (t4d[2]!=0):
            #     rz1, rz2 = rz1-n_overlap, rz2
            #     zA += n_overlap
            #     zB += n_overlap
            # if (t4d[2]!=rz-1):
            #     rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            #nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        ## not finished!
        
        ## with spd(fn_spd, 'a', driver=self.driver, comm=MPI.COMM_WORLD) as hfspd:
        ##     
        ##     # if verbose:
        ##     #     progress_bar = tqdm(total=ct*self.n_scalars, ncols=100, desc='export spd', leave=False, file=sys.stdout)
        ##     
        ##     for scalar in scalars_spd:
        ##         
        ##         dsn = f'data/{scalar}'
        ##         
        ##         if (dsn in self):
        ##             
        ##             dset_src = self[dsn]
        ##             dset_tgt = hfspd[dsn]
        ##             
        ##             dtype = dset_src.dtype
        ##             float_bytes = dtype.itemsize
        ##             
        ##             for ctl_ in ctl:
        ##                 ct1, ct2 = ctl_
        ##                 ntc = ct2 - ct1
        ##                 
        ##                 ## read
        ##                 self.comm.Barrier()
        ##                 t_start = timeit.default_timer()
        ##                 with dset_src.collective:
        ##                     data = np.copy( dset_src[ct1:ct2,zi,ry1:ry2,rx1:rx2].T )
        ##                 self.comm.Barrier()
        ##                 t_delta = timeit.default_timer() - t_start
        ##                 data_gb = float_bytes * ni * nj * ntc / 1024**3
        ##                 
        ##                 t_read       += t_delta
        ##                 data_gb_read += data_gb
        ##                 
        ##                 if verbose:
        ##                     tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
        ##                 
        ##                 ## write
        ##                 hfspd.comm.Barrier()
        ##                 t_start = timeit.default_timer()
        ##                 with dset_tgt.collective:
        ##                     dset_tgt[ri1:ri2,rj1:rj2,ct1:ct2] = data[xA:xB,yA:yB,:]
        ##                 hfspd.comm.Barrier()
        ##                 t_delta = timeit.default_timer() - t_start
        ##                 data_gb = float_bytes * ni * nj * ntc / 1024**3
        ##                 
        ##                 t_write       += t_delta
        ##                 data_gb_write += data_gb
        ##                 
        ##                 if verbose:
        ##                     tqdm.write(even_print(f'write: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
        ##                 
        ##                 # if verbose: progress_bar.update()
        ##     
        ##     # if verbose: progress_bar.close()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.export_polydata_zpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # ===
    
    def check_grid_validity_z(self,):
        '''
        check grid validity for CGD file (in [z] direction)
        - monotonically increasing
        - no Δ=0
        '''
        
        verbose = True
        
        if verbose: print('\n'+'cgd.check_grid_validity_z()'+'\n'+72*'-')
        
        # x = np.copy(self.x)
        # y = np.copy(self.y)
        # z = np.copy(self.z)
        # t = np.copy(self.t)
        
        z = np.copy(self['dims/z']).T
        
        nx,ny,nz = z.shape
        
        z_ref = np.copy(z[0,0,:])
        
        ## check no zero distance elements
        if (np.diff(z_ref).size - np.count_nonzero(np.diff(z_ref))) != 0.:
            if verbose: even_print('check: Δz!=0','failed')
        else:
            if verbose: even_print('check: Δz!=0','passed')
        
        ## check monotonically increasing
        if not np.all(np.diff(z_ref) > 0.):
            if verbose: even_print('check: z mono increasing','failed')
        else:
            if verbose: even_print('check: z mono increasing','passed')
        
        if verbose: print(72*'-')
        
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from CGD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        makeVectors = kwargs.get('makeVectors',True) ## write vectors (e.g. velocity, vorticity) to XDMF
        makeTensors = kwargs.get('makeTensors',True) ## write 3x3 tensors (e.g. stress, strain) to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if verbose: print('\n'+'cgd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for scalar in ['x','y','z']:
            if ('dims/'+scalar in self):
                data = self['dims/'+scalar]
                
                txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
                if verbose: even_print(scalar, txt)
                
                dataset_precision_dict[scalar] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[scalar] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[scalar] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        # scalar names dict
        # --> labels for Paraview could be customized (e.g. units could be added) using a dict
        # --> the block below shows one such example dict, though it is currently inactive
        
        if False:
            units = 'dimless'
            if (units=='SI') or (units=='si'): ## m,s,kg,K
                scalar_names = {'x':'x [m]',
                                'y':'y [m]',
                                'z':'z [m]', 
                                'u':'u [m/s]',
                                'v':'v [m/s]',
                                'w':'w [m/s]', 
                                'T':'T [K]',
                                'rho':'rho [kg/m^3]',
                                'p':'p [Pa]'}
            elif (units=='dimless') or (units=='dimensionless'):
                scalar_names = {'x':'x [dimless]',
                                'y':'y [dimless]',
                                'z':'z [dimless]', 
                                'u':'u [dimless]',
                                'v':'v [dimless]',
                                'w':'w [dimless]',
                                'T':'T [dimless]',
                                'rho':'rho [dimless]',
                                'p':'p [dimless]'}
            else:
                raise ValueError('choice of units not recognized : %s --> options are : %s / %s'%(units,'SI','dimless'))
        else:
            scalar_names = {} ## dummy/empty 
        
        ## refresh header
        self.get_header(verbose=False, read_grid=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')
        
        # === write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                ## <Topology TopologyType="3DRectMesh" NumberOfElements="{self.nz:d} {self.ny:d} {self.nx:d}"/>
                ## <Geometry GeometryType="VxVyVz">
                
                xdmf_str=f'''
                         <Topology TopologyType="3DSMesh" NumberOfElements="{self.nz:d} {self.ny:d} {self.nx:d}"/>
                         <Geometry GeometryType="X_Y_Z">
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                             {fname_base}:/dims/{'x'}
                           </DataItem>
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                             {fname_base}:/dims/{'y'}
                           </DataItem>
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['z']}" Precision="{dataset_precision_dict['z']:d}" Format="HDF">
                             {fname_base}:/dims/{'z'}
                           </DataItem>
                         </Geometry>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # =====
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===== .xdmf : <Grid> per 3D coordinate array
                    
                    for scalar in ['x','y','z']:
                        
                        dset_hf_path = 'dims/%s'%scalar
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        if scalar in scalar_names:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        xdmf_str=f'''
                                 <!-- ===== scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                     {fname_base}:/{dset_hf_path}
                                   </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # ===== .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        
                        dset_hf_path = 'data/%s'%scalar
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        if scalar in scalar_names:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        xdmf_str=f'''
                                 <!-- ===== scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                     <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                       {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                       {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                       {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                     </DataItem>
                                     <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                       {fname_base}:/{dset_hf_path}
                                     </DataItem>
                                   </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    if makeVectors:
                        
                        # === .xdmf : <Grid> per vector : velocity vector
                        
                        if ('u' in self.scalars) and ('v' in self.scalars) and ('w' in self.scalars):
                            
                            scalar_name    = 'velocity'
                            dset_hf_path_i = 'data/u'
                            dset_hf_path_j = 'data/v'
                            dset_hf_path_k = 'data/w'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['u']}" Precision="{dataset_precision_dict['u']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['v']}" Precision="{dataset_precision_dict['v']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['w']}" Precision="{dataset_precision_dict['w']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                        
                        # === .xdmf : <Grid> per vector : velocity vector
                        
                        if ('uI' in self.scalars) and ('vI' in self.scalars) and ('wI' in self.scalars):
                            
                            scalar_name    = 'velocityI'
                            dset_hf_path_i = 'data/uI'
                            dset_hf_path_j = 'data/vI'
                            dset_hf_path_k = 'data/wI'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['uI']}" Precision="{dataset_precision_dict['uI']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vI']}" Precision="{dataset_precision_dict['vI']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['wI']}" Precision="{dataset_precision_dict['wI']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                        
                        # === .xdmf : <Grid> per vector : vorticity vector
                        
                        if ('vort_x' in self.scalars) and ('vort_y' in self.scalars) and ('vort_z' in self.scalars):
                            
                            scalar_name    = 'vorticity'
                            dset_hf_path_i = 'data/vort_x'
                            dset_hf_path_j = 'data/vort_y'
                            dset_hf_path_k = 'data/vort_z'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_x']}" Precision="{dataset_precision_dict['vort_x']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_y']}" Precision="{dataset_precision_dict['vort_y']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_z']}" Precision="{dataset_precision_dict['vort_z']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    if makeTensors:
                        if all([('dudx' in self.scalars),('dvdx' in self.scalars),('dwdx' in self.scalars),
                                ('dudy' in self.scalars),('dvdy' in self.scalars),('dwdy' in self.scalars),
                                ('dudz' in self.scalars),('dvdz' in self.scalars),('dwdz' in self.scalars)]):
                            pass
                            pass ## TODO
                            pass
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if verbose: print('--w-> %s'%fname_xdmf_base)
        return

class rgd(h5py.File):
    '''
    Rectilinear Grid Data (RGD)
    ---------------------------
    - super()'ed h5py.File class
    - 4D dataset storage
    - dimension coordinates are 4x 1D arrays defining [x,y,z,t] 
    
    to clear:
    ---------
    > os.system('h5clear -s tmp.h5')
    > hf = h5py.File('tmp.h5', 'r', libver='latest')
    > hf.close()
    
    Structure
    ---------
    
    rgd.h5
    │
    ├── header/
    │   └── udef_char
    │   └── udef_real
    │
    ├── dims/ --> 1D
    │   └── x
    │   └── y
    │   └── z
    │   └── t
    │
    └-─ data/<<scalar>> --> 4D [t,z,y,x]
    
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.rgd()')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## rgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 16    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2     )
        perms          = kwargs.pop('perms'          , '640' )
        
        if not isinstance(stripe_count, int):
            raise ValueError
        if not isinstance(stripe_size_mb, int):
            raise ValueError
        if not isinstance(perms, str):
            raise ValueError
        if not len(perms)==3:
            raise ValueError
        if not re.fullmatch(r'\d{3}',perms):
            raise ValueError
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using romio --> causes crash
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        
        ## set ROMIO hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(16*1024**2))) ) ## 16 [MB]
                ##
                #mpi_info.Set('romio_no_indep_rw' , 'true' ) ## deferred open + only collective I/O 
                #mpi_info.Set('cb_nodes' , str(int(round(1*self.n_ranks))) )
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        ## | rdcc_nbytes:
        ## | ------------
        ## | Integer setting the total size of the raw data chunk cache for this dataset in bytes.
        ## | In most cases increasing this number will improve performance, as long as you have 
        ## | enough free memory. The default size is 1 MB
        
        ## --> gets passed to H5Pset_chunk_cache
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        ## | rdcc_nslots:
        ## | ------------
        ## | Integer defining the number of chunk slots in the raw data chunk cache for this dataset.
        
        ## if ('rdcc_nslots' not in kwargs):
        ##     kwargs['rdcc_nslots'] = 521
        
        ## rgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with rgd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(0.1)
                Path(self.fname).touch() ## touch a new file
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        ## call actual h5py.File.__init__()
        super(rgd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(rgd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed RGD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(rgd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of RGD class instance
        --> this gets called automatically upon opening the file
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if (self.rank!=0):
            verbose=False
        
        # === attrs
        if ('duration_avg' in self.attrs.keys()):
            self.duration_avg = self.attrs['duration_avg']
        # if ('rectilinear' in self.attrs.keys()):
        #     self.rectilinear = self.attrs['rectilinear']
        # if ('curvilinear' in self.attrs.keys()):
        #     self.curvilinear = self.attrs['curvilinear']
        
        ## these should be set in the (init_from_() funcs)
        if ('fclass' in self.attrs.keys()):
            self.fclass = self.attrs['fclass'] ## 'rgd','cgd',...
        if ('fsubtype' in self.attrs.keys()):
            self.fsubtype = self.attrs['fsubtype'] ## 'unsteady','mean','prime',...
        
        # === udef
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## make dict where keys are udef_char and values are udef_real
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            self.S_Suth      = self.udef['S_Suth']
            #self.C_Suth      = self.udef['C_Suth']
            
            self.C_Suth = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## [kg/(m·s·√K)]
            self.udef['C_Suth'] = self.C_Suth
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            
            # === characteristic values : derived
            
            ## mu_inf_1 = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            ## mu_inf_2 = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            ## mu_inf_3 = self.C_Suth*self.T_inf**(3/2)/(self.T_inf+self.S_Suth)
            ## if not np.isclose(mu_inf_1, mu_inf_2, rtol=1e-14):
            ##     raise AssertionError('inconsistency in Sutherland calc --> check')
            ## if not np.isclose(mu_inf_2, mu_inf_3, rtol=1e-14):
            ##     raise AssertionError('inconsistency in Sutherland calc --> check')
            ## mu_inf = self.mu_inf = mu_inf_2
            
            self.mu_inf    = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            self.rho_inf   = self.p_inf/(self.R*self.T_inf)
            self.nu_inf    = self.mu_inf/self.rho_inf
            self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            self.U_inf     = self.Ma*self.a_inf
            self.cp        = self.R*self.kappa/(self.kappa-1.)
            self.cv        = self.cp/self.kappa
            self.recov_fac = self.Pr**(1/3)
            self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            self.tchar = self.lchar / self.U_inf
            self.uchar = self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            if verbose: even_print('tchar'           , '%0.6E [s]'        % self.tchar     )
            if verbose: print(72*'-')
            #if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the RGD instance
            self.udef_deriv = { 'rho_inf':self.rho_inf,
                                'mu_inf':self.mu_inf,
                                'nu_inf':self.nu_inf,
                                'a_inf':self.a_inf,
                                'U_inf':self.U_inf,
                                'cp':self.cp,
                                'cv':self.cv,
                                'recov_fac':self.recov_fac,
                                'Taw':self.Taw,
                                'lchar':self.lchar,
                              }
        
        else:
            pass
        
        # === coordinate vectors
        
        if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
            
            x   = self.x   = np.copy(self['dims/x'][:])
            y   = self.y   = np.copy(self['dims/y'][:])
            z   = self.z   = np.copy(self['dims/z'][:])
            nx  = self.nx  = x.size
            ny  = self.ny  = y.size
            nz  = self.nz  = z.size
            ngp = self.ngp = nx*ny*nz
            
            #if verbose: print(72*'-')
            if verbose: even_print('nx', '%i'%nx )
            if verbose: even_print('ny', '%i'%ny )
            if verbose: even_print('nz', '%i'%nz )
            if verbose: even_print('ngp', '%i'%ngp )
            if verbose: print(72*'-')
            
            if verbose: even_print('x_min', '%0.2f'%x.min())
            if verbose: even_print('x_max', '%0.2f'%x.max())
            if (self.nx>2):
                if verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (x[1]-x[0]), (x[-1]-x[-2]) ))
            if verbose: even_print('y_min', '%0.2f'%y.min())
            if verbose: even_print('y_max', '%0.2f'%y.max())
            if (self.ny>2):
                if verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (y[1]-y[0]), (y[-1]-y[-2]) ))
            if verbose: even_print('z_min', '%0.2f'%z.min())
            if verbose: even_print('z_max', '%0.2f'%z.max())
            if (self.nz>2):
                if verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (z[1]-z[0]), (z[-1]-z[-2]) ))
            #if verbose: print(72*'-'+'\n')
            if verbose: print(72*'-')
        
        else:
            pass
        
        # === 1D grid filters
        
        self.hasGridFilter=False
        if ('dims/xfi' in self):
            xfi = np.copy(self['dims/xfi'][:])
            self.xfi = xfi
            if not np.array_equal(xfi, np.array(range(nx), dtype=np.int64)):
                self.hasGridFilter=True
        if ('dims/yfi' in self):
            yfi = np.copy(self['dims/yfi'][:])
            self.yfi = yfi
            if not np.array_equal(yfi, np.array(range(ny), dtype=np.int64)):
                self.hasGridFilter=True
        if ('dims/zfi' in self):
            zfi = np.copy(self['dims/zfi'][:])
            self.zfi = zfi
            if not np.array_equal(xfi, np.array(range(nz), dtype=np.int64)):
                self.hasGridFilter=True
        
        # === time vector
        
        if ('dims/t' in self):
            self.t = np.copy(self['dims/t'][:])
            
            if ('data' in self): ## check t dim and data arr agree
                nt,_,_,_ = self['data/%s'%list(self['data'].keys())[0]].shape ## 4D
                if (nt!=self.t.size):
                    raise AssertionError('nt!=self.t.size : %i!=%i'%(nt,self.t.size))
            
            nt = self.t.size
            
            try:
                self.dt = self.t[1] - self.t[0]
            except IndexError:
                self.dt = 0.
            
            self.nt       = nt       = self.t.size
            self.duration = duration = self.t[-1] - self.t[0]
            self.ti       = ti       = np.array(range(self.nt), dtype=np.int64)
        
        elif all([('data' in self),('dims/t' not in self)]): ## data but no time
            self.scalars = list(self['data'].keys())
            nt,_,_,_ = self['data/%s'%self.scalars[0]].shape
            self.nt  = nt
            self.t   =      np.arange(self.nt, dtype=np.float64)
            self.ti  = ti = np.arange(self.nt, dtype=np.int64)
            self.dt  = 1.
            self.duration = duration = self.t[-1]-self.t[0]
        
        else: ## no data, no time
            self.t  = np.array([], dtype=np.float64)
            self.ti = np.array([], dtype=np.int64)
            self.nt = nt = 0
            self.dt = 0.
            self.duration = duration = 0.
        
        #if verbose: print(72*'-')
        if verbose: even_print('nt', '%i'%self.nt )
        if verbose: even_print('dt', '%0.6f'%self.dt)
        if verbose: even_print('duration', '%0.2f'%self.duration )
        if hasattr(self, 'duration_avg'):
            if verbose: even_print('duration_avg', '%0.2f'%self.duration_avg )
        #if verbose: print(72*'-'+'\n')
        
        # if hasattr(self,'rectilinear'):
        #     if verbose: even_print('rectilinear', str(self.rectilinear) )
        # if hasattr(self,'curvilinear'):
        #     if verbose: even_print('curvilinear', str(self.curvilinear) )
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys()) ## 4D : string names of scalars : ['u','v','w'],...
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self[f'data/{scalar}'].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    # === I/O
    
    def init_from_eas4(self, fn_eas4, **kwargs):
        '''
        initialize an RGD from an EAS4 (NS3D output format)
        -----
        - x_min/max xi_min/max : min/max coord/index
        - stride filters (sx,sy,sz)
        '''
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        verbose = kwargs.get('verbose',True)
        if (self.rank!=0):
            verbose=False
        
        # === spatial resolution filter : take every nth grid point
        sx = kwargs.get('sx',1)
        sy = kwargs.get('sy',1)
        sz = kwargs.get('sz',1)
        #st = kwargs.get('st',1)
        
        # === spatial resolution filter : set x/y/z bounds
        x_min = kwargs.get('x_min',None)
        y_min = kwargs.get('y_min',None)
        z_min = kwargs.get('z_min',None)
        
        x_max = kwargs.get('x_max',None)
        y_max = kwargs.get('y_max',None)
        z_max = kwargs.get('z_max',None)
        
        xi_min = kwargs.get('xi_min',None)
        yi_min = kwargs.get('yi_min',None)
        zi_min = kwargs.get('zi_min',None)
        
        xi_max = kwargs.get('xi_max',None)
        yi_max = kwargs.get('yi_max',None)
        zi_max = kwargs.get('zi_max',None)
        
        ## set default attributes
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'rgd'
        
        if verbose: print('\n'+'rgd.init_from_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        # even_print('infile', os.path.basename(fn_eas4))
        # even_print('infile size', '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3))
        # even_print('outfile', self.fname)
        
        if verbose: even_print('infile', os.path.basename(fn_eas4))
        if verbose: even_print('infile size', '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3))
        if verbose: even_print('outfile', self.fname)
        
        with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=self.comm) as hf_eas4:
            
            if verbose: even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1_orig, gmode_dict[hf_eas4.gmode_dim1_orig] ) )
            if verbose: even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2_orig, gmode_dict[hf_eas4.gmode_dim2_orig] ) )
            if verbose: even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3_orig, gmode_dict[hf_eas4.gmode_dim3_orig] ) )
            
            # === check gmode (RGD should not have more than ALL_G)
            if (hf_eas4.gmode_dim1_orig > 4):
                raise ValueError('turbx.rgd cannot handle gmode > 4 (EAS4 gmode_dim1=%i)'%hf_eas4.gmode_dim1_orig)
            if (hf_eas4.gmode_dim2_orig > 4):
                raise ValueError('turbx.rgd cannot handle gmode > 4 (EAS4 gmode_dim2=%i)'%hf_eas4.gmode_dim2_orig)
            if (hf_eas4.gmode_dim3_orig > 4):
                raise ValueError('turbx.rgd cannot handle gmode > 4 (EAS4 gmode_dim3=%i)'%hf_eas4.gmode_dim3_orig)
            
            # === copy over header info if needed
            if all([('header/udef_real' in self),('header/udef_char' in self)]):
                raise ValueError('udef already present')
            else:
                udef         = hf_eas4.udef
                udef_real    = list(udef.values())
                udef_char    = list(udef.keys())
                udef_real_h5 = np.array(udef_real, dtype=np.float64)
                udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
                
                self.create_dataset('header/udef_real', data=udef_real_h5, dtype=np.float64)
                self.create_dataset('header/udef_char', data=udef_char_h5, dtype='S128')
                self.udef      = udef
                self.udef_real = udef_real
                self.udef_char = udef_char
             
            # === copy over dims info
            if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
                pass
                ## future: 2D/3D handling here
            else:
                x, y, z = hf_eas4.x, hf_eas4.y, hf_eas4.z
                nx  = x.size
                ny  = y.size
                nz  = z.size
                ngp = nx*ny*nz
                # nt = hf_eas4.nt --> no time data available yet
                
                if any([(xi_min is not None),\
                        (xi_max is not None),\
                        (yi_min is not None),\
                        (yi_max is not None),\
                        (zi_min is not None),\
                        (zi_max is not None),\
                        (x_min is not None),\
                        (x_max is not None),\
                        (y_min is not None),\
                        (y_max is not None),\
                        (z_min is not None),\
                        (z_max is not None),\
                        (sx!=1),\
                        (sy!=1),\
                        (sz!=1)]):
                            hasFilters=True
                            if verbose: print('filtered dim info\n'+72*'-')
                else:
                    hasFilters=False
                
                # === index arrays along each axis
                xfi = np.array(range(nx),  dtype=np.int64)
                yfi = np.array(range(ny),  dtype=np.int64)
                zfi = np.array(range(nz),  dtype=np.int64)
                #tfi = np.array(range(nt), dtype=np.int64) --> no time data available yet
                # === total bounds clip (physical nondimensional distance)
                if (x_min is not None):
                    xfi = np.array([i for i in xfi if (x[i] >= x_min)])
                    if verbose: even_print('x_min', '%0.3f'%x_min)
                if (x_max is not None):
                    xfi = np.array([i for i in xfi if (x[i] <= x_max)])
                    if verbose: even_print('x_max', '%0.3f'%x_max)
                if (y_min is not None):
                    yfi = np.array([i for i in yfi if (y[i] >= y_min)])
                    if verbose: even_print('y_min', '%0.3f'%y_min)
                if (y_max is not None):
                    yfi = np.array([i for i in yfi if (y[i] <= y_max)])
                    if verbose: even_print('y_max', '%0.3f'%y_max)
                if (z_min is not None):
                    zfi = np.array([i for i in zfi if (z[i] >= z_min)])
                    if verbose: even_print('z_min', '%0.3f'%z_min)
                if (z_max is not None):
                    zfi = np.array([i for i in zfi if (z[i] <= z_max)])
                    if verbose: even_print('z_max', '%0.3f'%z_max)
                
                # === total bounds clip (coordinate index)
                if (xi_min is not None):
                    
                    xfi_ = []
                    if verbose: even_print('xi_min', '%i'%xi_min)
                    for c in xfi:
                        if (xi_min<0) and (c>=(nx+xi_min)): ## support negative indexing
                            xfi_.append(c)
                        elif (xi_min>=0) and (c>=xi_min):
                            xfi_.append(c)
                    xfi=np.array(xfi_, dtype=np.int64)
                
                if (xi_max is not None):
                    
                    xfi_ = []
                    if verbose: even_print('xi_max', '%i'%xi_max)
                    for c in xfi:
                        if (xi_max<0) and (c<=(nx+xi_max)): ## support negative indexing
                            xfi_.append(c)
                        elif (xi_max>=0) and (c<=xi_max):
                            xfi_.append(c)
                    xfi=np.array(xfi_, dtype=np.int64)
                
                if (yi_min is not None):
                    
                    yfi_ = []
                    if verbose: even_print('yi_min', '%i'%yi_min)
                    for c in yfi:
                        if (yi_min<0) and (c>=(ny+yi_min)): ## support negative indexing
                            yfi_.append(c)
                        elif (yi_min>=0) and (c>=yi_min):
                            yfi_.append(c)
                    yfi=np.array(yfi_, dtype=np.int64)
                
                if (yi_max is not None):
                    
                    yfi_ = []
                    if verbose: even_print('yi_max', '%i'%yi_max)
                    for c in yfi:
                        if (yi_max<0) and (c<=(ny+yi_max)): ## support negative indexing
                            yfi_.append(c)
                        elif (yi_max>=0) and (c<=yi_max):
                            yfi_.append(c)
                    yfi=np.array(yfi_, dtype=np.int64)
                
                if (zi_min is not None):
                    
                    zfi_ = []
                    if verbose: even_print('zi_min', '%i'%zi_min)
                    for c in zfi:
                        if (zi_min<0) and (c>=(nz+zi_min)): ## support negative indexing
                            zfi_.append(c)
                        elif (zi_min>=0) and (c>=zi_min):
                            zfi_.append(c)
                    zfi=np.array(zfi_, dtype=np.int64)
                
                if (zi_max is not None):
                    
                    zfi_ = []
                    if verbose: even_print('zi_max', '%i'%zi_max)
                    for c in zfi:
                        if (zi_max<0) and (c<=(nz+zi_max)): ## support negative indexing
                            zfi_.append(c)
                        elif (zi_max>=0) and (c<=zi_max):
                            zfi_.append(c)
                    zfi=np.array(zfi_, dtype=np.int64)
                
                # === resolution filter (skip every n grid points in each direction)
                if (sx!=1):
                    if verbose: even_print('sx', '%i'%sx)
                    xfi = xfi[::sx]
                if (sy!=1):
                    if verbose: even_print('sy', '%i'%sy)
                    yfi = yfi[::sy]
                if (sz!=1):
                    if verbose: even_print('sz', '%i'%sz)
                    zfi = zfi[::sz]
                # if (st!=1):
                #     even_print('st', '%i'%st)
                #     tfi = tfi[::st]
                # ===
                self.xfi = xfi
                self.yfi = yfi
                self.zfi = zfi
                #self.tfi = tfi
                # ===
                self.create_dataset('dims/xfi', data=xfi)
                self.create_dataset('dims/yfi', data=yfi)
                self.create_dataset('dims/zfi', data=zfi)
                #self.create_dataset('dims/tfi', data=tfi, maxshape=np.shape(tfi))
                
                if (xfi.size==0):
                    raise ValueError('x grid filter is empty... check!')
                if (yfi.size==0):
                    raise ValueError('y grid filter is empty... check!')
                if (zfi.size==0):
                    raise ValueError('z grid filter is empty... check!')
                #if (tfi.size==0):
                #    raise ValueError('t grid filter is empty... check!')
                
                # === determine if gridFilter is active
                self.hasGridFilter=False
                if not np.array_equal(xfi, np.array(range(nx), dtype=np.int64)):
                    self.hasGridFilter=True
                if not np.array_equal(yfi, np.array(range(ny), dtype=np.int64)):
                    self.hasGridFilter=True
                if not np.array_equal(zfi, np.array(range(nz), dtype=np.int64)):
                    self.hasGridFilter=True
                #if not np.array_equal(tfi, np.array(range(nt), dtype=np.int64)):
                #    self.hasGridFilter=True
                # === overwrite & write 1D filter arrays
                x = np.copy(x[xfi])
                y = np.copy(y[yfi])
                z = np.copy(z[zfi])
                #t = np.copy(t[tfi])
                
                ### if self.hasGridFilter:
                ###     print('>>> grid filter is present')
                ### else:
                ###     print('>>> no grid filter present')
                
                # === (over)write coord vecs if filter present
                if self.hasGridFilter:
                    
                    nx = x.size
                    ny = y.size
                    nz = z.size
                    ngp = nx*ny*nz
                    #nt = t.size
                    
                    if verbose: even_print('nx',  '%i'%nx  )
                    if verbose: even_print('ny',  '%i'%ny  )
                    if verbose: even_print('nz',  '%i'%nz  )
                    if verbose: even_print('ngp', '%i'%ngp )
                    #if verbose: even_print('nt', '%i'%nt )
                    
                    self.nx  = nx
                    self.ny  = ny
                    self.nz  = nz
                    self.ngp = ngp
                    #self.nt = nt
                
                # === write 1D coord arrays
                if ('dims/x' in self):
                    del self['dims/x']
                self.create_dataset('dims/x', data=x)
                if ('dims/y' in self):
                    del self['dims/y']
                self.create_dataset('dims/y', data=y)
                if ('dims/z' in self):
                    del self['dims/z']
                self.create_dataset('dims/z', data=z)
                # if ('dims/t' in self):
                #     del self['dims/t']
                # self.create_dataset('dims/t', data=t, maxshape=t.shape)
                
                # === write 3D coord arrays
                if False:
                    size3DCoordArrays = 3*ngp*8/1024**2 ## [MB]
                    #print('>>> size of 3D coord arrays : %0.2f [GB]'%size3DCoordArrays)
                    if (size3DCoordArrays < 100): ## if less than 100 [MB]
                        xxx, yyy, zzz = np.meshgrid(x, y, z, indexing='ij')
                        self.create_dataset('dims/xxx', data=xxx.T)
                        self.create_dataset('dims/yyy', data=yyy.T)
                        self.create_dataset('dims/zzz', data=zzz.T)
        
        if verbose: print(72*'-')
        self.get_header(verbose=True)
        if verbose: print(72*'-')
        
        return
    
    def init_from_rgd(self, fn_rgd, **kwargs):
        '''
        initialize an RGD from an RGD (copy over header data & coordinate data)
        '''
        
        t_info = kwargs.get('t_info',True)
        #chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        
        verbose = kwargs.get('verbose',True)
        if (self.rank!=0):
            verbose=False
        
        ## set default attributes: fsubtype, fclass
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'rgd'
        
        with rgd(fn_rgd, 'r', driver=self.driver, comm=self.comm) as hf_ref:
            
            ## copy over fsubtype
            if hasattr(hf_ref,'fsubtype'):
                self.attrs['fsubtype'] = hf_ref.fsubtype
            
            # === copy over header info if needed
            
            if all([('header/udef_real' in self),('header/udef_char' in self)]):
                raise ValueError('udef already present')
            else:
                udef         = hf_ref.udef
                udef_real    = list(udef.values())
                udef_char    = list(udef.keys())
                udef_real_h5 = np.array(udef_real, dtype=np.float64)
                udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
                
                self.create_dataset('header/udef_real', data=udef_real_h5, maxshape=np.shape(udef_real_h5), dtype=np.float64)
                self.create_dataset('header/udef_char', data=udef_char_h5, maxshape=np.shape(udef_char_h5), dtype='S128')
                self.udef      = udef
                self.udef_real = udef_real
                self.udef_char = udef_char
            
            # === copy over spatial dim info
            
            x, y, z = hf_ref.x, hf_ref.y, hf_ref.z
            nx  = self.nx  = x.size
            ny  = self.ny  = y.size
            nz  = self.nz  = z.size
            ngp = self.ngp = nx*ny*nz
            if ('dims/x' in self):
                del self['dims/x']
            if ('dims/y' in self):
                del self['dims/y']
            if ('dims/z' in self):
                del self['dims/z']
            
            self.create_dataset('dims/x', data=x)
            self.create_dataset('dims/y', data=y)
            self.create_dataset('dims/z', data=z)
            
            # === copy over temporal dim info
            
            if t_info:
                self.t  = hf_ref.t
                self.nt = self.t.size
                self.create_dataset('dims/t', data=hf_ref.t)
            else:
                t = np.array([0.], dtype=np.float64)
                if ('dims/t' in self):
                    del self['dims/t']
                self.create_dataset('dims/t', data=t)
            
            # ===
            
            ## copy over [data_dim/<>] dsets if present
            if ('data_dim' in hf_ref):
                for dsn in hf_ref['data_dim'].keys():
                    data = np.copy( hf_ref[f'data_dim/{dsn}'][()] ) 
                    self.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                    if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False)
        return
    
    def import_eas4(self, fn_eas4_list, **kwargs):
        '''
        import data from a series of EAS4 files to a RGD
        '''
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        if verbose: print('\n'+'rgd.import_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        tt_min = kwargs.get('tt_min',None)
        tt_max = kwargs.get('tt_max',None)
        
        ## dont actually copy over data, just initialize datasets with 0's
        init_dsets_only = kwargs.get('init_dsets_only',False)
        
        ## delete EAS4s after import --> DANGER!
        delete_after_import = kwargs.get('delete_after_import',False)
        
        ## if you're just initializing, don't allow delete too
        if (init_dsets_only and delete_after_import):
            raise ValueError("if init_dsets_only=True, then delete_after_import should not be activated!")
        
        ## delete only allowed if no time ranges are selected
        if delete_after_import and any([(ti_min is not None),(ti_max is not None),(tt_min is not None),(tt_max is not None)]):
            raise ValueError("if delete_after_import=True, then ti_min,ti_max,tt_min,tt_max are not supported")
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        ## float precision when copying
        ## default is 'single' i.e. cast data to single
        ## 'same' will preserve the floating point precision from the EAS4 file
        prec = kwargs.get('prec',None)
        if (prec is None):
            prec = 'single'
        elif (prec=='single'):
            pass
        elif (prec=='same'):
            pass
        else:
            raise ValueError('prec not set correctly')
        
        ## HDF5 chunk parameters (t,z,y,x)
        if (chunk_constraint is None):
            chunk_constraint = (1,None,None,None) ## single [t] convention
            #chunk_constraint = (None,-1,1,-1) ## single [y] convention
        if (chunk_base is None):
            chunk_base = 2
        
        ## check for an often made mistake
        ## 'ts_min' / 'ts_max' should NOT be allowed as inputs
        ts_min = kwargs.get('ts_min',None)
        ts_max = kwargs.get('ts_max',None)
        if (ts_min is not None):
            raise AssertionError('ts_min is not an option --> did you mean ti_min or tt_min?')
        if (ts_max is not None):
            raise AssertionError('ts_max is not an option --> did you mean ti_max or tt_max?')
        del ts_min
        del ts_max
        
        ## check that the passed iterable of EAS4 files is OK
        if not hasattr(fn_eas4_list, '__iter__'):
            raise AssertionError('first arg \'fn_eas4_list\' must be iterable')
        for fn_eas4 in fn_eas4_list:
            if not os.path.isfile(fn_eas4):
                raise FileNotFoundError('%s not found!'%fn_eas4)
        
        ## ranks
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        ## skip dimensions --> spatial skips done in init_from_XXX()
        # sx = kwargs.get('sx',1)
        # sy = kwargs.get('sy',1)
        # sz = kwargs.get('sz',1)
        st = kwargs.get('st',1)
        
        ## update this RGD's header and attributes
        self.get_header(verbose=False)
        
        comm_eas4 = MPI.COMM_WORLD ## communicator for opening EAS4s
        
        ### # === get all time info & check --> having every process open in MPI mode is not optimal
        ### t = np.array([], dtype=np.float64)
        ### for fn_eas4 in fn_eas4_list:
        ###     with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
        ###         t = np.concatenate((t, hf_eas4.t))
        ### comm_eas4.Barrier()
        
        # === get all time info & check
        if (self.rank==0):
            t = np.array([], dtype=np.float64)
            for fn_eas4 in fn_eas4_list:
                with eas4(fn_eas4, 'r', verbose=False) as hf_eas4:
                    t_ = np.copy(hf_eas4.t)
                t = np.concatenate((t,t_))
        else:
            t = np.array([], dtype=np.float64) ## 't' must exist on all ranks
        
        if self.usingmpi:
            self.comm.Barrier()
        
        ## broadcast concatenated time vector to all ranks
        if self.usingmpi:
            t = self.comm.bcast(t, root=0)
        
        if verbose: even_print('n EAS4 files','%i'%len(fn_eas4_list))
        if verbose: even_print('nt all files','%i'%t.size)
        if verbose: even_print('delete after import',str(delete_after_import))
        if verbose: print(72*'-')
        
        if (t.size>1):
            
            ## check no zero distance elements
            if (np.diff(t).size - np.count_nonzero(np.diff(t))) != 0.:
                raise AssertionError('t arr has zero-distance elements')
            else:
                if verbose: even_print('check: Δt!=0','passed')
            
            ## check monotonically increasing
            if not np.all(np.diff(t) > 0.):
                raise AssertionError('t arr not monotonically increasing')
            else:
                if verbose: even_print('check: t mono increasing','passed')
            
            ## check constant Δt
            dt0 = np.diff(t)[0]
            if not np.all(np.isclose(np.diff(t), dt0, rtol=1e-3)):
                if (self.rank==0): print(np.diff(t))
                raise AssertionError('t arr not uniformly spaced')
            else:
                if verbose: even_print('check: constant Δt','passed')
        
        # === get all grid info & check
        
        if ( len(fn_eas4_list) > 1 ):
            
            eas4_x_arr = []
            eas4_y_arr = []
            eas4_z_arr = []
            for fn_eas4 in fn_eas4_list:
                with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                    eas4_x_arr.append( hf_eas4.x )
                    eas4_y_arr.append( hf_eas4.y )
                    eas4_z_arr.append( hf_eas4.z )
            comm_eas4.Barrier()
            
            ## check coordinate vectors are same
            if not np.all([np.allclose(eas4_z_arr[i],eas4_z_arr[0],rtol=1e-8) for i in range(len(fn_eas4_list))]):
                raise AssertionError('EAS4 files do not have the same z coordinates')
            else:
                if verbose: even_print('check: z coordinate vectors equal','passed')
            if not np.all([np.allclose(eas4_y_arr[i],eas4_y_arr[0],rtol=1e-8) for i in range(len(fn_eas4_list))]):
                raise AssertionError('EAS4 files do not have the same y coordinates')
            else:
                if verbose: even_print('check: y coordinate vectors equal','passed')
            if not np.all([np.allclose(eas4_x_arr[i],eas4_x_arr[0],rtol=1e-8) for i in range(len(fn_eas4_list))]):
                raise AssertionError('EAS4 files do not have the same x coordinates')
            else:
                if verbose: even_print('check: x coordinate vectors equal','passed')
        
        # === [t] resolution filter (skip every n timesteps)
        tfi = self.tfi = np.arange(t.size, dtype=np.int64)
        if (st!=1):
            if verbose: even_print('st', '%i'%st)
            #print('>>> st : %i'%st)
            tfi = self.tfi = tfi[::st]
        
        ## initialize 'doRead' vector --> boolean vector to be updated
        doRead = np.full((t.size,), True, dtype=bool)
        
        ## skip filter
        if hasattr(self, 'tfi'):
            doRead[np.isin(np.arange(t.size),self.tfi,invert=True)] = False
        
        ## min/max index filter
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
            doRead[:ti_min] = False
        if (ti_max is not None):
            if not isinstance(ti_max, int):
                raise TypeError('ti_max must be type int')
            doRead[ti_max:] = False
        
        if (tt_min is not None):
            if (tt_min>=0.):
                doRead[np.where((t-t.min())<tt_min)] = False
            elif (tt_min<0.):
                doRead[np.where((t-t.max())<tt_min)] = False
        
        if (tt_max is not None):
            if (tt_max>=0.):
                doRead[np.where((t-t.min())>tt_max)] = False
            elif (tt_max<0.):
                doRead[np.where((t-t.max())>tt_max)] = False
        
        # === RGD times
        self.t  = np.copy(t[doRead])
        self.nt = self.t.size
        self.ti = np.arange(self.nt, dtype=np.int64)
        
        # === write back time vector 'self.t' to file as 'dims/t'
        
        ## before the write, if appending, check that time vector is the same
        if (self.open_mode=='a') or (self.open_mode=='r+'):
            if ('dims/t' in self):
                t_ = np.copy(self['dims/t'][()])
                np.testing.assert_allclose(t_, self.t, rtol=1e-14, atol=1e-14)
        
        if ('dims/t' in self):
            del self['dims/t']
        self.create_dataset('dims/t', data=self.t)
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === determine RGD scalars (from EAS4 scalars)
        if not hasattr(self, 'scalars') or (len(self.scalars)==0):
            
            with eas4(fn_eas4_list[0], 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                self.scalars   = hf_eas4.scalars
                self.n_scalars = len(self.scalars)
                
                ## decide dtypes
                for scalar in hf_eas4.scalars:
                    
                    domainName = hf_eas4.domainName
                    ti = 0
                    dset_path = 'Data/%s/ts_%06d/par_%06d'%(domainName,ti,hf_eas4.scalar_n_map[scalar])
                    dset = hf_eas4[dset_path]
                    dtype = dset.dtype
                    
                    if (prec=='same'):
                        self.scalars_dtypes_dict[scalar] = dtype
                    elif (prec=='single'):
                        if (dtype!=np.float32) and (dtype!=np.float64): ## make sure its either a single or double float
                            raise ValueError
                        self.scalars_dtypes_dict[scalar] = np.dtype(np.float32)
                    else:
                        raise ValueError
        
        if self.usingmpi: comm_eas4.Barrier()
        
        # === initialize datasets
        
        for scalar in self.scalars:
            
            dtype       = self.scalars_dtypes_dict[scalar]
            float_bytes = dtype.itemsize
            data_gb     = float_bytes*self.nt*self.nz*self.ny*self.nx / 1024**3
            shape       = (self.nt,self.nz,self.ny,self.nx)
            chunks      = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
            
            do_dset_initialize = True ## default value which, if conditions are met, will be turned False
            
            dsn = f'data/{scalar}'
            
            ## if exists, check size & chunks & dtype
            if (self.open_mode=='a') or (self.open_mode=='r+'):
                if (dsn in self):
                    
                    dset = self[dsn]
                    if verbose: even_print(f'dset {dsn} already exists', str(True))
                    
                    if ( dset.shape == shape ):
                        if verbose: even_print(f'dset {dsn} shape matches', str(True))
                    else:
                        if verbose: even_print(f'dset {dsn} shape matches', str(False))
                    
                    if ( dset.chunks == chunks ):
                        if verbose: even_print(f'dset {dsn} chunks match', str(True))
                    else:
                        if verbose: even_print(f'dset {dsn} chunks match', str(False))
                    
                    if ( dset.dtype == dtype ):
                        if verbose: even_print(f'dset {dsn} dtype matches', str(True))
                    else:
                        if verbose: even_print(f'dset {dsn} dtype matches', str(False))
                    
                    if ( dset.shape == shape ) and ( dset.chunks == chunks ) and ( dset.dtype == dtype ):
                        do_dset_initialize = False
            
            if do_dset_initialize:
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if verbose:
                    even_print(f'initializing data/{scalar}', f'{data_gb:0.2f} [GB]')
                
                dset = self.create_dataset(dsn, 
                                           shape=shape, 
                                           dtype=dtype,
                                           chunks=chunks,
                                           )
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]')
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)', str(dset.chunks))
                even_print('chunk size', f'{int(round(chunk_kb_)):d} [KB]')
        
        if verbose: print(72*'-')
        
        # === report size of RGD after initialization
        if verbose: tqdm.write(even_print(os.path.basename(self.fname), '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3), s=True))
        if verbose: print(72*'-')
        
        # === open EAS4s, read, write to RGD
        
        if not init_dsets_only:
            
            if verbose:
                progress_bar = tqdm(
                    total=self.nt*self.n_scalars,
                    ncols=100,
                    desc='import',
                    smoothing=0.,
                    leave=False,
                    file=sys.stdout,
                    bar_format="{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}\n\033[F\r",
                    ascii="░█",
                    )
            
            data_gb_read  = 0.
            data_gb_write = 0.
            t_read  = 0.
            t_write = 0.
            
            tii  = -1 ## counter full series
            tiii = -1 ## counter RGD-local
            for fn_eas4 in fn_eas4_list:
                with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                    
                    if verbose: tqdm.write(even_print(os.path.basename(fn_eas4), '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3), s=True))
                    
                    # if verbose: tqdm.write(even_print('gmode_dim1' , '%i'%hf_eas4.gmode_dim1  , s=True))
                    # if verbose: tqdm.write(even_print('gmode_dim2' , '%i'%hf_eas4.gmode_dim2  , s=True))
                    # if verbose: tqdm.write(even_print('gmode_dim3' , '%i'%hf_eas4.gmode_dim3  , s=True))
                    
                    if verbose: tqdm.write(even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1_orig, gmode_dict[hf_eas4.gmode_dim1_orig] ), s=True ))
                    if verbose: tqdm.write(even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2_orig, gmode_dict[hf_eas4.gmode_dim2_orig] ), s=True ))
                    if verbose: tqdm.write(even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3_orig, gmode_dict[hf_eas4.gmode_dim3_orig] ), s=True ))
                    
                    if verbose: tqdm.write(even_print('duration' , '%0.2f'%hf_eas4.duration , s=True))
                    
                    # === write buffer
                    
                    # ## 5D [scalar][x,y,z,t] structured array
                    # buff = np.zeros(shape=(nxr, nyr, nzr, bt), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
                    
                    # ===
                    
                    #domainName = 'DOMAIN_000000' ## only one domain supported
                    domainName = hf_eas4.domainName
                    
                    for ti in range(hf_eas4.nt):
                        tii += 1 ## EAS4 series counter
                        if doRead[tii]:
                            tiii += 1 ## RGD counter
                            for scalar in hf_eas4.scalars:
                                if (scalar in self.scalars):
                                    
                                    # === collective read
                                    
                                    dset_path = 'Data/%s/ts_%06d/par_%06d'%(domainName,ti,hf_eas4.scalar_n_map[scalar])
                                    dset = hf_eas4[dset_path]
                                    
                                    if hf_eas4.usingmpi: comm_eas4.Barrier()
                                    t_start = timeit.default_timer()
                                    if hf_eas4.usingmpi: 
                                        with dset.collective:
                                            data = np.copy( dset[rx1:rx2,ry1:ry2,rz1:rz2] )
                                    else:
                                        data = np.copy( dset[()] )
                                    if hf_eas4.usingmpi: comm_eas4.Barrier()
                                    t_delta = timeit.default_timer() - t_start
                                    
                                    data_gb       = data.nbytes / 1024**3
                                    t_read       += t_delta
                                    data_gb_read += data_gb
                                    
                                    if False:
                                        if verbose:
                                            txt = even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                            tqdm.write(txt)
                                    
                                    ## reduce precision (e.g. for restart which is usually double precision)
                                    # if (data.dtype == np.float64):
                                    #     data = np.copy( data.astype(np.float32) )
                                    
                                    ## reduce precision (if kwarg prec='single' and the incoming data is double)
                                    if (data.dtype != self.scalars_dtypes_dict[scalar]):
                                        data = np.copy( data.astype(self.scalars_dtypes_dict[scalar]) )
                                    
                                    data_gb = data.nbytes / 1024**3
                                    
                                    # === collective write
                                    
                                    dset = self['data/%s'%scalar]
                                    
                                    if self.usingmpi: self.comm.Barrier()
                                    t_start = timeit.default_timer()
                                    if self.usingmpi:
                                        with dset.collective:
                                            dset[tiii,rz1:rz2,ry1:ry2,rx1:rx2] = data.T
                                    else:
                                        
                                        if self.hasGridFilter:
                                            data = data[self.xfi[:,np.newaxis,np.newaxis],
                                                        self.yfi[np.newaxis,:,np.newaxis],
                                                        self.zfi[np.newaxis,np.newaxis,:]]
                                        
                                        dset[tiii,:,:,:] = data.T
                                    
                                    if self.usingmpi: self.comm.Barrier()
                                    t_delta = timeit.default_timer() - t_start
                                    
                                    t_write       += t_delta
                                    data_gb_write += data_gb
                                    
                                    if False:
                                        if verbose:
                                            txt = even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                            tqdm.write(txt)
                                    
                                    if verbose: progress_bar.update()
                
                ## delete source EAS4 file
                if delete_after_import:
                    if (self.rank==0):
                        tqdm.write(even_print('deleting', fn_eas4, s=True))
                        os.remove(fn_eas4)
                    self.comm.Barrier()
            
            if verbose: progress_bar.close()
            
            if hf_eas4.usingmpi: comm_eas4.Barrier()
            if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False)
        
        ## get read read/write totals all ranks
        if not init_dsets_only:
            if self.usingmpi:
                G = self.comm.gather([data_gb_read, data_gb_write, self.rank], root=0)
                G = self.comm.bcast(G, root=0)
                data_gb_read  = sum([x[0] for x in G])
                data_gb_write = sum([x[1] for x in G])
        
        if init_dsets_only:
            if verbose: print('>>> init_dsets_only=True, so no EAS4 data was imported')
        
        if verbose: print(72*'-')
        if verbose: even_print('nt',       '%i'%self.nt )
        if verbose: even_print('dt',       '%0.8f'%self.dt )
        if verbose: even_print('duration', '%0.2f'%self.duration )
        
        if not init_dsets_only:
            if verbose: print(72*'-')
            if verbose: even_print('time read',format_time_string(t_read))
            if verbose: even_print('time write',format_time_string(t_write))
            if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
            if verbose: even_print('read total avg', f'{data_gb_read:0.2f} [GB]  {t_read:0.2f} [s]  {(data_gb_read/t_read):0.3f} [GB/s]')
            if verbose: even_print('write total avg', f'{data_gb_write:0.2f} [GB]  {t_write:0.2f} [s]  {(data_gb_write/t_write):0.3f} [GB/s]')
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.import_eas4() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    @staticmethod
    def copy(fn_rgd_src, fn_rgd_tgt, **kwargs):
        '''
        copy header info, selected scalars, and [x,y,z,t] range to new RGD file
        --> this currently does NOT work in serial mode
        '''
        
        #comm    = MPI.COMM_WORLD
        rank    = MPI.COMM_WORLD.Get_rank()
        n_ranks = MPI.COMM_WORLD.Get_size()
        
        if (rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.copy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        rt       = kwargs.get('rt',1)
        force    = kwargs.get('force',False) ## overwrite or raise error if exists
        
        ti_min   = kwargs.get('ti_min',None)
        ti_max   = kwargs.get('ti_max',None)
        scalars  = kwargs.get('scalars',None)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing RGD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        xi_min = kwargs.get('xi_min',None) ## 4D coordinate 
        xi_max = kwargs.get('xi_max',None)
        yi_min = kwargs.get('yi_min',None)
        yi_max = kwargs.get('yi_max',None)
        zi_min = kwargs.get('zi_min',None)
        zi_max = kwargs.get('zi_max',None)
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        
        ct = kwargs.get('ct',1) ## 'chunks' in time
        
        xi_step = kwargs.get('xi_step',1)
        yi_step = kwargs.get('yi_step',1)
        zi_step = kwargs.get('zi_step',1)
        
        prec_coords = kwargs.get('prec_coords',None)
        if (prec_coords is None):
            prec_coords = 'same'
        elif (prec_coords=='single'):
            pass
        elif (prec_coords=='same'):
            pass
        else:
            raise ValueError('prec_coords not set correctly')
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz!=n_ranks):
            raise AssertionError('rx*ry*rz!=n_ranks')
        if not os.path.isfile(fn_rgd_src):
            raise FileNotFoundError('%s not found!'%fn_rgd_src)
        if os.path.isfile(fn_rgd_tgt) and not force:
            raise FileExistsError('%s already exists. delete it or use \'force=True\' kwarg'%fn_rgd_tgt)
        
        # ===
        
        with rgd(fn_rgd_src, 'r', comm=MPI.COMM_WORLD, driver='mpio') as hf_src:
            with rgd(fn_rgd_tgt, 'w', comm=MPI.COMM_WORLD, driver='mpio', force=force, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_tgt:
                
                ## copy over header info (source --> target)
                hf_tgt.init_from_rgd(fn_rgd_src)
                
                if (scalars is None):
                    scalars = hf_src.scalars
                
                if verbose:
                    even_print('fn_rgd_src' , fn_rgd_src )
                    even_print('nx' , '%i'%hf_src.nx )
                    even_print('ny' , '%i'%hf_src.ny )
                    even_print('nz' , '%i'%hf_src.nz )
                    even_print('nt' , '%i'%hf_src.nt )
                    if verbose: print(72*'-')
                
                if (rx>hf_src.nx):
                    raise AssertionError('rx>nx')
                if (ry>hf_src.ny):
                    raise AssertionError('ry>ny')
                if (rz>hf_src.nz):
                    raise AssertionError('rz>nz')
                if (rt>hf_src.nt):
                    raise AssertionError('rt>nt')
                
                ## for RGD, can just load full grid on every rank
                x = np.copy( hf_src.x )
                y = np.copy( hf_src.y )
                z = np.copy( hf_src.z )
                t = np.copy( hf_src.t )
                
                xi = np.arange(hf_src.nx, dtype=np.int64) ## arange index vector, doesnt get touched!
                yi = np.arange(hf_src.ny, dtype=np.int64)
                zi = np.arange(hf_src.nz, dtype=np.int64)
                ti = np.arange(hf_src.nt, dtype=np.int64)
                
                xfi = np.arange(hf_src.nx, dtype=np.int64) ## gets clipped depending on x/y/z/t_min/max opts
                yfi = np.arange(hf_src.ny, dtype=np.int64)
                zfi = np.arange(hf_src.nz, dtype=np.int64)
                tfi = np.arange(hf_src.nt, dtype=np.int64)
                
                # === total bounds clip (coordinate index) --> supports negative indexing!
                
                if True: ## code folding
                    
                    if (xi_min is not None):
                        xfi_ = []
                        if verbose:
                            if (xi_min<0):
                                even_print('xi_min', '%i / %i'%(xi_min,xi[xi_min]))
                            else:
                                even_print('xi_min', '%i'%(xi_min,))
                        for c in xfi:
                            if (xi_min<0) and (c>=(hf_src.nx+xi_min)):
                                xfi_.append(c)
                            elif (xi_min>=0) and (c>=xi_min):
                                xfi_.append(c)
                        xfi=np.array(xfi_, dtype=np.int64)
                    else:
                        xi_min = 0
                    
                    if (xi_max is not None):
                        xfi_ = []
                        if verbose:
                            if (xi_max<0):
                                even_print('xi_max', '%i / %i'%(xi_max,xi[xi_max]))
                            else:
                                even_print('xi_max', '%i'%(xi_max,))
                        for c in xfi:
                            if (xi_max<0) and (c<=(hf_src.nx+xi_max)):
                                xfi_.append(c)
                            elif (xi_max>=0) and (c<=xi_max):
                                xfi_.append(c)
                        xfi=np.array(xfi_, dtype=np.int64)
                    else:
                        xi_max = xi[-1]
                    
                    ## check x
                    if ((xi[xi_max]-xi[xi_min]+1)<1):
                        raise ValueError('invalid xi range requested')
                    if (rx>(xi[xi_max]-xi[xi_min]+1)):
                        raise ValueError('more ranks than grid points in x')
                    
                    if (yi_min is not None):
                        yfi_ = []
                        if verbose:
                            if (yi_min<0):
                                even_print('yi_min', '%i / %i'%(yi_min,yi[yi_min]))
                            else:
                                even_print('yi_min', '%i'%(yi_min,))
                        for c in yfi:
                            if (yi_min<0) and (c>=(hf_src.ny+yi_min)):
                                yfi_.append(c)
                            elif (yi_min>=0) and (c>=yi_min):
                                yfi_.append(c)
                        yfi=np.array(yfi_, dtype=np.int64)
                    else:
                        yi_min = 0
                    
                    if (yi_max is not None):
                        yfi_ = []
                        if verbose:
                            if (yi_max<0):
                                even_print('yi_max', '%i / %i'%(yi_max,yi[yi_max]))
                            else:
                                even_print('yi_max', '%i'%(yi_max,))
                        for c in yfi:
                            if (yi_max<0) and (c<=(hf_src.ny+yi_max)):
                                yfi_.append(c)
                            elif (yi_max>=0) and (c<=yi_max):
                                yfi_.append(c)
                        yfi=np.array(yfi_, dtype=np.int64)
                    else:
                        yi_max = yi[-1]
                    
                    ## check y
                    if ((yi[yi_max]-yi[yi_min]+1)<1):
                        raise ValueError('invalid yi range requested')
                    if (ry>(yi[yi_max]-yi[yi_min]+1)):
                        raise ValueError('more ranks than grid points in y')
                    
                    if (zi_min is not None):
                        zfi_ = []
                        if verbose:
                            if (zi_min<0):
                                even_print('zi_min', '%i / %i'%(zi_min,zi[zi_min]))
                            else:
                                even_print('zi_min', '%i'%(zi_min,))
                        for c in zfi:
                            if (zi_min<0) and (c>=(hf_src.nz+zi_min)):
                                zfi_.append(c)
                            elif (zi_min>=0) and (c>=zi_min):
                                zfi_.append(c)
                        zfi=np.array(zfi_, dtype=np.int64)
                    else:
                        zi_min = 0
                    
                    if (zi_max is not None):
                        zfi_ = []
                        if verbose:
                            if (zi_max<0):
                                even_print('zi_max', '%i / %i'%(zi_max,zi[zi_max]))
                            else:
                                even_print('zi_max', '%i'%(zi_max,))
                        for c in zfi:
                            if (zi_max<0) and (c<=(hf_src.nz+zi_max)):
                                zfi_.append(c)
                            elif (zi_max>=0) and (c<=zi_max):
                                zfi_.append(c)
                        zfi=np.array(zfi_, dtype=np.int64)
                    else:
                        zi_max = zi[-1]
                    
                    ## check z
                    if ((zi[zi_max]-zi[zi_min]+1)<1):
                        raise ValueError('invalid zi range requested')
                    if (rz>(zi[zi_max]-zi[zi_min]+1)):
                        raise ValueError('more ranks than grid points in z')
                    
                    if (ti_min is not None):
                        tfi_ = []
                        if verbose:
                            if (ti_min<0):
                                even_print('ti_min', '%i / %i'%(ti_min,ti[ti_min]))
                            else:
                                even_print('ti_min', '%i'%(ti_min,))
                        for c in tfi:
                            if (ti_min<0) and (c>=(hf_src.nt+ti_min)):
                                tfi_.append(c)
                            elif (ti_min>=0) and (c>=ti_min):
                                tfi_.append(c)
                        tfi=np.array(tfi_, dtype=np.int64)
                    else:
                        ti_min = 0
                    
                    if (ti_max is not None):
                        tfi_ = []
                        if verbose:
                            if (ti_max<0):
                                even_print('ti_max', '%i / %i'%(ti_max,ti[ti_max]))
                            else:
                                even_print('ti_max', '%i'%(ti_max,))
                        for c in tfi:
                            if (ti_max<0) and (c<=(hf_src.nt+ti_max)):
                                tfi_.append(c)
                            elif (ti_max>=0) and (c<=ti_max):
                                tfi_.append(c)
                        tfi=np.array(tfi_, dtype=np.int64)
                    else:
                        ti_max = ti[-1]
                    
                    ## check t
                    if ((ti[ti_max]-ti[ti_min]+1)<1):
                        raise ValueError('invalid ti range requested')
                    if (ct>(ti[ti_max]-ti[ti_min]+1)):
                        raise ValueError('more chunks than timesteps')
                
                # === 3D/4D communicator
                
                comm4d = hf_src.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                t4d = comm4d.Get_coords(rank)
                
                rxl_ = np.array_split(xfi,rx)
                ryl_ = np.array_split(yfi,ry)
                rzl_ = np.array_split(zfi,rz)
                #rtl_ = np.array_split(tfi,rt)
                
                rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                
                ## the rank-local bounds for READ --> takes into acct clip but not step!
                rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
                
                ## the global dim sizes for READ
                nx_read = xfi.shape[0]
                ny_read = yfi.shape[0]
                nz_read = zfi.shape[0]
                
                # === global step
                
                ## take every nth index (of the already bounds-clipped) index-to-take vector
                xfi = np.copy(xfi[::xi_step])
                yfi = np.copy(yfi[::yi_step])
                zfi = np.copy(zfi[::zi_step])
                
                ## the global dim sizes for WRITE
                nx = xfi.shape[0]
                ny = yfi.shape[0]
                nz = zfi.shape[0]
                
                # ===
                
                ## grid for target file (rectilinear case)
                x = np.copy(x[xfi]) ## target file
                y = np.copy(y[yfi])
                z = np.copy(z[zfi])
                t = np.copy(t[tfi])
                
                nx = x.shape[0] ## target file
                ny = y.shape[0]
                nz = z.shape[0]
                nt = t.shape[0]
                
                if verbose:
                    even_print('fn_rgd_tgt' , fn_rgd_tgt )
                    even_print('nx' , '%i'%nx )
                    even_print('ny' , '%i'%ny )
                    even_print('nz' , '%i'%nz )
                    even_print('nt' , '%i'%nt )
                    print(72*'-')
                
                ## replace coordinate dimension arrays in target file
                if ('dims/x' in hf_tgt):
                    del hf_tgt['dims/x']
                    hf_tgt.create_dataset('dims/x', data=x, dtype=np.float64, chunks=None)
                if ('dims/y' in hf_tgt):
                    del hf_tgt['dims/y']
                    hf_tgt.create_dataset('dims/y', data=y, dtype=np.float64, chunks=None)
                if ('dims/z' in hf_tgt):
                    del hf_tgt['dims/z']
                    hf_tgt.create_dataset('dims/z', data=z, dtype=np.float64, chunks=None)
                if ('dims/t' in hf_tgt):
                    del hf_tgt['dims/t']
                    hf_tgt.create_dataset('dims/t', data=t, dtype=np.float64, chunks=None)
                
                ## write filter index arrays to file
                if ('filters/xfi' in hf_tgt):
                    del hf_tgt['filters/xfi']
                    hf_tgt.create_dataset('filters/xfi', data=xfi, dtype=np.int64, chunks=None)
                if ('filters/yfi' in hf_tgt):
                    del hf_tgt['filters/yfi']
                    hf_tgt.create_dataset('filters/yfi', data=yfi, dtype=np.int64, chunks=None)
                if ('filters/zfi' in hf_tgt):
                    del hf_tgt['filters/zfi']
                    hf_tgt.create_dataset('filters/zfi', data=zfi, dtype=np.int64, chunks=None)
                
                # === bounds for outfile WRITE
                
                xiw     = np.array( [ i for i in xfi if all([(i>=rx1),(i<rx2)]) ], dtype=np.int32 ) ## the global indices in my local rank, taking into acct clip AND step
                nxiw    = xiw.shape[0]
                xiw_off = len([ i for i in xfi if (i<rx1) ]) ## this rank's left offset in the OUTFILE context
                rx1w    = xiw_off
                rx2w    = xiw_off + nxiw
                
                yiw     = np.array( [ i for i in yfi if all([(i>=ry1),(i<ry2)]) ], dtype=np.int32 )
                nyiw    = yiw.shape[0]
                yiw_off = len([ i for i in yfi if (i<ry1) ])
                ry1w    = yiw_off
                ry2w    = yiw_off + nyiw
                
                ziw     = np.array( [ i for i in zfi if all([(i>=rz1),(i<rz2)]) ], dtype=np.int32 )
                nziw    = ziw.shape[0]
                ziw_off = len([ i for i in zfi if (i<rz1) ])
                rz1w    = ziw_off
                rz2w    = ziw_off + nziw
                
                ## xiw,yiw,ziw are used to 'filter' the rank-local data that is read in
                ## xiw,yiw,ziw are currently in the global context, so we need to subtract off the left READ bound
                ## which is NOT just the min xiw
                xiw -= rx1
                yiw -= ry1
                ziw -= rz1
                
                # ===
                
                ## time 'chunks' split (number of timesteps to read / write at a time)
                ctl_ = np.array_split(tfi,ct)
                ctl = [[b[0],b[-1]+1] for b in ctl_ ]
                
                shape  = (nt,nz,ny,nx) ## target
                hf_tgt.scalars = []
                
                ## initialize scalar datasets
                t_start = timeit.default_timer()
                for scalar in hf_src.scalars:
                    
                    dtype = hf_src.scalars_dtypes_dict[scalar]
                    float_bytes = dtype.itemsize
                    
                    chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                    data_gb = float_bytes * nx * ny * nz * nt / 1024**3
                    
                    if (scalar in scalars):
                        if verbose:
                            even_print('initializing data/%s'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        dset = hf_tgt.create_dataset(
                                                    'data/%s'%scalar,
                                                    shape=shape,
                                                    dtype=dtype,
                                                    chunks=chunks,
                                                    )
                        hf_tgt.scalars.append(scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                t_initialize = timeit.default_timer() - t_start
                if verbose:
                    even_print('time initialize',format_time_string(t_initialize))
                    print(72*'-')
                
                # ===
                
                hf_tgt.n_scalars = len(hf_tgt.scalars)
                
                # ===
                
                data_gb_read  = 0.
                data_gb_write = 0.
                t_read  = 0.
                t_write = 0.
                
                if verbose:
                    progress_bar = tqdm(total=len(ctl)*hf_tgt.n_scalars, ncols=100, desc='copy', leave=False, file=sys.stdout)
                
                for scalar in hf_tgt.scalars:
                    dset_src = hf_src[f'data/{scalar}']
                    dset_tgt = hf_tgt[f'data/{scalar}']
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    for ctl_ in ctl:
                        
                        ct1, ct2 = ctl_
                        
                        ct1_ = ct1 - ti[ti_min] ## coords in target file
                        ct2_ = ct2 - ti[ti_min]
                        
                        ## read
                        hf_src.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_src.collective:
                            data = dset_src[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                        hf_src.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * nx_read * ny_read * nz_read * (ct2-ct1) / 1024**3
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        if verbose:
                            tqdm.write(even_print('read: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        try:
                            data_out = np.copy( data[ np.ix_(xiw,yiw,ziw) ] )
                            #data_out = np.copy( data[ xiw[:,np.newaxis,np.newaxis], yiw[np.newaxis,:,np.newaxis], ziw[np.newaxis,np.newaxis,:] ] )
                        except:
                            print('rgd.copy() : error in xiw,yiw,ziw')
                            MPI.COMM_WORLD.Abort(1)
                        
                        ## write
                        hf_tgt.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_tgt.collective:
                            dset_tgt[ct1_:ct2_,rz1w:rz2w,ry1w:ry2w,rx1w:rx2w] = data_out.T
                        hf_tgt.flush()
                        hf_tgt.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        #data_gb = n_ranks * data.nbytes / 1024**3 ## approximate
                        data_gb = float_bytes*nx*ny*nz*(ct2-ct1) / 1024**3
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            tqdm.write(even_print('write: %s'%scalar, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        if verbose:
                            progress_bar.update()
                
                if verbose:
                    progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: even_print('time initialize',format_time_string(t_initialize))
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_rgd_src, '%0.2f [GB]'%(os.path.getsize(fn_rgd_src)/1024**3))
        if verbose: even_print(fn_rgd_tgt, '%0.2f [GB]'%(os.path.getsize(fn_rgd_tgt)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.copy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def delete_scalars(self,scalar_list,**kwargs):
        '''
        delete scalars from RGD 
        '''
        pass
        return
    
    def read(self,**kwargs):
        '''
        read data from file & return structured array
        '''
        
        verbose_master = kwargs.get('verbose',True)
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose:
            verbose = verbose_master
        
        if verbose: print('\n'+'rgd.read()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print(self.fname,'%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: print(72*'-')
        
        rx       = kwargs.get('rx',1)
        ry       = kwargs.get('ry',1)
        rz       = kwargs.get('rz',1)
        rt       = kwargs.get('rt',1)
        scalars_to_read = kwargs.get('scalars',None)
        
        if (rx*ry*rz*rt!=self.n_ranks):
            raise AssertionError('rx*ry*rz*rt!=self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if (scalars_to_read is None):
            scalars_to_read = self.scalars
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d    = comm4d.Get_coords(self.rank)
            ##
            rxl_   = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_   = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_   = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            rtl_   = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            rxl    = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl    = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl    = [[b[0],b[-1]+1] for b in rzl_ ]
            rtl    = [[b[0],b[-1]+1] for b in rtl_ ]
            ##
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            ntr = self.nt
        
        t_read = 0.
        data_gb_read = 0.
        
        data_gb = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        
        # ===
        
        names   = [ s for s in scalars_to_read if s in self.scalars ]
        formats = [ self.scalars_dtypes_dict[n] for n in names ]
        
        ## 5D [scalar][x,y,z,t] structured array
        data = np.zeros(shape=(nxr,nyr,nzr,ntr), dtype={'names':names, 'formats':formats})
        
        for scalar in names:
            
            # === collective read
            dset = self['data/%s'%scalar]
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi: 
                with dset.collective:
                    data[scalar] = dset[rt1:rt2,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                data[scalar] = dset[()].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            
            if verbose:
                even_print( 'read: %s'%scalar , '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)) )
            
            t_read       += t_delta
            data_gb_read += data_gb
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('rgd.read()', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: print(72*'-')
        
        return data
    
    # === test data populators
    
    def populate_abc_flow(self, **kwargs):
        '''
        populate (unsteady) ABC flow dummy data
        -----
        https://en.wikipedia.org/wiki/Arnold%E2%80%93Beltrami%E2%80%93Childress_flow
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.populate_abc_flow()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        ##
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        
        self.nx = nx = kwargs.get('nx',100)
        self.ny = ny = kwargs.get('ny',100)
        self.nz = nz = kwargs.get('nz',100)
        self.nt = nt = kwargs.get('nt',100)
        
        data_gb = 3 * 4*nx*ny*nz*nt / 1024.**3
        if verbose: even_print(self.fname, '%0.2f [GB]'%(data_gb,))
        
        self.x = x = np.linspace(0., 2*np.pi, nx, dtype=np.float32)
        self.y = y = np.linspace(0., 2*np.pi, ny, dtype=np.float32)
        self.z = z = np.linspace(0., 2*np.pi, nz, dtype=np.float32)
        #self.t = t = np.linspace(0., 10.,     nt, dtype=np.float32)
        self.t = t = 0.1 * np.arange(nt, dtype=np.float32)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        # ===
        
        comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
        t4d = comm4d.Get_coords(self.rank)
        
        rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ## per-rank dim range
        xr = x[rx1:rx2]
        yr = y[ry1:ry2]
        zr = z[rz1:rz2]
        #tr = t[rt1:rt2]
        tr = np.copy(t)
        
        ## write dims
        self.create_dataset('dims/x', data=x)
        self.create_dataset('dims/y', data=y)
        self.create_dataset('dims/z', data=z)
        self.create_dataset('dims/t', data=t)
        
        shape  = (self.nt,self.nz,self.ny,self.nx)
        chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, base=4, itemsize=4)
        
        ## initialize
        data_gb = 4*nx*ny*nz*nt / 1024.**3
        for scalar in ['u','v','w']:
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            if verbose:
                even_print('initializing data/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            dset = self.create_dataset('data/%s'%scalar, 
                                        shape=shape,
                                        dtype=np.float32,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # === make 4D ABC flow data
        
        t_start = timeit.default_timer()
        A = np.sqrt(3)
        B = np.sqrt(2)
        C = 1.
        na = np.newaxis
        u = (A + 0.5 * tr[na,na,na,:] * np.sin(np.pi*tr[na,na,na,:])) * np.sin(zr[na,na,:,na]) + \
            B * np.cos(yr[na,:,na,na]) + \
            0.*xr[:,na,na,na]
        v = B * np.sin(xr[:,na,na,na]) + \
            C * np.cos(zr[na,na,:,na]) + \
            0.*yr[na,:,na,na] + \
            0.*tr[na,na,na,:]
        w = C * np.sin(yr[na,:,na,na]) + \
            (A + 0.5 * tr[na,na,na,:] * np.sin(np.pi*tr[na,na,na,:])) * np.cos(xr[:,na,na,na]) + \
            0.*zr[na,na,:,na]
        
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('calc flow','%0.3f [s]'%(t_delta,))
        
        # ===
        
        data_gb = 4*nx*ny*nz*nt / 1024.**3
        
        self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['data/u']
        with ds.collective:
            ds[:,rz1:rz2,ry1:ry2,rx1:rx2] = u.T
        self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: u','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['data/v']
        with ds.collective:
            ds[:,rz1:rz2,ry1:ry2,rx1:rx2] = v.T
        self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: v','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        self.comm.Barrier()
        t_start = timeit.default_timer()
        ds = self['data/w']
        with ds.collective:
            ds[:,rz1:rz2,ry1:ry2,rx1:rx2] = w.T
        self.comm.Barrier()
        t_delta = timeit.default_timer() - t_start
        if verbose: even_print('write: w','%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
        
        # ===
        
        if verbose: print('\n'+72*'-')
        if verbose: print('total time : rgd.populate_abc_flow() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def populate_white_noise(self, **kwargs):
        '''
        populate white noise dummy data
        --> hardcoded single precision output
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.populate_white_noise()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        N = kwargs.get('N',1) ## number of timesteps to write at a time
        
        chunk_kb         = kwargs.get('chunk_kb',2*1024)
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None))
        chunk_base       = kwargs.get('chunk_base',2)
        
        self.nx = nx = kwargs.get('nx',128)
        self.ny = ny = kwargs.get('ny',128)
        self.nz = nz = kwargs.get('nz',128)
        self.nt = nt = kwargs.get('nt',128)
        
        if not isinstance(N, int):
            raise TypeError('N must be type int')
        if (self.nt%N !=0 ):
            raise ValueError(f'{self.nt:d}%{N:d}!=0')
        
        #data_gb = 3 * 4*nx*ny*nz*nt / 1024.**3
        data_gb = 1 * 4*nx*ny*nz*nt / 1024.**3
        
        if verbose: even_print(self.fname, '%0.2f [GB]'%(data_gb,))
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('rt','%i'%rt)
        if verbose: print(72*'-')
        
        self.x = x = np.linspace(0., 2*np.pi, nx, dtype=np.float32)
        self.y = y = np.linspace(0., 2*np.pi, ny, dtype=np.float32)
        self.z = z = np.linspace(0., 2*np.pi, nz, dtype=np.float32)
        #self.t = t = np.linspace(0., 10.,     nt, dtype=np.float32)
        self.t = t = 0.1 * np.arange(nt, dtype=np.float32)
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d    = comm4d.Get_coords(self.rank)
            
            rxl_   = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_   = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_   = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            rtl_   = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            rxl    = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl    = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl    = [[b[0],b[-1]+1] for b in rzl_ ]
            rtl    = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
            
            ## ## per-rank dim range
            ## xr = x[rx1:rx2]
            ## yr = y[ry1:ry2]
            ## zr = z[rz1:rz2]
            ## tr = t[rt1:rt2]
        
        else:
            nxr = nx
            nyr = ny
            nzr = nz
            ntr = nt
        
        ## write dims (independent)
        self.create_dataset('dims/x', data=x, chunks=None)
        self.create_dataset('dims/y', data=y, chunks=None)
        self.create_dataset('dims/z', data=z, chunks=None)
        self.create_dataset('dims/t', data=t, chunks=None)
        
        shape  = (self.nt,self.nz,self.ny,self.nx)
        float_bytes = 4
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        #self.scalars = ['u','v','w']
        self.scalars = ['u']
        self.scalars_dtypes = [ np.dtype(np.float32) for s in self.scalars ]
        
        ## initialize datasets
        data_gb = 4*nx*ny*nz*nt / 1024.**3
        
        if self.usingmpi: self.comm.Barrier()
        t_start_initialize = timeit.default_timer()
        
        for scalar in self.scalars:
            
            if ('data/%s'%scalar in self):
                del self['data/%s'%scalar]
            if verbose:
                even_print('initializing data/%s'%(scalar,),'%0.2f [GB]'%(data_gb,))
            
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            
            dset = self.create_dataset(
                                    f'data/{scalar}', 
                                    shape=shape,
                                    dtype=np.float32,
                                    chunks=chunks,
                                    )
            
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            
            if verbose: even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]')
            
            chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if self.usingmpi: self.comm.Barrier()
        t_initialize = timeit.default_timer() - t_start_initialize
        
        if 1: ## write N ts at a time
            
            data_gb_write = 0.
            t_write = 0.
            
            rng = np.random.default_rng(seed=self.rank) ## random number generator
            data = np.zeros(shape=(nxr,nyr,nzr,N), dtype=np.float32)
            
            if verbose:
                progress_bar = tqdm(total=len(self.scalars)*(nt//N), ncols=100, desc='write', leave=False, file=sys.stdout, smoothing=0.)
            
            for scalar in self.scalars:
                for ti in range(nt//N):
                    
                    cy1 = ti * N
                    cy2 = (ti+1) * N
                    
                    data[:,:,:,:] = rng.uniform(-1, +1, size=(nxr,nyr,nzr,N)).astype(np.float32)
                    
                    ds = self[f'data/{scalar}']
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with ds.collective:
                            ds[cy1:cy2,rz1:rz2,ry1:ry2,rx1:rx2] = data.T
                    else:
                        ds[cy1:cy2,:,:,:] = data.T
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = 4*nx*ny*nz*N / 1024**3
                    
                    t_write       += t_delta
                    data_gb_write += data_gb
                    
                    if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: even_print('time initialize',format_time_string(t_initialize))
        if verbose: even_print('write total', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.populate_white_noise() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # === post-processing
    
    def get_mean(self, **kwargs):
        '''
        get mean in [t] --> leaves [x,y,z,1]
        --> save to new RGD file
        -----
        - this version uses accumulator buffers and does *(1/n) at end to calculate mean
        - this allows for low RAM usage, as the time dim can be sub-chunked (ct=N)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.get_mean()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_rgd_mean  = kwargs.get('fn_rgd_mean',None)
        #sfm         = kwargs.get('scalars',None) ## scalars to take (for mean)
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        
        ct           = kwargs.get('ct',1)
        
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing mean file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === mean file name (for writing)
        if (fn_rgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_rgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_rgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_rgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if verbose: even_print('fn_rgd'       , self.fname   )
        if verbose: even_print('fn_rgd_mean'  , fn_rgd_mean  )
        #if verbose: even_print('fn_rgd_prime' , fn_rgd_prime )
        if verbose: even_print('do Favre avg' , str(favre)   )
        if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        t_avg_start  = self.t[ti_for_avg[0]]
        t_avg_end    = self.t[ti_for_avg[-1]]
        duration_avg = t_avg_end - t_avg_start
        
        #if not isinstance(ct, (int,np.int32,np.int64)):
        if not isinstance(ct, int):
            raise ValueError
        if (ct<1):
            raise ValueError
        
        ## [t] sub chunk range
        ctl_ = np.array_split( ti_for_avg, min(ct,nt_avg) )
        ctl = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## check that no sub ranges are <=1
        for a_ in [ ctl_[1]-ctl_[0] for ctl_ in ctl ]:
            if (a_ <= 1):
                raise ValueError
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        if verbose: even_print('Δt','%0.2f [-]'%(dt0,))
        #if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        ##data_gb      = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        #data_gb      = 4*self.nx*self.ny*self.nz*nt_avg / 1024**3
        #data_gb_mean = 4*self.nx*self.ny*self.nz*1      / 1024**3
        
        scalars_re = ['u','v','w','p','T','rho']
        scalars_fv = ['u','v','w','T'] ## 'p','rho'
        
        ## do a loop through to get names
        scalars_mean_names  = []
        scalars_mean_dtypes = []
        for scalar in self.scalars:
            dtype = self.scalars_dtypes_dict[scalar]
            if reynolds:
                if True: ## always
                    sc_name = scalar
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
            if favre:
                if (scalar in scalars_fv):
                    sc_name = f'r_{scalar}'
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
        
        #with rgd(fn_rgd_mean, 'w', force=force, driver='mpio', comm=MPI.COMM_WORLD) as hf_mean:
        with rgd(fn_rgd_mean, 'w', force=force, driver=self.driver, comm=self.comm, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_mean:
            
            ## initialize the mean file from the opened unsteady rgd file
            hf_mean.init_from_rgd(self.fname)
            
            ## set some top-level attributes
            hf_mean.attrs['duration_avg'] = duration_avg ## duration of mean
            #hf_mean.attrs['duration_avg'] = self.duration
            hf_mean.attrs['dt'] = dt0
            #hf_mean.attrs['fclass'] = 'rgd'
            hf_mean.attrs['fsubtype'] = 'mean'
            
            if verbose: print(72*'-')
            
            # === initialize mean datasets
            for scalar in self.scalars:
                
                dtype = self.scalars_dtypes_dict[scalar]
                float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                
                data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                
                shape  = (1,self.nz,self.ny,self.nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        if ('data/%s'%scalar in hf_mean):
                            del hf_mean['data/%s'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    
                    if (scalar in scalars_fv):
                        if ('data/%s_fv'%scalar in hf_mean):
                            del hf_mean['data/%s_fv'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}_fv' , f'{data_gb_mean:0.3f} [GB]' )
                        dset = hf_mean.create_dataset(f'data/{scalar}_fv',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s_fv'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            ## accumulator array for local rank --> initialize
            data_sum = np.zeros(shape=(nxr,nyr,nzr,1), dtype={'names':scalars_mean_names, 'formats':[np.float64 for _ in scalars_mean_names]})
            
            # === main loop
            
            if verbose:
                progress_bar = tqdm(total=ct, ncols=100, desc='mean', leave=False, file=sys.stdout)
            
            ct_counter=0
            for ctl_ in ctl:
                ct_counter += 1
                ct1, ct2 = ctl_
                ntc = ct2 - ct1
                
                if (ct>1):
                    if verbose:
                        mesg = f'[t] sub chunk {ct_counter:d}/{ct:d}'
                        tqdm.write( mesg )
                        tqdm.write( '-'*len(mesg) )
                
                # === read rho
                if favre:
                    
                    dset = self['data/rho']
                    
                    dtype = self.scalars_dtypes_dict['rho']
                    if (dtype!=dset.dtype):
                        raise ValueError
                    float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if self.usingmpi: 
                        with dset.collective:
                            ##rho = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            #rho = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            rho = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                    else:
                        #rho = dset[()].T
                        #rho = dset[ti_min:,:,:,:].T
                        rho = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                    data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                    
                    if verbose:
                        txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                
                # === read data, sum
                for scalar in self.scalars:
                    
                    dset = self[f'data/{scalar}']
                    dtype = self.scalars_dtypes_dict[scalar]
                    if (dtype!=dset.dtype):
                        raise ValueError
                    float_bytes = dtype.itemsize
                    
                    # === collective read
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if self.usingmpi:
                        with dset.collective:
                            ##data = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            #data = dset[ti_min:,rz1:rz2,ry1:ry2,rx1:rx2].T
                            data = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                    else:
                        ##data = dset[()].T
                        #data = dset[ti_min:,:,:,:].T
                        data = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                    data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                    
                    if verbose:
                        txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_read       += t_delta
                    data_gb_read += data_gb
                    
                    # === do sum, add to accumulator
                    if reynolds:
                        sc_name = scalar
                        data_sum[sc_name] += np.sum(data, axis=-1, dtype=np.float64, keepdims=True)
                    if favre:
                        if (scalar in scalars_fv):
                            sc_name = f'r_{scalar}'
                            data_sum[sc_name] += np.sum(data*rho, axis=-1, dtype=np.float64, keepdims=True)
                    
                    if self.usingmpi: self.comm.Barrier()
                
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                
                if verbose: progress_bar.update()
                if verbose: tqdm.write(72*'-')
            if verbose: progress_bar.close()
            
            # ==========================================================
            # multiply accumulators by (1/n)
            # ==========================================================
            
            for scalar in self.scalars:
                if reynolds:
                    sc_name = scalar
                    data_sum[sc_name] *= (1/nt_avg)
                if favre:
                    if (scalar in scalars_fv):
                        sc_name = f'r_{scalar}'
                        data_sum[sc_name] *= (1/nt_avg)
            
            # ==========================================================
            # 'data_sum' now contains averages, not sums!
            # ==========================================================
            
            ## Favre avg : φ_tilde = avg[ρ·φ]/avg[ρ]
            rho_mean = np.copy( data_sum['rho'] )
            
            # === write
            for scalar in self.scalars:
                
                if reynolds:
                    
                    dset = hf_mean[f'data/{scalar}']
                    dtype = dset.dtype
                    float_bytes = dtype.itemsize
                    
                    if (dtype==np.float32):
                        data_out = np.copy( data_sum[scalar].astype(np.float32) )
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                    else:
                        dset[:,:,:,:] = data_out.T
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                    
                    if verbose:
                        txt = even_print(f'write: {scalar}', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                        tqdm.write(txt)
                    
                    t_write       += t_delta
                    data_gb_write += data_gb_mean
                
                if favre:
                    if (scalar in scalars_fv):
                        
                        dset = hf_mean[f'data/{scalar}_fv']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        
                        ## φ_tilde = avg[ρ·φ]/avg[ρ]
                        data_out = np.copy( data_sum[f'r_{scalar}'] / rho_mean )
                        
                        if (dtype==np.float32):
                            data_out = np.copy( data_out.astype(np.float32) )
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[:,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                        else:
                            dset[:,:,:,:] = data_out.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                        
                        if verbose:
                            txt = even_print(f'write: {scalar}_fv', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
            
            # === replace dims/t array --> take last time of series
            t = np.array([self.t[-1]],dtype=np.float64)
            if ('dims/t' in hf_mean):
                del hf_mean['dims/t']
            hf_mean.create_dataset('dims/t', data=t)
            
            if hasattr(hf_mean, 'duration_avg'):
                if verbose: even_print('duration avg', '%0.2f [-]'%hf_mean.duration_avg)
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_rgd_mean, '%0.2f [GB]'%(os.path.getsize(fn_rgd_mean)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.get_mean() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def add_mean_dimensional_data_xpln(self, **kwargs):
        '''
        get dimensionalized mean data for [x] plane
        --> save to existing RGD file with fsubtype=mean
        - assumes volume which is thin in [x] direction
        - an RGD which is the output of rgd.get_mean() should be opened here
        - not parallel
        '''
        
        verbose      = kwargs.get('verbose',True)
        epsilon      = kwargs.get('epsilon',5e-5)
        acc          = kwargs.get('acc',6)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.add_mean_dimensional_data_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## this func is not parallel
        if self.usingmpi:
            raise NotImplementedError('rgd.add_mean_dimensional_data_xpln() is not a parallel function')
        
        ## 'r' and 'w' open modes are not allowed
        if not (self.open_mode=='a') or (self.open_mode=='r+'):
            raise ValueError(f'open mode is {self.open_mode}')
        
        ## assert that this is a mean flow file ( i.e. output from rgd.get_mean() )
        if (self.fsubtype!='mean'):
            print(self.fsubtype)
            raise ValueError
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## read in 1D coordinate arrays, then dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        nx = self.nx
        ny = self.ny
        nz = self.nz
        
        if (x.ndim!=1):
            raise ValueError
        if (y.ndim!=1):
            raise ValueError
        if (z.ndim!=1):
            raise ValueError
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-7)):
            raise NotImplementedError
        
        ## check if mean rgd has attr 'dt'
        if ('dt' in self.attrs.keys()):
            dt = self.attrs['dt']
            if (dt is not None):
                dt *= self.tchar
        else:
            raise ValueError
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration_avg,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration_avg*self.tchar,))
        if verbose: print(72*'-')
        
        ## re-dimensionalize
        u   =  np.copy(  self.U_inf                     *  self['data/u'][()].T   )
        v   =  np.copy(  self.U_inf                     *  self['data/v'][()].T   )
        w   =  np.copy(  self.U_inf                     *  self['data/w'][()].T   )
        rho =  np.copy(  self.rho_inf                   *  self['data/rho'][()].T )
        p   =  np.copy(  (self.rho_inf * self.U_inf**2) *  self['data/p'][()].T   )
        T   =  np.copy(  self.T_inf                     *  self['data/T'][()].T   )
        
        # mu1 = np.copy( self.C_Suth * T**(3/2) / (T + self.S_Suth) )
        # mu2 = np.copy( self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth)) )
        # np.testing.assert_allclose(mu1, mu2, rtol=2e-7, atol=2e-7)
        # mu = np.copy(mu2)
        
        mu = np.copy( self.C_Suth * T**(3/2) / (T + self.S_Suth) )
        nu = np.copy( mu / rho )
        
        # === average in [x,z] --> leave 1D [y]
        
        u     = np.squeeze( np.mean( u   , axis=(0,2), dtype=np.float64) )
        v     = np.squeeze( np.mean( v   , axis=(0,2), dtype=np.float64) )
        w     = np.squeeze( np.mean( w   , axis=(0,2), dtype=np.float64) )
        rho   = np.squeeze( np.mean( rho , axis=(0,2), dtype=np.float64) )
        p     = np.squeeze( np.mean( p   , axis=(0,2), dtype=np.float64) )
        T     = np.squeeze( np.mean( T   , axis=(0,2), dtype=np.float64) )
        mu    = np.squeeze( np.mean( mu  , axis=(0,2), dtype=np.float64) )
        nu    = np.squeeze( np.mean( nu  , axis=(0,2), dtype=np.float64) )
        
        # ## determine finite difference order / size of central stencil based on [nx]
        # if (nx<3):
        #     raise ValueError('dx[] not possible because nx<3')
        # elif (nx>=3) and (nx<5):
        #     acc = 2
        # elif (nx>=5) and (nx<7):
        #     acc = 4
        # elif (nx>=7):
        #     acc = 6
        # else:
        #     raise ValueError('this should never happen')
        
        if verbose: even_print('acc','%i'%acc)
        if verbose: even_print('edge_stencil',edge_stencil)
        if verbose: print(72*'-')
        
        ## get [y] gradients --> size [y]
        ddy_u   = gradient(u   , y, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        ddy_v   = gradient(v   , y, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        ddy_T   = gradient(T   , y, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        ddy_p   = gradient(p   , y, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        ddy_rho = gradient(rho , y, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        
        ## wall quantities
        ddy_u_wall  = float( ddy_u[0] )
        ddy_T_wall  = float( ddy_T[0] )
        rho_wall    = float( rho[0]   )
        nu_wall     = float( nu[0]    )
        mu_wall     = float( mu[0]    )
        T_wall      = float( T[0]     )
        tau_wall    = mu_wall * ddy_u_wall
        
        u_tau   = np.sqrt( tau_wall / rho_wall ) ## uτ
        sc_u_in = u_tau
        sc_l_in = nu_wall / u_tau     ## δν
        sc_t_in = nu_wall / u_tau**2  ## tν
        np.testing.assert_allclose(sc_t_in, sc_l_in/sc_u_in, rtol=1e-14, atol=1e-14)
        
        if verbose: even_print('ρw',f'{rho_wall:0.5f} [kg/m³]')
        if verbose: even_print('Tw',f'{T_wall:0.2f} [K]')
        if verbose: even_print('μw',f'{mu_wall:0.5E} [kg/(m·s)]')
        if verbose: even_print('νw',f'{nu_wall:0.5E} [m²/s]')
        if verbose: even_print('uτ',f'{u_tau:0.5E} [m/s]')
        if verbose: even_print('τw',f'{tau_wall:0.5E} [Pa]')
        if verbose: even_print('δν',f'{sc_l_in:0.5E} [m]')
        if verbose: even_print('tν',f'{sc_t_in:0.5E} [s]')
        if verbose: even_print('(du/dy)_w',f'{ddy_u_wall:0.3E} [1/s]')
        if verbose: even_print('(dT/dy)_w',f'{ddy_T_wall:0.3E} [K/m]')
        if verbose: print(72*'-')
        
        ongrid = True ## snap to [y] index
        
        ddy_u_plus = np.copy( ddy_u / (u_tau/sc_l_in) ) ## du+/dy+ = (du/dy)/(uτ/δν) = (du/dy)/(uτ^2/νw)
        
        ## get edge --> where |du+/dy+|<ϵ
        y_edge = calc_profile_edge_1d(
                        y=y, ## dimensional
                        ddy_u=ddy_u_plus,
                        ongrid=ongrid,
                        epsilon=epsilon,
                        acc=acc,
                        edge_stencil=edge_stencil,
                        interp_kind='cubic',
                        )
        
        j_edge = np.abs( y - y_edge ).argmin()
        if not np.isclose(y[j_edge], y_edge, rtol=1e-14):
            raise ValueError
        
        if verbose: even_print('ϵ',f'{epsilon:0.5E}')
        if verbose: even_print('j_edge',f'{j_edge:d}')
        if verbose: even_print('(du+/dy+)_edge',f'{ddy_u_plus[j_edge]:0.5E}')
        if verbose: even_print('y_edge',f'{y_edge:0.5E} [m]')
        if verbose: even_print('y+_edge',f'{y_edge/sc_l_in:0.3f}')
        
        aa = (y_edge-y.min())/(y.max()-y.min())
        if verbose: even_print('(y_edge-ymin)/(ymax-ymin)',f'{aa:0.3f}')
        
        # ===
        
        u_edge   =   u[j_edge]
        v_edge   =   v[j_edge]
        w_edge   =   w[j_edge]
        T_edge   =   T[j_edge]
        rho_edge = rho[j_edge]
        mu_edge  =  mu[j_edge]
        nu_edge  =  nu[j_edge]
        
        ## δ99
        d99 = calc_d99_1d(
                y=y,
                u=u,
                y_edge=y_edge,
                u_edge=u_edge,
                rtol=1e-3,
                interp_kind='cubic',
                )
        
        # ## grid-snapped δ99 --> not used
        # j99  = np.abs( y - d99 ).argmin()
        # d99g = y[j99]
        
        u_99 = float( sp.interpolate.interp1d(y, u, kind='cubic', bounds_error=True)(d99) )
        
        ## outer scales
        sc_l_out = d99
        sc_u_out = u_99
        sc_t_out = d99/u_99
        np.testing.assert_allclose(sc_t_out, sc_l_out/sc_u_out, rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration_avg * (self.lchar / self.U_inf)
        t_eddy = t_meas / ( d99 / u_tau )
        
        ## get BL integral quantities
        dd = calc_bl_integral_quantities_1d(
                y=y,
                u=u,
                rho=rho,
                u_tau=u_tau,
                d99=d99,
                y_edge=y_edge,
                rho_edge=rho_edge,
                nu_edge=nu_edge,
                u_edge=u_edge,
                nu_wall=nu_wall,
                )
        
        # === add to file
        
        gn = 'data_dim'
        
        ## if group already exists in file, delete it entirely
        if (gn in self):
            del self[gn]
        
        ## 1D
        
        self.create_dataset(f'{gn}/u'     , data=u     , chunks=None)
        self.create_dataset(f'{gn}/rho'   , data=rho   , chunks=None)
        
        self.create_dataset(f'{gn}/ddy_u'   , data=ddy_u   , chunks=None)
        self.create_dataset(f'{gn}/ddy_v'   , data=ddy_v   , chunks=None)
        self.create_dataset(f'{gn}/ddy_T'   , data=ddy_T   , chunks=None)
        self.create_dataset(f'{gn}/ddy_p'   , data=ddy_p   , chunks=None)
        self.create_dataset(f'{gn}/ddy_rho' , data=ddy_rho , chunks=None)
        
        self.create_dataset(f'{gn}/z1d' , data=z , chunks=None)
        self.create_dataset(f'{gn}/z'   , data=z , chunks=None)
        
        ## 0D
        
        self.create_dataset(f'{gn}/dz0'        , data=dz0 , chunks=None)
        self.create_dataset(f'{gn}/dt'         , data=dt  , chunks=None)
        
        self.create_dataset(f'{gn}/y_edge'     , data=y_edge    , chunks=None)
        self.create_dataset(f'{gn}/d99'        , data=d99       , chunks=None)
        self.create_dataset(f'{gn}/u_99'       , data=u_99      , chunks=None)
        
        self.create_dataset(f'{gn}/u_edge'     , data=u_edge     , chunks=None)
        self.create_dataset(f'{gn}/rho_edge'   , data=rho_edge   , chunks=None)
        self.create_dataset(f'{gn}/mu_edge'    , data=mu_edge    , chunks=None)
        self.create_dataset(f'{gn}/nu_edge'    , data=nu_edge    , chunks=None)
        self.create_dataset(f'{gn}/T_edge'     , data=T_edge     , chunks=None)
        
        self.create_dataset(f'{gn}/u_tau'      , data=u_tau      , chunks=None)
        
        self.create_dataset(f'{gn}/ddy_u_wall' , data=ddy_u_wall , chunks=None )
        self.create_dataset(f'{gn}/ddy_T_wall' , data=ddy_T_wall , chunks=None )
        self.create_dataset(f'{gn}/rho_wall'   , data=rho_wall   , chunks=None )
        self.create_dataset(f'{gn}/nu_wall'    , data=nu_wall    , chunks=None )
        self.create_dataset(f'{gn}/mu_wall'    , data=mu_wall    , chunks=None )
        self.create_dataset(f'{gn}/T_wall'     , data=T_wall     , chunks=None )
        self.create_dataset(f'{gn}/tau_wall'   , data=tau_wall   , chunks=None )
        #self.create_dataset(f'{gn}/q_wall'     , data=q_wall     , chunks=None )
        
        self.create_dataset(f'{gn}/sc_u_in'    , data=sc_u_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_l_in'    , data=sc_l_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_t_in'    , data=sc_t_in    , chunks=None)
        self.create_dataset(f'{gn}/sc_u_out'   , data=sc_u_out   , chunks=None)
        self.create_dataset(f'{gn}/sc_l_out'   , data=sc_l_out   , chunks=None)
        self.create_dataset(f'{gn}/sc_t_out'   , data=sc_t_out   , chunks=None)
        
        ## add integrated quantities (all 0D)
        for key,val in dd.items():
            self.create_dataset(f'{gn}/{key}', data=val, chunks=None)
        
        ## report
        if verbose:
            print(72*'-')
            even_print('Reτ'                       , '%0.1f'%dd['Re_tau']            )
            even_print('Reθ'                       , '%0.1f'%dd['Re_theta']          )
            even_print('θ'                         , '%0.5E [m]'%dd['theta_cmp']     )
            even_print('δ*'                        , '%0.5E [m]'%dd['dstar_cmp']     )
            even_print('H12'                       , '%0.5f'%dd['H12']               )
            even_print('δ99'                       , '%0.5E [m]'%d99                 )
            even_print('θ/δ99'                     , '%0.5f'%(dd['theta_cmp']/d99)   )
            even_print('δ*/δ99'                    , '%0.5f'%(dd['dstar_cmp']/d99)   )
            
            even_print('uτ'                        , f'{u_tau:0.5E} [m/s]'    )
            even_print('νw'                        , f'{nu_wall:0.5E} [m²/s]' )
            even_print('τw'                        , f'{tau_wall:0.5E} [Pa]'  )
            
            even_print('τw/q_inf'                  , '%0.5E'%(tau_wall/(self.rho_inf*self.U_inf**2)) )
            even_print('cf = 2·τw/(ρe·ue²)'        , '%0.5E'%(2*tau_wall/(rho_edge*u_edge**2)) )
            even_print('t_meas'                    , '%0.5E [s]'%t_meas              )
            even_print('t_meas/tchar'              , '%0.1f'%(t_meas/self.tchar)     )
            even_print('t_eddy = t_meas/(δ99/uτ)'  , '%0.2f'%t_eddy                  )
            even_print('t_meas/(δ99/u_99)'         , '%0.2f'%(t_meas/(d99/u_99))     )
            even_print('t_meas/(20·δ99/u_99)'      , '%0.2f'%(t_meas/(20*d99/u_99))  )
            print(72*'-')
            even_print('sc_u_in = uτ'                , '%0.5E [m/s]'%(sc_u_in,)  )
            even_print('sc_l_in = δν = νw/uτ'        , '%0.5E [m]'%(sc_l_in,)    )
            even_print('sc_t_in = tν = νw/uτ²'       , '%0.5E [s]'%(sc_t_in,)    )
            even_print('sc_u_out = u_99'             , '%0.5E [m/s]'%(sc_u_out,) )
            even_print('sc_l_out = δ99'              , '%0.5E [m]'%(sc_l_out,)   )
            even_print('sc_t_out = δ99/u_99'         , '%0.5E [s]'%(sc_t_out,)   )
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.add_mean_dimensional_data_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_mean_dimensional_data_ypln(self, **kwargs):
        '''
        get dimensionalized mean data for [y] plane
        --> save to existing RGD file with fsubtype=mean
        - assumes volume which is thin in [y] direction
        - an RGD which is the output of rgd.get_mean() should be opened here
        - not parallel
        '''
        
        pass
        raise NotImplementedError
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.add_mean_dimensional_data_ypln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_mean_dimensional_data_zpln(self, **kwargs):
        '''
        get dimensionalized mean data for [z] plane
        --> save to existing RGD file with fsubtype=mean
        - assumes volume which is thin in [z] direction
        - an RGD which is the output of rgd.get_mean() should be opened here
        - not parallel
        '''
        
        pass
        raise NotImplementedError
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.add_mean_dimensional_data_zpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def get_prime(self, **kwargs):
        '''
        get mean-removed (prime) variables in [t]
        --> save to new RGD file
        -----
        XI  : Reynolds primes : mean(XI)=0
        XII : Favre primes    : mean(ρ·XII)=0 --> mean(XII)≠0 !!
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        ## assert that the opened RGD has fsubtype 'unsteady'
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        if verbose: print('\n'+'rgd.get_prime()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        ct = kwargs.get('ct',1) ## n chunks [t]
        
        fn_rgd_mean  = kwargs.get('fn_rgd_mean',None)
        fn_rgd_prime = kwargs.get('fn_rgd_prime',None)
        sfp          = kwargs.get('scalars',None) ## scalars (for prime)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing prime file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        ## start timestep index
        ti_min = kwargs.get('ti_min',None)
        
        ## if writing Favre primes, copy over ρ --> mean(ρ·XII)=0 / mean(XII)≠0 !!
        if favre:
            copy_rho = True
        else:
            copy_rho = False
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if (sfp is None):
            sfp = self.scalars
        
        # === ranks
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === get times to take for prime
        if (ti_min is not None):
            ti_for_prime = np.copy( self.ti[ti_min:] )
        else:
            ti_for_prime = np.copy( self.ti )
        
        nt_prime       = ti_for_prime.shape[0]
        t_prime_start  = self.t[ti_for_prime[0]]
        t_prime_end    = self.t[ti_for_prime[-1]]
        duration_prime = t_prime_end - t_prime_start
        
        # === chunks
        #ctl_ = np.array_split(np.arange(self.nt),min(ct,self.nt))
        ctl_ = np.array_split(ti_for_prime,min(ct,nt_prime))
        ctl  = [[b[0],b[-1]+1] for b in ctl_ ]
        
        # === mean file name (for reading)
        if (fn_rgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_rgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_rgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_rgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if not os.path.isfile(fn_rgd_mean):
            raise FileNotFoundError(fn_rgd_mean)
        
        # === prime file name (for writing)
        if (fn_rgd_prime is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_prime_h5_base = fname_root+'_prime.h5'
            #fn_rgd_prime = os.path.join(fname_path, fname_prime_h5_base)
            fn_rgd_prime = str(PurePosixPath(fname_path, fname_prime_h5_base))
            #fn_rgd_prime = Path(fname_path, fname_prime_h5_base)
        
        if verbose: even_print('fn_rgd'          , self.fname    )
        if verbose: even_print('fn_rgd_mean'     , fn_rgd_mean   )
        if verbose: even_print('fn_rgd_prime'    , fn_rgd_prime  )
        if verbose: even_print('do Favre avg'    , str(favre)    )
        if verbose: even_print('do Reynolds avg' , str(reynolds) )
        if verbose: even_print('copy rho'        , str(copy_rho) )
        if verbose: even_print('ct'              , '%i'%ct       )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('n timesteps prime','%i/%i'%(nt_prime,self.nt))
        if verbose: even_print('t index prime start','%i'%(ti_for_prime[0],))
        if verbose: even_print('t index prime end','%i'%(ti_for_prime[-1],))
        if verbose: even_print('t prime start','%0.2f [-]'%(t_prime_start,))
        if verbose: even_print('t prime end','%0.2f [-]'%(t_prime_end,))
        if verbose: even_print('duration prime','%0.2f [-]'%(duration_prime,))
        if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        #data_gb      = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        data_gb      = 4*self.nx*self.ny*self.nz*nt_prime / 1024**3
        data_gb_mean = 4*self.nx*self.ny*self.nz*1       / 1024**3
        
        scalars_re = ['u','v','w','T','p','rho']
        scalars_fv = ['u','v','w','T'] ## p'' and ρ'' are never really needed
        
        scalars_re_ = []
        for scalar in scalars_re:
            if (scalar in self.scalars) and (scalar in sfp):
                scalars_re_.append(scalar)
        scalars_re = scalars_re_
        
        scalars_fv_ = []
        for scalar in scalars_fv:
            if (scalar in self.scalars) and (scalar in sfp):
                scalars_fv_.append(scalar)
        scalars_fv = scalars_fv_
        
        # ===
        
        comm_rgd_prime = MPI.COMM_WORLD
        
        with rgd(fn_rgd_prime, 'w', force=force, driver=self.driver, comm=self.comm, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_prime:
            
            ## initialize prime rgd from rgd
            hf_prime.init_from_rgd(self.fname)
            
            ## add top-level attributes
            #hf_prime.attrs['fclass'] = 'rgd'
            hf_prime.attrs['fsubtype'] = 'prime'
            
            #shape  = (self.nt,self.nz,self.ny,self.nx)
            shape  = (nt_prime,self.nz,self.ny,self.nx)
            
            ## determine dtypes for prime file
            for scalar in self.scalars:
                dset = self[f'data/{scalar}']
                dtype = dset.dtype
                if reynolds and (scalar in scalars_re):
                    hf_prime.scalars_dtypes_dict[f'{scalar}I'] = dtype
                if favre and (scalar in scalars_fv):
                    hf_prime.scalars_dtypes_dict[f'{scalar}II'] = dtype
            
            # === initialize prime datasets + rho
            
            if copy_rho:
                
                dtype = self.scalars_dtypes_dict['rho']
                float_bytes = dtype.itemsize
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                
                if verbose:
                    even_print('initializing data/rho','%0.1f [GB]'%(data_gb,))
                
                dset = hf_prime.create_dataset('data/rho',
                                               shape=shape,
                                               dtype=dtype,
                                               chunks=chunks)
                
                chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                if verbose:
                    even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            for scalar in self.scalars:
                
                if reynolds:
                    if (scalar in scalars_re):
                        
                        dtype = hf_prime.scalars_dtypes_dict[f'{scalar}I']
                        float_bytes = dtype.itemsize
                        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                        
                        ## if ('data/%sI'%scalar in hf_prime):
                        ##     del hf_prime['data/%sI'%scalar]
                        if verbose:
                            even_print('initializing data/%sI'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        
                        dset = hf_prime.create_dataset(f'data/{scalar}I',
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_prime.scalars.append(f'{scalar}I')
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    if (scalar in scalars_fv):
                        
                        dtype = hf_prime.scalars_dtypes_dict[f'{scalar}II']
                        float_bytes = dtype.itemsize
                        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*nt_prime / 1024**3
                        
                        ## if ('data/%sII'%scalar in hf_prime):
                        ##     del hf_prime['data/%sII'%scalar]
                        if verbose:
                            even_print('initializing data/%sII'%(scalar,),'%0.1f [GB]'%(data_gb,))
                        
                        dset = hf_prime.create_dataset(f'data/{scalar}II',
                                                       shape=shape,
                                                       dtype=dtype,
                                                       chunks=chunks)
                        hf_prime.scalars.append(f'{scalar}II')
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if hf_prime.usingmpi: comm_rgd_prime.Barrier()
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            # === read unsteady + mean, do difference, write
            
            n_pbar = 0
            if favre or copy_rho:
                n_pbar += 1
            for scalar in self.scalars:
                if (scalar in scalars_re) and reynolds:
                    n_pbar += 1
                if (scalar in scalars_fv) and favre:
                    n_pbar += 1
            
            comm_rgd_mean = MPI.COMM_WORLD
            
            with rgd(fn_rgd_mean, 'r', driver=self.driver, comm=self.comm) as hf_mean:
                
                ## copy over 'data_dim' from mean file
                if ('data_dim' in hf_mean):
                    grp = hf_mean['data_dim']
                    for dsn in grp.keys():
                        ds = hf_mean[f'data_dim/{dsn}']
                        data = np.copy(ds[()])
                        #if (f'data_dim/{dsn}' in hf_prime):
                        #    del hf_prime[f'data_dim/{dsn}']
                        
                        #if self.usingmpi:
                        #    with dset.collective:
                        #        hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                        #else:
                        #    hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                        
                        hf_prime.create_dataset(f'data_dim/{dsn}', data=data, chunks=None)
                    
                    if verbose:
                        even_print('data_dim copied to prime',str(True))
                else:
                    if verbose:
                        even_print('data_dim copied to prime',str(False))
                
                if verbose:
                    progress_bar = tqdm(total=ct*n_pbar, ncols=100, desc='prime', leave=False, file=sys.stdout)
                
                for ctl_ in ctl:
                    ct1, ct2 = ctl_
                    ntc = ct2 - ct1
                    
                    #if verbose: tqdm.write(f'ct1,ct2 = {ct1:d},{ct2:d}')
                    #if verbose: tqdm.write(f'ntc = {ntc:d}')
                    
                    ## chunk range for writing to file (offset from read if using ti_min)
                    if (ti_min is not None):
                        #ct1w,ct2w = ct1-ti_min, ct2-ti_min ## doesnt work for (-) ti_min
                        ct1w,ct2w = ct1-ti_for_prime[0], ct2-ti_for_prime[0]
                    else:
                        ct1w,ct2w = ct1,ct2
                    
                    #if verbose: tqdm.write(f'ct1w,ct2w = {ct1w:d},{ct2w:d}')
                    
                    if favre or copy_rho:
                        
                        ## read rho
                        dset = self['data/rho']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                rho = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                        else:
                            rho = dset[ct1:ct2,:,:,:].T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if verbose:
                            txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        ## write a copy of rho to the prime file
                        dset = hf_prime['data/rho']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if hf_prime.usingmpi: hf_prime.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = rho.T
                                dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = rho.T
                        else:
                            #dset[ct1:ct2,:,:,:] = rho.T
                            dset[ct1w:ct2w,:,:,:] = rho.T
                        if hf_prime.usingmpi: hf_prime.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            txt = even_print('write: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        if verbose: progress_bar.update()
                    
                    for scalar in self.scalars:
                        
                        if (scalar in scalars_re) or (scalar in scalars_fv):
                            
                            ## read RGD data
                            dset = self['data/%s'%scalar]
                            dtype = dset.dtype
                            float_bytes = dtype.itemsize
                            data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                            
                            if self.usingmpi: self.comm.Barrier()
                            t_start = timeit.default_timer()
                            if self.usingmpi:
                                with dset.collective:
                                    data = dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T
                            else:
                                data = dset[ct1:ct2,:,:,:].T
                            if self.usingmpi: self.comm.Barrier()
                            t_delta = timeit.default_timer() - t_start
                            
                            if verbose:
                                txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                tqdm.write(txt)
                            
                            t_read       += t_delta
                            data_gb_read += data_gb
                            
                            # === do prime Reynolds
                            
                            if (scalar in scalars_re) and reynolds:
                                
                                ## read Reynolds avg from mean file
                                dset = hf_mean['data/%s'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_mean.nx * hf_mean.ny * hf_mean.nz * 1 / 1024**3
                                
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_mean.usingmpi:
                                    with dset.collective:
                                        data_mean_re = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                                else:
                                    data_mean_re = dset[()].T
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                ## if verbose:
                                ##     txt = even_print('read: %s (Re avg)'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                ##     tqdm.write(txt)
                                
                                ## calc mean-removed Reynolds
                                data_prime_re = data - data_mean_re
                                
                                ## if False:
                                ##     data_prime_re_mean = np.mean(data_prime_re, axis=-1, dtype=np.float64, keepdims=True).astype(np.float32)
                                ##     
                                ##     ## normalize [mean(prime)] by mean
                                ##     data_prime_re_mean = np.abs(np.divide(data_prime_re_mean,
                                ##                                           data_mean_re, 
                                ##                                           out=np.zeros_like(data_prime_re_mean), 
                                ##                                           where=data_mean_re!=0))
                                ##     
                                ##     # np.testing.assert_allclose( data_prime_re_mean , 
                                ##     #                             np.zeros_like(data_prime_re_mean, dtype=np.float32), atol=1e-4)
                                ##     if verbose:
                                ##         tqdm.write('max(abs(mean(%sI)/mean(%s)))=%0.4e'%(scalar,scalar,data_prime_re_mean.max()))
                                
                                ## write Reynolds prime
                                dset = hf_prime['data/%sI'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_prime.nx * hf_prime.ny * hf_prime.nz * ntc / 1024**3
                                
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_prime.usingmpi:
                                    with dset.collective:
                                        #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_re.T
                                        dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_re.T
                                else:
                                    #dset[ct1:ct2,:,:,:] = data_prime_re.T
                                    dset[ct1w:ct2w,:,:,:] = data_prime_re.T
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if verbose:
                                    txt = even_print('write: %sI'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                    tqdm.write(txt)
                                
                                if verbose: progress_bar.update()
                            
                            # === do prime Favre
                            
                            if (scalar in scalars_fv) and favre:
                                
                                ## read Favre avg from mean file
                                dset = hf_mean['data/%s_fv'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_mean.nx * hf_mean.ny * hf_mean.nz * 1 / 1024**3
                                
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_mean.usingmpi:
                                    with dset.collective:
                                        data_mean_fv = dset[:,rz1:rz2,ry1:ry2,rx1:rx2].T
                                else:
                                    data_mean_fv = dset[()].T
                                if hf_mean.usingmpi: hf_mean.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                ## if verbose:
                                ##     txt = even_print('read: %s (Fv avg)'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                ##     tqdm.write(txt)
                                
                                ## calc mean-removed Favre
                                ## data_prime_fv = ( data - data_mean_fv ) * rho ## pre-multiply with ρ (has zero mean) --> better to not do this here
                                data_prime_fv = data - data_mean_fv
                                
                                ## write Favre prime
                                dset = hf_prime['data/%sII'%scalar]
                                dtype = dset.dtype
                                float_bytes = dtype.itemsize
                                data_gb = float_bytes * hf_prime.nx * hf_prime.ny * hf_prime.nz * ntc / 1024**3
                                
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_start = timeit.default_timer()
                                if hf_prime.usingmpi:
                                    with dset.collective:
                                        #dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_fv.T
                                        dset[ct1w:ct2w,rz1:rz2,ry1:ry2,rx1:rx2] = data_prime_fv.T
                                else:
                                    #dset[ct1:ct2,:,:,:] = data_prime_fv.T
                                    dset[ct1w:ct2w,:,:,:] = data_prime_fv.T
                                if hf_prime.usingmpi: hf_prime.comm.Barrier()
                                t_delta = timeit.default_timer() - t_start
                                
                                t_write       += t_delta
                                data_gb_write += data_gb
                                
                                if verbose:
                                    txt = even_print('write: %sII'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                                    tqdm.write(txt)
                                
                                pass
                                
                                if verbose: progress_bar.update()
                        
                        if self.usingmpi: self.comm.Barrier()
                        if hf_prime.usingmpi: comm_rgd_prime.Barrier()
                        if hf_mean.usingmpi: comm_rgd_mean.Barrier()
                    
                    ## report mem free
                    mem_avail_gb = psutil.virtual_memory().available/1024**3
                    mem_free_gb  = psutil.virtual_memory().free/1024**3
                    if verbose:
                        tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                
                if verbose:
                    progress_bar.close()
            
            # === replace dims/t array in prime file (if ti_min was given)
            if (ti_min is not None):
                t = np.copy( self.t[ti_min:] )
                if ('dims/t' in hf_prime):
                    del hf_prime['dims/t']
                hf_prime.create_dataset('dims/t', data=t)
            
            if hf_mean.usingmpi: comm_rgd_mean.Barrier()
        if hf_prime.usingmpi: comm_rgd_prime.Barrier()
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_rgd_prime, '%0.2f [GB]'%(os.path.getsize(fn_rgd_prime)/1024**3))
        if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.get_prime() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_lambda2(self, **kwargs):
        '''
        calculate λ₂ & Q
        Jeong & Hussain (1996) : https://doi.org/10.1017/S0022112095000462
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx           = kwargs.get('rx',1)
        ry           = kwargs.get('ry',1)
        rz           = kwargs.get('rz',1)
        
        save_Q       = kwargs.get('save_Q',True)
        save_lambda2 = kwargs.get('save_lambda2',True)
        
        acc          = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','half')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        d = 1 ## derivative order
        stencil_npts = 2*math.floor((d+1)/2) - 1 + acc
        
        if ((stencil_npts-1)%2 != 0):
            raise AssertionError
        
        stencil_npts_one_side = int( (stencil_npts-1)/2 )
        
        # ===
        
        if verbose: print('\n'+'turbx.rgd.calc_lambda2()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## checks
        if all([(save_Q is False),(save_lambda2 is False)]):
            raise AssertionError('neither λ-2 nor Q set to be solved')
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('data/w' in self):
            raise ValueError('data/w not in hdf5')
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('save_Q','%s'%save_Q)
        if verbose: even_print('save_lambda2','%s'%save_lambda2)
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        t_q_crit = 0.
        t_l2     = 0.
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # === extend the rank ranges (spatial range overlap)
        
        if self.usingmpi:
            
            n_overlap = stencil_npts_one_side + 3
            
            xA = 0
            xB = nxr
            yA = 0
            yB = nyr
            zA = 0
            zB = nzr
            
            ## backup non-overlapped bounds
            rx1_orig, rx2_orig = rx1, rx2
            ry1_orig, ry2_orig = ry1, ry2
            rz1_orig, rz2_orig = rz1, rz2
            
            ## overlap in [x]
            if (t4d[0]!=0):
                rx1, rx2 = rx1-n_overlap, rx2
                xA += n_overlap
                xB += n_overlap
            if (t4d[0]!=rx-1):
                rx1, rx2 = rx1, rx2+n_overlap
            
            ## overlap in [y]
            if (t4d[1]!=0):
                ry1, ry2 = ry1-n_overlap, ry2
                yA += n_overlap
                yB += n_overlap
            if (t4d[1]!=ry-1):
                ry1, ry2 = ry1, ry2+n_overlap
            
            ## overlap in [z]
            if (t4d[2]!=0):
                rz1, rz2 = rz1-n_overlap, rz2
                zA += n_overlap
                zB += n_overlap
            if (t4d[2]!=rz-1):
                rz1, rz2 = rz1, rz2+n_overlap
            
            ## update (rank local) nx,ny,nz
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
        
        ## check rank / grid distribution
        if self.usingmpi and False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : rx1=%i rx2=%i ry1=%i ry2=%i rz1=%i rz2=%i'%(self.rank, rx1,rx2, ry1,ry2, rz1,rz2))
                    sys.stdout.flush()
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        dtype = self['data/u'].dtype
        itemsize = dtype.itemsize
        float_bytes = dtype.itemsize
        
        data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
        shape  = (self.nt,self.nz,self.ny,self.nx)
        chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
        
        # === initialize 4D arrays in HDF5
        
        if save_lambda2:
            
            self.scalars_dtypes_dict['lambda2'] = dtype
            
            if verbose:
                even_print('initializing data/lambda2','%0.2f [GB]'%(data_gb,))
            if ('data/lambda2' in self):
                del self['data/lambda2']
            dset = self.create_dataset('data/lambda2', 
                                        shape=shape, 
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if save_Q:
            
            self.scalars_dtypes_dict['Q'] = dtype
            
            if verbose:
                even_print('initializing data/Q','%0.2f [GB]'%(data_gb,))
            if ('data/Q' in self):
                del self['data/Q']
            dset = self.create_dataset('data/Q', 
                                        shape=shape, 
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
            
            chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
            if verbose:
                even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # ===
        
        if verbose:
            progress_bar = tqdm(total=self.nt, ncols=100, desc='calc λ2', leave=False, file=sys.stdout)
        
        for ti in self.ti:
            
            # === read u,v,w
            
            dset = self['data/u']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    u_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                u_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read u', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/v']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    v_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                v_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read v', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            dset = self['data/w']
            dtype = dset.dtype
            float_bytes = dtype.itemsize
            if self.usingmpi: self.comm.Barrier()
            t_start = timeit.default_timer()
            if self.usingmpi:
                with dset.collective:
                    w_ = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
            else:
                w_ = dset[ti,:,:,:].T
            if self.usingmpi: self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            data_gb = float_bytes * self.nx * self.ny * self.nz * 1 / 1024**3
            if verbose:
                tqdm.write( even_print('read w', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True ) )
            
            # ===
            
            if self.usingmpi:
                x_ = np.copy(self.x[rx1:rx2])
                y_ = np.copy(self.y[ry1:ry2])
                z_ = np.copy(self.z[rz1:rz2])
            else:
                x_ = np.copy(self.x)
                y_ = np.copy(self.y)
                z_ = np.copy(self.z)
            
            # === ∂(u)/∂(x,y,z)
            
            ddx_u = gradient(u_, x_, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_u = gradient(u_, y_, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_u = gradient(u_, z_, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            # === ∂(v)/∂(x,y,z)
            
            ddx_v = gradient(v_, x_, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_v = gradient(v_, y_, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_v = gradient(v_, z_, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            # === ∂(w)/∂(x,y,z)
            
            ddx_w = gradient(w_, x_, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_w = gradient(w_, y_, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddz_w = gradient(w_, z_, axis=2, acc=acc, edge_stencil=edge_stencil, d=1)
            
            ## free memory
            u_ = None; del u_
            v_ = None; del v_
            w_ = None; del w_
            gc.collect()
            
            strain = np.copy( np.stack((np.stack((ddx_u, ddy_u, ddz_u), axis=3),
                                        np.stack((ddx_v, ddy_v, ddz_v), axis=3),
                                        np.stack((ddx_w, ddy_w, ddz_w), axis=3)), axis=4) )
            
            t_delta = timeit.default_timer() - t_start
            if verbose: tqdm.write( even_print('get strain ∂(u,v,v)/∂(x,y,z)' , '%0.3f [s]'%(t_delta,), s=True) )
            
            ## free memory
            ddx_u = None; del ddx_u
            ddy_u = None; del ddy_u
            ddz_u = None; del ddz_u
            ddx_v = None; del ddx_v
            ddy_v = None; del ddy_v
            ddz_v = None; del ddz_v
            ddx_w = None; del ddx_w
            ddy_w = None; del ddy_w
            ddz_w = None; del ddz_w
            gc.collect()
            
            # === get the rate-of-strain & vorticity tensors
            
            S = np.copy( 0.5*(strain + np.transpose(strain, axes=(0,1,2,4,3))) ) ## strain rate tensor (symmetric)
            O = np.copy( 0.5*(strain - np.transpose(strain, axes=(0,1,2,4,3))) ) ## rotation rate tensor (anti-symmetric)
            # np.testing.assert_allclose(S+O, strain, atol=1.e-6)
            
            ## free memory
            strain = None; del strain
            gc.collect()
            
            # === Q : second invariant of characteristics equation: λ³ + Pλ² + Qλ + R = 0
            
            if save_Q:
                
                t_start = timeit.default_timer()
                
                O_norm  = np.linalg.norm(O, ord='fro', axis=(3,4)) ## Frobenius norm
                S_norm  = np.linalg.norm(S, ord='fro', axis=(3,4))
                Q       = 0.5*(O_norm**2 - S_norm**2)
                
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print('calc Q','%s'%format_time_string(t_delta), s=True))
                
                dset = self['data/Q']
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ti,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = Q[xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ti,:,:,:] = Q.T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = float_bytes * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print('write Q','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                # === second invariant : Q --> an equivalent formulation using eigenvalues (but much slower)
                if False:
                    Q_bak = np.copy(Q)
                    t_start = timeit.default_timer()
                    eigvals = np.linalg.eigvals(strain)
                    P       = -1*np.sum(eigvals, axis=-1) ## first invariant : P
                    SijSji  = np.einsum('xyzij,xyzji->xyz', S, S)
                    OijOji  = np.einsum('xyzij,xyzji->xyz', O, O)
                    Q       = 0.5*(P**2 - SijSji - OijOji)
                    t_delta = timeit.default_timer() - t_start
                    if verbose: tqdm.write(even_print('calc Q','%s'%format_time_string(t_delta), s=True))
                    np.testing.assert_allclose(Q.imag, np.zeros_like(Q.imag, dtype=np.float32), atol=1e-6)
                    Q = np.copy(Q.real)
                    np.testing.assert_allclose(Q, Q_bak, rtol=1e-2, atol=1e-5)
                
                ## free memory
                O_norm = None; del O_norm
                S_norm = None; del S_norm
                Q = None; del Q
                gc.collect()
            
            # === λ₂
            
            if save_lambda2:
                
                t_start = timeit.default_timer()
                
                # === S² and Ω²
                SikSkj = np.einsum('xyzik,xyzkj->xyzij', S, S)
                OikOkj = np.einsum('xyzik,xyzkj->xyzij', O, O)
                #np.testing.assert_allclose(np.matmul(S,S), SikSkj, atol=1e-6)
                #np.testing.assert_allclose(np.matmul(O,O), OikOkj, atol=1e-6)
                
                ## free memory
                S = None; del S
                O = None; del O
                gc.collect()
                
                # === Eigenvalues of (S²+Ω²) --> a real symmetric (Hermitian) matrix
                eigvals            = np.linalg.eigvalsh(SikSkj+OikOkj, UPLO='L')
                #eigvals_sort_order = np.argsort(np.abs(eigvals), axis=3) ## sort order of λ --> magnitude (wrong)
                eigvals_sort_order = np.argsort(eigvals, axis=3) ## sort order of λ
                eigvals_sorted     = np.take_along_axis(eigvals, eigvals_sort_order, axis=3) ## do λ sort
                lambda2            = np.squeeze(eigvals_sorted[:,:,:,1]) ## λ2 is the second eigenvalue (index=1)
                t_delta            = timeit.default_timer() - t_start
                
                if verbose: tqdm.write(even_print('calc λ2','%s'%format_time_string(t_delta), s=True))
                
                dset = self['data/lambda2']
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                if self.usingmpi:
                    with dset.collective:
                        dset[ti,rz1_orig:rz2_orig,ry1_orig:ry2_orig,rx1_orig:rx2_orig] = lambda2[xA:xB,yA:yB,zA:zB].T
                else:
                    dset[ti,:,:,:] = lambda2.T
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = float_bytes * self.nx * self.ny * self.nz / 1024**3
                
                if verbose: tqdm.write(even_print('write λ2','%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## free memory
                lambda2 = None; del lambda2
                eigvals = None; del eigvals
                eigvals_sort_order = None; del eigvals_sort_order
                eigvals_sorted = None; del eigvals_sorted
                gc.collect()
            
            if verbose: progress_bar.update()
            if verbose and (ti<self.nt-1): tqdm.write( '---' )
        if verbose: progress_bar.close()
        if verbose: print(72*'-')
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.rgd.calc_lambda2() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def get_prime_from_ztmd(self, fn_ztmd, **kwargs):
        '''
        calculate primes with respect to [z,t] mean (ZTMD)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        
        chunk_kb         = kwargs.get('chunk_kb',2*1024) ## h5 chunk size: default 2 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        if verbose: print('\n'+'turbx.rgd.get_prime_from_ztmd()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_ztmd):
            raise FileNotFoundError(fn_ztmd)
        
        ## checks
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('ngp','%0.1f [M]'%(self.ngp/1e6,))
        if verbose: print(72*'-')
        
        ## report memory
        mem_total_gb = psutil.virtual_memory().total/1024**3
        mem_avail_gb = psutil.virtual_memory().available/1024**3
        mem_free_gb  = psutil.virtual_memory().free/1024**3
        if verbose: even_print('mem total', '%0.1f [GB]'%mem_total_gb)
        if verbose: even_print('mem available', '%0.1f [GB]'%mem_avail_gb)
        if verbose: even_print('mem free', '%0.1f [GB]'%mem_free_gb)
        if verbose: print(72*'-')
        
        ## ranks in [x,y,z]
        if self.usingmpi:
            
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        else:
            
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        if self.usingmpi: self.comm.Barrier()
        
        # ===
        
        with ztmd(fn_ztmd,'r') as f1: ## all ranks open ZTMD in serial mode
            
            x = np.copy(f1.x/f1.lchar) ## dimless, from ZTMD
            y = np.copy(f1.y/f1.lchar)
            
            if (x.ndim!=1):
                raise ValueError
            if (y.ndim!=1):
                raise ValueError
            
            xfi = []
            for x_ in self.x[rx1:rx2]:
                xfi.append( np.abs( x - x_ ).argmin() )
            xfi = np.array( xfi , dtype=np.int64 )
            if (self.x[rx1:rx2].shape[0] != xfi.shape[0]):
                print(f'something went wrong on rank {self.rank:d}')
                self.comm.Abort(1)
            if not np.allclose( self.x[rx1:rx2] , x[xfi] , rtol=1e-7 ):
                print(f'[x] coords not equal on rank {self.rank:d}')
                self.comm.Abort(1)
            
            yfi = []
            for y_ in self.y[ry1:ry2]:
                yfi.append( np.abs( y - y_ ).argmin() )
            yfi = np.array( yfi , dtype=np.int64 )
            if (self.y[ry1:ry2].shape[0] != yfi.shape[0]):
                print(f'something went wrong on rank {self.rank:d}')
                self.comm.Abort(1)
            if not np.allclose( self.y[ry1:ry2] , y[yfi] , rtol=1e-7 ):
                print(f'[y] coords not equal on rank {self.rank:d}')
                self.comm.Abort(1)
            
            ## initialize 4D datasets
            scalars_to_remove_zt_mean = []
            for scalar in self.scalars:
                if (scalar in f1.scalars) and (scalar in ['u','v','w']): ## if in ZTMD + extra conditions
                    
                    scalars_to_remove_zt_mean.append(scalar)
                    
                    dtype       = self[f'data/{scalar}'].dtype
                    itemsize    = dtype.itemsize
                    float_bytes = dtype.itemsize
                    
                    data_gb = float_bytes*self.nx*self.ny*self.nz*self.nt / 1024**3
                    shape   = (self.nt,self.nz,self.ny,self.nx)
                    chunks  = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                    
                    dsn = f'{scalar}I'
                    
                    if (f'data/{dsn}' in self):
                        del self[f'data/{dsn}']
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if verbose:
                        even_print(f'initializing data/{dsn}', f'{data_gb:0.2f} [GB]')
                    
                    dset = self.create_dataset(
                                            f'data/{dsn}', 
                                            shape=shape, 
                                            dtype=dtype,
                                            chunks=chunks,
                                            )
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    if verbose:
                        even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]')
                    
                    chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                    if verbose:
                        even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                        even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if verbose: print(72*'-')
            
            if verbose:
                progress_bar = tqdm(total=self.nt*len(scalars_to_remove_zt_mean), ncols=100, desc='prime', leave=False, file=sys.stdout, smoothing=0.)
            
            for ti in range(self.nt):
                for scalar in scalars_to_remove_zt_mean:
                    
                    ## name of output dset
                    dsn = f'{scalar}I'
                    
                    ## read
                    dset = self[f'data/{scalar}']
                    dtype = dset.dtype
                    float_bytes = dtype.itemsize
                    data_gb = float_bytes*self.nx*self.ny*self.nz * 1 / 1024**3
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            data = dset[ti,rz1:rz2,ry1:ry2,rx1:rx2].T
                    else:
                        data = dset[ti,:,:,:].T
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    if verbose:
                        tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    ## get 2D data from ZTMD (full)
                    data_mean = np.copy( f1[f'data/{scalar}'][()].T )
                    
                    ## indices for reduction (this rank!)
                    xxi,yyi = np.ix_(xfi,yfi)
                    
                    ## filter mean data
                    data_mean = np.copy( data_mean[xxi,yyi] )
                    
                    ## non-dimensionalize data from ZTMD
                    if scalar in ['u','v','w',]:
                        data_mean /= f1.U_inf
                    elif scalar in ['T',]:
                        data_mean /= f1.T_inf
                    elif scalar in ['rho',]:
                        data_mean /= f1.rho_inf
                    elif scalar in ['p',]:
                        data_mean /= ( f1.rho_inf * f1.U_inf**2 )
                    else:
                        raise ValueError(f"condition needed for redimensionalizing \'{str(scalar)}\'")
                    
                    ## perform mean removal
                    dataI = np.copy( data - data_mean[:,:,np.newaxis] )
                    
                    ## det to write to
                    dset = self[f'data/{dsn}']
                    
                    ## write
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            dset[ti,rz1:rz2,ry1:ry2,rx1:rx2] = dataI.T
                    else:
                        dset[ti,:,:,:] = dataI.T
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    
                    if verbose:
                        tqdm.write(even_print(f'write: {dsn}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: even_print(self.fname, '%0.2f [GB]'%(os.path.getsize(self.fname)/1024**3))
        
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.rgd.get_prime_from_ztmd() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_statistics_xpln(self,**kwargs):
        '''
        calculate statistics for an unsteady volumetric measurement
          which has a small number of points in the [x] direction
        - mean
        - covariance
        - skewness, kurtosis
        - probability distribution function (PDF)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_statistics_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## assert that the opened RGD has fsubtype 'unsteady' (i.e. is NOT a prime file)
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        cy = kwargs.get('cy',1) ## number of chunks per [y] rank range
        
        #n_threads = kwargs.get('n_threads',1)
        try:
            n_threads = int(os.environ.get('OMP_NUM_THREADS'))
        except TypeError: ## not set
            n_threads = os.cpu_count()
        
        #fn_dat_stats = kwargs.get('fn_dat_stats',None) ## filename for output pickle (.dat) file
        fn_h5_stats  = kwargs.get('fn_h5_stats',None) ## filename for output HDF5 (.h5) file
        
        n_bins       = kwargs.get('n_bins',512) ## n bins for histogram (PDF) calculation
        
        ## for now only distribute data in [y] --> allows [x,z] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if not isinstance(cy, (int,np.int32,np.int64)):
            raise TypeError
        
        # === distribute 4D data over ranks --> here only in [y]
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        ## check all [y] ranges have same size --> this limitation could be lifted
        for ryl_ in ryl:
            if not (ryl_[1]-ryl_[0]==nyr):
                raise ValueError('[y] chunks are not even in size')
        
        cyl_ = np.array_split( np.arange(ry1,ry2) , min(cy,nyr) )
        cyl  = [[b[0],b[-1]+1] for b in cyl_ ]
        
        ## assert no sub ranges are empty
        for nyc_ in [ cyl_[1]-cyl_[0] for cyl_ in cyl ]:
            if (nyc_ < 1):
                #raise ValueError
                print(f'rank {self.rank:d}: sub-range is <1')
                self.comm.Abort(1)
        
        if 1: ## assert that [y] sub-chunk ranges are correct
            
            yi = np.arange(self.ny, dtype=np.int32)
            
            local_indices = []
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                local_indices += [ yi_ for yi_ in yi[cy1:cy2] ]
            
            G = self.comm.gather([ self.rank , local_indices ], root=0)
            G = self.comm.bcast(G, root=0)
            
            all_indices = []
            for G_ in G:
                all_indices += G_[1]
            all_indices = np.array( sorted(all_indices), dtype=np.int32 )
            
            if not np.array_equal( all_indices , yi ):
                raise AssertionError
        
        # ## output filename : pickle (.dat)
        # if (fn_dat_stats is None):
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_stats_dat_base = fname_root+'_stats.dat'
        #     fn_dat_stats = str(PurePosixPath(fname_path, fname_stats_dat_base))
        # if (Path(fn_dat_stats).suffix != '.dat'):
        #     raise ValueError(f"fn_dat_stats='{str(fn_dat_stats)}' must end in .dat")
        
        ## output filename : HDF5 (.h5)
        if (fn_h5_stats is None): ## automatically determine name
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_stats_h5_base = fname_root+'_stats.h5'
            fn_h5_stats = str(PurePosixPath(fname_path, fname_stats_h5_base))
        if (Path(fn_h5_stats).suffix != '.h5'):
            raise ValueError(f"fn_h5_stats='{str(fn_h5_stats)}' must end in .h5")
        if os.path.isfile(fn_h5_stats):
            #if (os.path.getsize(fn_h5_stats) > 8*1024**3):
            #    raise ValueError(f"fn_h5_stats='{str(fn_h5_stats)}' exists and is >8 [GB]. exiting for your own safety.")
            if (fn_h5_stats == self.fname):
                raise ValueError(f"fn_h5_stats='{str(fn_h5_stats)}' cannot be same as input filename.")
        
        # ===
        
        if verbose: even_print( 'fn_h5'         , self.fname   )
        #if verbose: even_print( 'fn_dat_stats'  , fn_dat_stats )
        if verbose: even_print( 'fn_h5_stats'  , fn_h5_stats )
        if verbose: print(72*'-')
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        ## infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx',f'{self.nx:d}')
        if verbose: even_print('ny',f'{self.ny:d}')
        if verbose: even_print('nz',f'{self.nz:d}')
        if verbose: even_print('nt',f'{self.nt:d}')
        if verbose: even_print('ngp',f'{self.ngp/1e6:0.1f} [M]')
        if verbose: even_print('cy',f'{cy:d}')
        if verbose: even_print('n_ranks',f'{self.n_ranks:d}')
        if verbose: even_print('n_threads',f'{n_threads:d}')
        if verbose: print(72*'-')
        
        ## 0D freestream scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays & re-dimensionalize
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        ## assert constant Δz
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-6)):
            raise NotImplementedError('Δz not constant')
        dz = np.diff(z)[0]
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-12, atol=1e-12)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-12, atol=1e-12)
        
        zrange = z.max() - z.min()
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz'] = dz
        data['zrange'] = zrange
        
        if verbose: even_print( 'Δt/tchar'       , f'{dt/self.tchar:0.8f}' )
        if verbose: even_print( 'Δt'             , f'{dt:0.3e} [s]'        )
        if verbose: even_print( 'duration/tchar' , f'{self.duration:0.1f}' )
        if verbose: even_print( 'duration'       , f'{self.duration*self.tchar:0.3e} [s]' )
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('Δt'     , f'{dt    :0.5e} [s]' )
            even_print('t_meas' , f'{t_meas:0.5e} [s]' )
            even_print('Δz'     , f'{dz0   :0.5e} [m]' )
            even_print('zrange' , f'{zrange:0.5e} [m]' )
            #print(72*'-')
        
        # ===
        
        ## key    = str:scalar name
        ## value  = tuple:recipe,
        ##          bool:do_mean,
        ##          bool:do_pdf,
        ##          bool:do_skew_kurt
        ##
        ## recipe elements:
        ##    - if tuple : ( str:scalar , bool:ρ weighting ) --> always mean-removed
        ##    - if str   : str:scalar
        
        scalars_dict = {
            
            'r_uIIuII' : [ ( 'rho', ('u',True ), ('u',True ) ), True,True,True ], ## ρ·u″u″
            'r_vIIvII' : [ ( 'rho', ('v',True ), ('v',True ) ), True,True,True ], ## ρ·v″v″
            'r_wIIwII' : [ ( 'rho', ('w',True ), ('w',True ) ), True,True,True ], ## ρ·w″w″
            'r_uIIvII' : [ ( 'rho', ('u',True ), ('v',True ) ), True,True,True ], ## ρ·u″v″
            'r_uIIwII' : [ ( 'rho', ('u',True ), ('w',True ) ), True,True,True ], ## ρ·u″w″
            'r_vIIwII' : [ ( 'rho', ('v',True ), ('w',True ) ), True,True,True ], ## ρ·v″w″
            
            'uIuI'     : [ (        ('u',False), ('u',False) ), True,True,True ], ## u′u′
            'vIvI'     : [ (        ('v',False), ('v',False) ), True,True,True ], ## v′v′
            'wIwI'     : [ (        ('w',False), ('w',False) ), True,True,True ], ## w′w′
            'uIvI'     : [ (        ('u',False), ('v',False) ), True,True,True ], ## u′v′
            'uIwI'     : [ (        ('u',False), ('w',False) ), True,True,True ], ## u′w′
            'vIwI'     : [ (        ('v',False), ('w',False) ), True,True,True ], ## v′w′
            
            'uITI'     : [ (        ('u',False), ('T',False) ), True,True,True ], ## u′T′
            'vITI'     : [ (        ('v',False), ('T',False) ), True,True,True ], ## v′T′
            'wITI'     : [ (        ('w',False), ('T',False) ), True,True,True ], ## w′T′
            
            'TITI'     : [ (        ('T',False), ('T',False) ), True,True,False ], ## T′T′
            'TIITII'   : [ (        ('T',True ), ('T',True ) ), True,True,False ], ## T″T″
            'r_TIITII' : [ ( 'rho', ('T',True ), ('T',True ) ), True,True,False ], ## ρ·T″T″
            
            'rho'      : [ ( 'rho',              ) ,  True, True, False, ], ## ρ
            'rhoI'     : [ (      ('rho',False), ) , False, True,  True, ], ## ρ′
            
            'T'        : [ ( 'T',                )   ,  True, True, False, ], ## T
            'TI'       : [ (        ('T',False), )   , False, True,  True, ], ## T′
            'TII'      : [ (        ('T',True ), )   , False, True,  True, ], ## T″
            'r_TII'    : [ ( 'rho', ('T',True ), )   , False, True,  True, ], ## ρ·T″
            
            'p'        : [ ( 'p',              )   ,  True, True, False, ], ## p
            'pI'       : [ (      ('p',False), )   , False, True,  True, ], ## p′
            
            'u'        : [ ( 'u',                ) ,  True, True, False, ], ## u
            'uI'       : [ (        ('u',False), ) , False, True,  True, ], ## u′
            #'uII'      : [ (        ('u',True ), ) , False, True,  True, ], ## u″
            'r_uII'    : [ ( 'rho', ('u',True ), ) , False, True,  True, ], ## ρ·u″
            
            'v'        : [ ( 'v',                ) ,  True, True, False, ], ## v
            'vI'       : [ (        ('v',False), ) , False, True,  True, ], ## v′
            'r_vII'    : [ ( 'rho', ('v',True ), ) , False, True,  True, ], ## ρ·v″
            
            'w'        : [ ( 'w',                ) ,  True, True, False, ], ## w
            'wI'       : [ (        ('w',False), ) , False, True,  True, ], ## w′
            'r_wII'    : [ ( 'rho', ('w',True ), ) , False, True,  True, ], ## ρ·w″
            
            }
        
        dtype_unsteady = np.float64
        float_bytes = 8
        
        scalars_avg  = []
        scalars_pdf  = []
        scalars_hos_ = []
        for s,ss in scalars_dict.items():
            recipe, do_mean, do_pdf, do_skew_kurt = ss
            if do_mean:
                scalars_avg.append(s)
            if do_pdf:
                scalars_pdf.append(s)
            if do_skew_kurt:
                scalars_hos_.append(s)
        
        scalars_hos=[]
        for s_ in ['skew','kurt']:
            for ss_ in scalars_hos_:
                scalars_hos.append(f'{ss_}_{s_}')
        
        data_avg  = np.zeros(shape=(nyr,),          dtype={'names':scalars_avg, 'formats':[ np.float64 for sss in scalars_avg ]})
        data_bins = np.zeros(shape=(nyr,n_bins+1),  dtype={'names':scalars_pdf, 'formats':[ np.float64 for sss in scalars_pdf ]})
        data_pdf  = np.zeros(shape=(nyr,n_bins),    dtype={'names':scalars_pdf, 'formats':[ np.float64 for sss in scalars_pdf ]})
        data_hos  = np.zeros(shape=(nyr,),          dtype={'names':scalars_hos, 'formats':[ np.float64 for sss in scalars_hos ]})
        
        ## main loop
        self.comm.Barrier()
        if verbose:
            progress_bar = tqdm(total=cy*len(scalars_dict), ncols=100, desc='statistics', leave=False, file=sys.stdout, smoothing=0.)
        
        ## scalar dict loop
        for s,ss in scalars_dict.items(): ## e.g. where s='r_uIIvII' & ss=[ ( 'rho', ('u',True ), ('v',True ) ), True,True,False ]
            
            if verbose: tqdm.write(72*'-')
            
            recipe, do_mean, do_pdf, do_skew_kurt = ss
            
            if verbose:
                tqdm.write(even_print('computing',s,s=True,))
            
            if verbose: ## report per-rank read shape
                nyc_max = max([ cyl_[1]-cyl_[0] for cyl_ in cyl ])
                data_gb = 8 * nx * nyc_max * nz * nt / 1024**3
                tqdm.write(even_print('read shape', f'[{nx:d},{nyc_max:d},{nz:d},{nt:d}] · 8 [Bytes] --> {data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print('mem per read', f'{self.n_ranks*data_gb:0.1f} [GB]', s=True))
                #tqdm.write(even_print('mem per read ×3 / 245 [GB/Node]', f'{self.n_ranks*data_gb*3/245.:0.3f} [Nodes]', s=True))
                tqdm.write(even_print('mem per read ×4.5 / 245 [GB/Node]', f'{self.n_ranks*data_gb*4.5/245.:0.3f} [Nodes]', s=True)) ## 16Nx4R : resources_used.mem=2897098888kb
            
            ## should ρ be read?
            read_rho = False
            for s_ in recipe:
                if isinstance(s_, str) and (s_=='rho'):
                    read_rho = True
                elif isinstance(s_, str) and (s_!='rho'):
                    pass
                elif isinstance(s_, tuple):
                    if not isinstance(s_[1], bool):
                        raise ValueError
                    if s_[1]: ## i.e. density-weighted
                        read_rho = True
                else:
                    raise ValueError
            
            ## [y] loop outer (chunks within rank)
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                nyc = cy2 - cy1
                
                ## read ρ
                if read_rho:
                    
                    #rho = np.zeros(shape=(nx,nyc,nz,nt), dtype=dtype_unsteady) ## chunk range context
                    
                    dset = self[f'data/rho']
                    self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            #rho[:,:,:,:] = dset[:,:,cy1:cy2,:].T
                            rho = np.copy( dset[:,:,cy1:cy2,:].T )
                    else:
                        #rho[:,:,:,:] = dset[()].T
                        rho = np.copy( dset[()].T )
                    self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = dset.dtype.itemsize * self.nx * ry * (cy2-cy1) * self.nz * self.nt / 1024**3
                    if verbose:
                        tqdm.write(even_print(f'read: ρ', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    rho = rho.astype(np.float64) ## cast to double
                    
                    ## re-dimensionalize
                    rho *= self.rho_inf
                    
                    ## ρ mean in [t] --> leave [x,y,z,1]
                    rho_avg = np.mean(rho, axis=3, dtype=np.float64, keepdims=True)
                
                else:
                    rho     = None ; del rho
                    rho_avg = None ; del rho_avg
                
                ## product buffer for multiplication --> !!! notices ones() here and not zeros() !!!
                data_accum = np.ones(shape=(nx,nyc,nz,nt), dtype=dtype_unsteady) ## chunk range context
                
                ## read unsteady scalar data, remove mean, <density weight>, multiply
                for sss in recipe:
                    
                    if isinstance(sss, str) and (sss=='rho'): ## ρ
                        
                        ## multiply product accumulator, ρ was already read and is already dimensional
                        data_accum *= rho
                    
                    elif isinstance(sss, str) and (sss!='rho'): ## scalar which will NOT be mean-removed
                        
                        ## read
                        #data_X = np.zeros(shape=(nx,nyc,nz,nt), dtype=dtype_unsteady) ## chunk range context
                        dset = self[f'data/{sss}']
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                #data_X[:,:,:,:] = dset[:,:,cy1:cy2,:].T
                                data_X = np.copy( dset[:,:,cy1:cy2,:].T )
                        else:
                            #data_X[:,:,:,:] = dset[()].T
                            data_X = np.copy( dset[()].T )
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = dset.dtype.itemsize * self.nx * ry * (cy2-cy1) * self.nz * self.nt / 1024**3
                        if verbose:
                            tqdm.write(even_print(f'read: {sss}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        data_X = data_X.astype(np.float64) ## cast to double
                        
                        ## redimensionalize
                        if sss in ['u','v','w',]:
                            data_X *= self.U_inf
                        elif sss in ['T',]:
                            data_X *= self.T_inf
                        elif sss in ['rho',]:
                            data_X *= self.rho_inf
                        elif sss in ['p',]:
                            data_X *= self.rho_inf * self.U_inf**2
                        else:
                            raise ValueError(f"condition needed for redimensionalizing '{str(sss)}'")
                        
                        ## MULTIPLY product accumulator
                        data_accum *= data_X
                    
                    elif isinstance(sss, tuple): ## scalar which WILL be mean-removed (with- or without ρ-weighting)
                        
                        if (len(sss)!=2):
                            raise ValueError
                        
                        sn, do_density_weighting = sss ## e.g. ('u',True)
                        
                        ## read
                        #data_X = np.zeros(shape=(nx,nyc,nz,nt), dtype=dtype_unsteady) ## chunk range context
                        dset = self[f'data/{sn}']
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                #data_X[:,:,:,:] = dset[:,:,cy1:cy2,:].T
                                data_X = np.copy( dset[:,:,cy1:cy2,:].T )
                        else:
                            #data_X[:,:,:,:] = dset[()].T
                            data_X = np.copy( dset[()].T )
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = dset.dtype.itemsize * self.nx * ry * (cy2-cy1) * self.nz * self.nt / 1024**3
                        if verbose:
                            tqdm.write(even_print(f'read: {sn}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        data_X = data_X.astype(np.float64) ## cast to double
                        
                        ## redimensionalize
                        if sn in ['u','v','w',]:
                            data_X *= self.U_inf
                        elif sn in ['T',]:
                            data_X *= self.T_inf
                        elif sn in ['rho',]:
                            data_X *= self.rho_inf
                        elif sn in ['p',]:
                            data_X *= self.rho_inf * self.U_inf**2
                        else:
                            raise ValueError(f"condition needed for redimensionalizing '{str(sn)}'")
                        
                        ## avg(□) or avg(ρ·□)/avg(ρ)
                        if do_density_weighting:
                            data_X_mean = np.mean( rho*data_X , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                            data_X_mean /= rho_avg
                        else:
                            data_X_mean = np.mean( data_X , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                        
                        ## Reynolds prime □′ or Favre prime □″
                        data_X -= data_X_mean
                        
                        ## assert avg(□′)==0 or avg(ρ·□″)==0
                        if do_density_weighting:
                            a_ = np.mean(rho*data_X, axis=3, dtype=np.float64, keepdims=True)
                        else:
                            a_ = np.mean(data_X, axis=3, dtype=np.float64, keepdims=True)
                        #np.testing.assert_allclose(a_, np.zeros_like(a_), atol=1e-3)
                        if not np.allclose( a_, np.zeros_like(a_), atol=1e-3 ):
                            print(f'rank {self.rank:d}: avg(□′)!=0 or avg(ρ·□″)!=0')
                            self.comm.Abort(1)
                        
                        ## MULTIPLY product accumulator by □′ | □″
                        data_accum *= data_X
                    
                    else:
                        raise ValueError
                
                # ===============================================================================
                # At this point you have the UNSTEADY 4D [x,y,z,t] data according to 'recipe'
                # ===============================================================================
                
                yiA = cy1 - ry1
                yiB = cy2 - ry1
                
                ## mean in [t] --> leave [x,y,z,1]
                d_mean = np.mean( data_accum, axis=(3,), keepdims=True, dtype=np.float64)
                if ( d_mean.shape != (nx,nyc,nz,1,) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                if do_mean: ## mean in [x,z] --> leave [y]
                    data_avg[s][yiA:yiB] = np.squeeze( np.mean( d_mean, axis=(0,2), dtype=np.float64) )
                
                ## [y] loop inner (indices within chunk within rank)
                for yi in range(cy1,cy2):
                    yii  = yi - cy1 ## chunk local
                    yiii = yi - ry1 ## rank local
                    
                    if do_pdf:
                        
                        d_ = np.copy( data_accum[:,yii,:,:].ravel() )
                        if ( d_.shape != (nx*1*nz*nt,) ):
                            print(f'rank {self.rank:d}: shape violation')
                            self.comm.Abort(1)
                        pdf_ , bin_edges_ = np.histogram( d_ , bins=n_bins , density=True )
                        data_bins[s][yiii,:] = bin_edges_
                        data_pdf[s][yiii,:]  = pdf_
                    
                    if do_skew_kurt:
                        
                        #d_ = np.copy( data_accum[:,yii,:,:] ) ## [x,1,z,t]
                        d_ = np.zeros(shape=(nx,1,nz,nt), dtype=dtype_unsteady)
                        d_[:,:,:,:] = data_accum[:,yii,:,:][:,np.newaxis,:,:] ## [x,1,z,t]
                        if ( d_.shape != (nx,1,nz,nt) ):
                            print(f'rank {self.rank:d}: shape violation')
                            self.comm.Abort(1)
                        
                        ## ## is it interpretable as a Reynolds mean-removed quantity? (□′)
                        ## interpretable_as_Re_mean = False
                        ## a_ = np.mean(d_, axis=3, dtype=np.float64, keepdims=True) ## mean in [t]
                        ## np.testing.assert_equal( a_.shape, (nx,1,nz,1,) )
                        ## if np.allclose(a_, np.zeros_like(a_), atol=1e-10):
                        ##     interpretable_as_Re_mean = True
                        ## 
                        ## ## is it interpretable as a Favre mean-removed quantity? (□″)
                        ## interpretable_as_Fv_mean = False
                        ## if read_rho and not interpretable_as_Re_mean:
                        ##     #rho_ = np.copy( rho[:,yii,:,:] )
                        ##     rho_ = np.zeros(shape=(nx,1,nz,nt), dtype=dtype_unsteady)
                        ##     rho_[:,:,:,:] = rho[:,yii,:,:][:,np.newaxis,:,:] ## [x,1,z,t]
                        ##     np.testing.assert_equal( rho_.shape, (nx,1,nz,nt) )
                        ##     
                        ##     a_ = np.mean(d_*rho_, axis=3, dtype=np.float64, keepdims=True) ## mean in [t]
                        ##     np.testing.assert_equal( a_.shape, (nx,1,nz,1,) )
                        ##     if np.allclose(a_, np.zeros_like(a_), atol=1e-10):
                        ##         interpretable_as_Fv_mean = True
                        ## 
                        ## if (not interpretable_as_Re_mean) and (not interpretable_as_Fv_mean):
                        ##     print(f'{s} has non-zero Favre and Reynolds mean, i.e. is not <□′>!=0 and <ρ□″>!=0')
                        ##     self.comm.Abort(1)
                        
                        dI = np.copy( d_.ravel() ) ## □′ or □″
                        if ( dI.shape != (nx*1*nz*nt,) ):
                            print(f'rank {self.rank:d}: shape violation')
                            self.comm.Abort(1)
                        
                        d_std = np.sqrt( np.mean( dI**2 , dtype=np.float64 ) )
                        
                        if np.isclose(d_std, 0., atol=1e-08):
                            d_skew = 0.
                            d_kurt = 0.
                        else:
                            d_skew = np.mean( dI**3 , dtype=np.float64 ) / d_std**3
                            d_kurt = np.mean( dI**4 , dtype=np.float64 ) / d_std**4
                        
                        data_hos[f'{s}_skew'][yiii] = d_skew
                        data_hos[f'{s}_kurt'][yiii] = d_kurt
                
                self.comm.Barrier() ## [y] loop outer (chunks within rank)
                if verbose: progress_bar.update()
        
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        # ==============================================================
        # write HDF5 (.h5) file
        # ==============================================================
        
        ## open on rank 0 and write attributes, dimensions, etc.
        if (self.rank==0):
            with h5py.File(fn_h5_stats, 'w') as hfw:
                
                ## write floats,ints as top-level attributes
                for key,val in data.items():
                    if isinstance(data[key], (int,np.int32,np.int64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], (float,np.float32,np.float64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], np.ndarray):
                        pass
                    else:
                        print(f'key {key} is type {str(type(data[key]))}')
                        self.comm.Abort(1)
                
                ## write numpy arrays
                hfw.create_dataset( 'dims/x' , data=x ) ## [m]
                hfw.create_dataset( 'dims/y' , data=y ) ## [m]
                hfw.create_dataset( 'dims/z' , data=z ) ## [m]
                hfw.create_dataset( 'dims/t' , data=t ) ## [s]
        
        self.comm.Barrier()
        
        with h5py.File(fn_h5_stats, 'a', driver='mpio', comm=self.comm) as hfw:
            
            ## initialize datasets
            for scalar in scalars_avg:
                hfw.create_dataset( f'avg/{scalar}', shape=(ny,), dtype=np.float64, chunks=(1,) )
            for scalar in scalars_pdf:
                hfw.create_dataset( f'bins/{scalar}', shape=(ny,n_bins+1), dtype=np.float64, chunks=(1,n_bins+1) )
            for scalar in scalars_pdf:
                hfw.create_dataset( f'pdf/{scalar}', shape=(ny,n_bins), dtype=np.float64, chunks=(1,n_bins) )
            for scalar in scalars_hos:
                hfw.create_dataset( f'hos/{scalar}', shape=(ny,), dtype=np.float64, chunks=(1,) )
            
            ## collectively write data
            for scalar in scalars_avg:
                dset = hfw[f'avg/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = data_avg[scalar][:]
            for scalar in scalars_pdf:
                dset = hfw[f'bins/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:] = data_bins[scalar][:,:]
            for scalar in scalars_pdf:
                dset = hfw[f'pdf/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:] = data_pdf[scalar][:,:]
            for scalar in scalars_hos:
                dset = hfw[f'hos/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = data_hos[scalar][:]
        
        ## report file contents
        self.comm.Barrier()
        if (self.rank==0):
            with h5py.File(fn_h5_stats,'r') as hfr:
                h5_print_contents(hfr)
        self.comm.Barrier()
        
        self.comm.Barrier()
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_statistics_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_2D_pdfs(self, **kwargs):
        '''
        calculate 2D probability distribution functions e.g. PDF[u' × v'] & PDF[ρ·u″ × ρ·v″]
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        ## assert that the opened RGD has fsubtype 'unsteady' (as opposed to e.g. 'prime')
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        if verbose: print('\n'+'rgd.calc_2D_pdfs()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## assert that the opened RGD has fsubtype 'unsteady' (i.e. is NOT a prime file)
        if (self.fsubtype!='unsteady'):
            raise ValueError
        if not self.usingmpi:
            raise NotImplementedError('function is not implemented for non-MPI usage')
        
        h5py_is_mpi_build = h5py.h5.get_config().mpi
        if not h5py_is_mpi_build:
            if verbose: print('h5py was not compiled for parallel usage! exiting.')
            sys.exit(1)
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        cy = kwargs.get('cy','max') ## number of chunks per [y] rank range --> positive integer or 'max'
        
        ## #n_threads = kwargs.get('n_threads',1)
        ## try:
        ##     n_threads = int(os.environ.get('OMP_NUM_THREADS'))
        ## except TypeError: ## not set
        ##     n_threads = os.cpu_count()
        
        fn_h5_out = kwargs.get('fn_h5_out',None)
        n_bins    = kwargs.get('n_bins',512) ## n bins for histogram (PDF) calculation
        
        ## for now only distribute data in [y] --> allows [x,z] mean before Send/Recv
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # ===
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        ## check all [y] ranges have same size --> this limitation could be lifted with some work
        for ryl_ in ryl:
            if not (ryl_[1]-ryl_[0]==nyr):
                raise ValueError('[y] chunks are not even in size')
        
        ## check 'cy' input (n chunks within [y] rank range)
        if isinstance(cy, (int,np.int32,np.int64)):
            if (cy<1):
                #raise ValueError
                print(f'rank {self.rank:d}: sub-range is <1')
                self.comm.Abort(1)
        elif isinstance(cy,str):
            if (cy=='max'):
                cy = nyr
            else:
                raise ValueError(f"cy='{cy}' not a valid input, did you mean 'max'?")
        elif (cy is None):
            cy = nyr
        else:
            raise ValueError("cy should be positive integer, 'max' or None (='max')")
        
        ## get 'chunk' index ranges within rank range
        cyl_ = np.array_split( np.arange(ry1,ry2) , min(cy,nyr) )
        cyl  = [[b[0],b[-1]+1] for b in cyl_ ]
        
        ## assert no sub ranges are empty
        for nyc_ in [ cyl_[1]-cyl_[0] for cyl_ in cyl ]:
            if (nyc_ < 1):
                #raise ValueError
                print(f'rank {self.rank:d}: sub-range is <1')
                self.comm.Abort(1)
        
        if 1: ## check that [y] sub-chunk ranges are correct
            
            yi = np.arange(self.ny, dtype=np.int32)
            
            local_indices = []
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                local_indices += [ yi_ for yi_ in yi[cy1:cy2] ]
            
            G = self.comm.gather([ self.rank , local_indices ], root=0)
            G = self.comm.bcast(G, root=0)
            
            all_indices = []
            for G_ in G:
                all_indices += G_[1]
            all_indices = np.array( sorted(all_indices), dtype=np.int32 )
            
            if not np.array_equal( all_indices , yi ):
                raise AssertionError
        
        ## output filename : HDF5 (.h5)
        if (fn_h5_out is None): ## automatically determine name
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_tp_h5_base = fname_root+'_pdf2D.h5'
            fn_h5_out = str(PurePosixPath(fname_path, fname_tp_h5_base))
        if (Path(fn_h5_out).suffix != '.h5'):
            raise ValueError(f"fn_h5_out='{str(fn_h5_out)}' must end in .h5")
        if os.path.isfile(fn_h5_out):
            #if (os.path.getsize(fn_h5_out) > 8*1024**3):
            #    raise ValueError(f"fn_h5_out='{str(fn_h5_out)}' exists and is >8 [GB]. exiting for your own safety.")
            if (fn_h5_out == self.fname):
                raise ValueError(f"fn_h5_out='{str(fn_h5_out)}' cannot be same as input filename.")
        
        # ===
        
        if verbose: even_print( 'fn_h5'     , self.fname )
        if verbose: even_print( 'fn_h5_out' , fn_h5_out  )
        if verbose: print(72*'-')
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        ## infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx',f'{self.nx:d}')
        if verbose: even_print('ny',f'{self.ny:d}')
        if verbose: even_print('nz',f'{self.nz:d}')
        if verbose: even_print('nt',f'{self.nt:d}')
        if verbose: even_print('ngp',f'{self.ngp/1e6:0.1f} [M]')
        if verbose: even_print('cy',f'{cy:d}')
        if verbose: print(72*'-')
        
        ## 0D freestream scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays & re-dimensionalize
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        ## assert constant Δz
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-6)):
            raise NotImplementedError('Δz not constant')
        dz = np.diff(z)[0]
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-12, atol=1e-12)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-12, atol=1e-12)
        
        zrange = z.max() - z.min()
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz'] = dz
        data['zrange'] = zrange
        
        if verbose: even_print( 'Δt/tchar'       , f'{dt/self.tchar:0.8f}' )
        if verbose: even_print( 'Δt'             , f'{dt:0.3e} [s]'        )
        if verbose: even_print( 'duration/tchar' , f'{self.duration:0.1f}' )
        if verbose: even_print( 'duration'       , f'{self.duration*self.tchar:0.3e} [s]' )
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('Δt'     , f'{dt    :0.5e} [s]' )
            even_print('t_meas' , f'{t_meas:0.5e} [s]' )
            even_print('Δz'     , f'{dz0   :0.5e} [m]' )
            even_print('zrange' , f'{zrange:0.5e} [m]' )
            #print(72*'-')
        
        # ===
        
        ## key    = str:scalar name
        ## value  = list:[ tuple:recipe_L , tuple:recipe_R ],
        ##
        ## recipe elements:
        ##    - if tuple : ( str:scalar , bool:ρ weighting ) --> always mean-removed
        ##    - if str   : str:scalar
        
        scalars_dict = {
            'ruII_rvII' : [ ( 'rho', ('u',True ), ) , ( 'rho', ('v',True ), ) ], ## ρ·u″ × ρ·v″
            'rvII_rwII' : [ ( 'rho', ('v',True ), ) , ( 'rho', ('w',True ), ) ], ## ρ·v″ × ρ·w″
            'ruII_rwII' : [ ( 'rho', ('u',True ), ) , ( 'rho', ('w',True ), ) ], ## ρ·u″ × ρ·w″
            'ruII_TI'   : [ ( 'rho', ('u',True ), ) , (        ('T',False), ) ], ## ρ·u″ × T′
            'ruII_rTII' : [ ( 'rho', ('u',True ), ) , ( 'rho', ('T',True ), ) ], ## ρ·u″ × ρ·T″
            'rvII_rTII' : [ ( 'rho', ('v',True ), ) , ( 'rho', ('T',True ), ) ], ## ρ·v″ × ρ·T″
            'rvII_TI'   : [ ( 'rho', ('v',True ), ) , (        ('T',False), ) ], ## ρ·v″ × T′
            'uI_vI'     : [ (        ('u',False), ) , (        ('v',False), ) ], ## u′   × v′
            'vI_wI'     : [ (        ('v',False), ) , (        ('w',False), ) ], ## v′   × w′
            'uI_wI'     : [ (        ('u',False), ) , (        ('w',False), ) ], ## u′   × w′
            'uI_TI'     : [ (        ('u',False), ) , (        ('T',False), ) ], ## u′   × T′
            #'uI_rI'     : [ (        ('u',False), ) , (      ('rho',False), ) ], ## u′   × ρ′
            }
        
        dtype_unsteady = np.dtype(np.float64)
        float_bytes = 8
        
        data_bins = np.zeros(shape=(nyr,n_bins+1,2)    , dtype={'names':[ n for n in scalars_dict.keys() ], 'formats':[ np.float64 for sss in scalars_dict.keys() ]})
        data_pdf  = np.zeros(shape=(nyr,n_bins,n_bins) , dtype={'names':[ n for n in scalars_dict.keys() ], 'formats':[ np.float64 for sss in scalars_dict.keys() ]})
        
        ## main loop
        self.comm.Barrier()
        if verbose:
            progress_bar = tqdm(total=cy*len(scalars_dict), ncols=100, desc='2D PDF', leave=False, file=sys.stdout, smoothing=0.)
        
        ## scalar dict loop
        for s,ss in scalars_dict.items():
            
            if verbose: tqdm.write(72*'-')
            
            recipe_L, recipe_R = ss
            
            if verbose:
                tqdm.write(even_print('computing',s,s=True))
            
            ## report per-rank read shape
            ## 8N job reading 2x [y] layers: 1435702884kb / 245gb = 5.86
            ## fac=6 --> 5.325, so needs to be 6*(5.86/5.325)=6.6x as factor
            fac = 6.6
            if verbose:
                nyc_max = max([ cyl_[1]-cyl_[0] for cyl_ in cyl ])
                data_gb = 8 * nx * nyc_max * nz * nt / 1024**3
                tqdm.write(even_print('read shape', f'[{nx:d},{nyc_max:d},{nz:d},{nt:d}] · 8 [Bytes] --> {data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print('mem per read', f'{self.n_ranks*data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print(f'mem per read ×{fac:0.1f} / 245 [GB/Node]', f'{self.n_ranks*data_gb*fac/245.:0.3f} [Nodes]', s=True))
            
            ## should ρ be read?
            read_rho = False
            for recipe in [recipe_L,recipe_R]:
                for s_ in recipe:
                    if isinstance(s_, str) and (s_=='rho'):
                        read_rho = True
                    elif isinstance(s_, str) and (s_!='rho'):
                        pass
                    elif isinstance(s_, tuple):
                        if not isinstance(s_[1], bool):
                            raise ValueError
                        if s_[1]: ## i.e. density-weighted
                            read_rho = True
                    else:
                        raise ValueError
            del recipe
            
            ## [y] loop outer (chunks within rank)
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                nyc = cy2 - cy1
                
                ## read ρ
                if read_rho:
                    
                    dset = self[f'data/rho']
                    self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            rho = np.copy( dset[:,:,cy1:cy2,:].T )
                    else:
                        rho = np.copy( dset[()].T )
                    self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = dset.dtype.itemsize * self.nx * ry * (cy2-cy1) * self.nz * self.nt / 1024**3
                    if verbose:
                        tqdm.write(even_print(f'read: ρ', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    rho = rho.astype(np.float64) ## cast to double
                    
                    ## re-dimensionalize
                    rho *= self.rho_inf
                    
                    ## ρ mean in [t] --> leave [x,y,z,1]
                    rho_avg = np.mean(rho, axis=3, dtype=np.float64, keepdims=True)
                
                else:
                    rho     = None ; del rho
                    rho_avg = None ; del rho_avg
                
                ## product buffer for multiplication --> !!! notices ones() here and not zeros() !!!
                ## chunk range context
                data_accum_LR = np.ones(shape=(nx,nyc,nz,nt,2), dtype=dtype_unsteady)
                
                ## read unsteady scalar data, remove mean, <density weight>, multiply
                for recipe,iLR in [ (recipe_L,0), (recipe_R,1) ]:
                    for sss in recipe:
                        
                        if isinstance(sss, str) and (sss=='rho'): ## ρ
                            
                            ## multiply product accumulator, ρ was already read and is already dimensional
                            data_accum_LR[:,:,:,:,iLR] *= rho
                        
                        elif isinstance(sss, str) and (sss!='rho'): ## scalar which will NOT be mean-removed
                            
                            ## read
                            dset = self[f'data/{sss}']
                            self.comm.Barrier()
                            t_start = timeit.default_timer()
                            if self.usingmpi:
                                with dset.collective:
                                    data_X = np.copy( dset[:,:,cy1:cy2,:].T )
                            else:
                                data_X = np.copy( dset[()].T )
                            self.comm.Barrier()
                            t_delta = timeit.default_timer() - t_start
                            data_gb = dset.dtype.itemsize * self.nx * ry * (cy2-cy1) * self.nz * self.nt / 1024**3
                            if verbose:
                                tqdm.write(even_print(f'read: {sss}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                            
                            data_X = data_X.astype(np.float64) ## cast to double
                            
                            ## redimensionalize
                            if sss in ['u','v','w',]:
                                data_X *= self.U_inf
                            elif sss in ['T',]:
                                data_X *= self.T_inf
                            elif sss in ['rho',]:
                                data_X *= self.rho_inf
                            elif sss in ['p',]:
                                data_X *= self.rho_inf * self.U_inf**2
                            else:
                                raise ValueError(f"condition needed for redimensionalizing '{str(sss)}'")
                            
                            ## MULTIPLY product accumulator
                            data_accum_LR[:,:,:,:,iLR] *= data_X
                            
                            data_X = None ; del data_X
                        
                        elif isinstance(sss, tuple): ## scalar which WILL be mean-removed (with- or without ρ-weighting)
                            
                            if (len(sss)!=2):
                                raise ValueError
                            
                            sn, do_density_weighting = sss ## e.g. ('u',True)
                            
                            ## read
                            dset = self[f'data/{sn}']
                            self.comm.Barrier()
                            t_start = timeit.default_timer()
                            if self.usingmpi:
                                with dset.collective:
                                    data_X = np.copy( dset[:,:,cy1:cy2,:].T )
                            else:
                                data_X = np.copy( dset[()].T )
                            self.comm.Barrier()
                            t_delta = timeit.default_timer() - t_start
                            data_gb = dset.dtype.itemsize * self.nx * ry * (cy2-cy1) * self.nz * self.nt / 1024**3
                            if verbose:
                                tqdm.write(even_print(f'read: {sn}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                            
                            data_X = data_X.astype(np.float64) ## cast to double
                            
                            ## redimensionalize
                            if sn in ['u','v','w',]:
                                data_X *= self.U_inf
                            elif sn in ['T',]:
                                data_X *= self.T_inf
                            elif sn in ['rho',]:
                                data_X *= self.rho_inf
                            elif sn in ['p',]:
                                data_X *= self.rho_inf * self.U_inf**2
                            else:
                                raise ValueError(f"condition needed for redimensionalizing '{str(sn)}'")
                            
                            ## avg(□) or avg(ρ·□)/avg(ρ)
                            if do_density_weighting:
                                data_X_mean = np.mean( rho*data_X , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                                data_X_mean /= rho_avg
                            else:
                                data_X_mean = np.mean( data_X , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                            
                            ## Reynolds prime □′ or Favre prime □″
                            data_X -= data_X_mean
                            
                            ## assert avg(□′)==0 or avg(ρ·□″)==0
                            if do_density_weighting:
                                a_ = np.mean(rho*data_X, axis=3, dtype=np.float64, keepdims=True)
                            else:
                                a_ = np.mean(data_X, axis=3, dtype=np.float64, keepdims=True)
                            #np.testing.assert_allclose(a_, np.zeros_like(a_), atol=1e-3)
                            if not np.allclose( a_, np.zeros_like(a_), atol=1e-3 ):
                                print(f'rank {self.rank:d}: avg(□′)!=0 or avg(ρ·□″)!=0')
                                self.comm.Abort(1)
                            
                            ## MULTIPLY product accumulator by □′ | □″
                            data_accum_LR[:,:,:,:,iLR] *= data_X
                            
                            data_X = None ; del data_X
                        
                        else:
                            raise ValueError
                
                # ===============================================================================
                # At this point you have the UNSTEADY 4D [x,y,z,t] data according to 'recipe'
                # ===============================================================================
                
                yiA = cy1 - ry1
                yiB = cy2 - ry1
                
                ## [y] loop inner (indices within chunk within rank)
                for yi in range(cy1,cy2):
                    yii  = yi - cy1 ## chunk local
                    yiii = yi - ry1 ## rank local
                    
                    dL_ = np.copy( data_accum_LR[:,yii,:,:,0].ravel() )
                    dR_ = np.copy( data_accum_LR[:,yii,:,:,1].ravel() )
                    
                    if ( dL_.shape != (nx*1*nz*nt,) ) or ( dR_.shape != (nx*1*nz*nt,) ):
                        print(f'rank {self.rank:d}: shape violation')
                        self.comm.Abort(1)
                    
                    ## compute 2D PDF
                    pdf_ , bin_edges_L_ , bin_edges_R_ = np.histogram2d( dL_ , dR_ , bins=n_bins , density=True )
                    
                    data_bins[s][yiii,:,0] = bin_edges_L_
                    data_bins[s][yiii,:,1] = bin_edges_R_
                    data_pdf[s][yiii,:,:]  = pdf_
                
                self.comm.Barrier() ## [y] loop outer (chunks within rank)
                if verbose: progress_bar.update()
        
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        # ==============================================================
        # write HDF5 (.h5) file
        # ==============================================================
        
        ## open on rank 0 and write attributes, dimensions, etc.
        if (self.rank==0):
            with h5py.File(fn_h5_out, 'w') as hfw:
                
                ## write floats,ints as top-level attributes
                for key,val in data.items():
                    if isinstance(data[key], (int,np.int32,np.int64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], (float,np.float32,np.float64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], np.ndarray):
                        pass
                    else:
                        print(f'key {key} is type {str(type(data[key]))}')
                        self.comm.Abort(1)
                
                ## write numpy arrays
                hfw.create_dataset( 'dims/x' , data=x ) ## [m]
                hfw.create_dataset( 'dims/y' , data=y ) ## [m]
                hfw.create_dataset( 'dims/z' , data=z ) ## [m]
                hfw.create_dataset( 'dims/t' , data=t ) ## [s]
        
        self.comm.Barrier()
        
        with h5py.File(fn_h5_out, 'a', driver='mpio', comm=self.comm) as hfw:
            
            ## initialize datasets
            for scalar in scalars_dict.keys():
                hfw.create_dataset( f'bins/{scalar}', shape=(ny,n_bins+1,2), dtype=np.float64, chunks=(1,n_bins+1,1) )
            for scalar in scalars_dict.keys():
                hfw.create_dataset( f'pdf/{scalar}', shape=(ny,n_bins,n_bins), dtype=np.float64, chunks=(1,n_bins,n_bins) )
            
            ## collectively write data
            for scalar in scalars_dict.keys():
                dset = hfw[f'bins/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:,:] = data_bins[scalar][:,:,:]
            for scalar in scalars_dict.keys():
                dset = hfw[f'pdf/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:,:] = data_pdf[scalar][:,:,:]
        
        ## report file contents
        self.comm.Barrier()
        if (self.rank==0):
            with h5py.File(fn_h5_out,'r') as hfr:
                h5_print_contents(hfr)
        
        self.comm.Barrier()
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_2D_pdfs() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_turb_cospectrum_xpln(self, **kwargs):
        '''
        calculate FFT cospectrum in [z,t] at every [x,y]
        - designed for analyzing unsteady, thin planes in [x]
        - multithreaded, make sure OMP_NUM_THREADS is set
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_turb_cospectrum_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## assert that the opened RGD has fsubtype 'unsteady' (i.e. is NOT a prime file)
        if (self.fsubtype!='unsteady'):
            raise ValueError
        if not self.usingmpi:
            raise NotImplementedError('function is not implemented for non-MPI usage')
        
        h5py_is_mpi_build = h5py.h5.get_config().mpi
        if not h5py_is_mpi_build:
            if verbose: print('h5py was not compiled for parallel usage! exiting.')
            sys.exit(1)
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        cy = kwargs.get('cy',1) ## number of chunks per rank [y] range
        
        #n_threads = kwargs.get('n_threads',1)
        try:
            n_threads = int(os.environ.get('OMP_NUM_THREADS'))
        except TypeError: ## not set
            n_threads = os.cpu_count()
        
        #fn_dat_fft = kwargs.get('fn_dat_fft',None) ## filename for output pickle (.dat) file
        fn_h5_fft  = kwargs.get('fn_h5_fft',None) ## filename for output HDF5 (.h5) file
        
        overlap_fac_nom = kwargs.get('overlap_fac_nom',0.50)
        n_win           = kwargs.get('n_win',8)
        window_type     = kwargs.get('window_type','hann') ## 'tukey','hann'
        
        ## only distribute data across [y]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if not isinstance(cy, (int,np.int32,np.int64)):
            raise TypeError
        
        ## distribute 4D data over ranks --> here only in [y]
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        ## check all [y] ranges have same size
        for ryl_ in ryl:
            if not (ryl_[1]-ryl_[0]==nyr):
                raise ValueError('[y] chunks are not even in size')
        
        ## [y] sub chunk range --> cyl = list of ranges in ry1:ry2
        ## cy is the NUMBER of chunks for the rank sub-range
        cyl_ = np.array_split( np.arange(ry1,ry2) , min(cy,nyr) )
        cyl  = [[b[0],b[-1]+1] for b in cyl_ ]
        
        ## check sub ranges
        for nyc_ in [ cyl_[1]-cyl_[0] for cyl_ in cyl ]:
            if (nyc_ < 1):
                #raise ValueError
                print(f'rank {self.rank:d}: sub-range is <1')
                self.comm.Abort(1)
        
        if 1: ## assert that [y] sub-chunk ranges are correct
            
            yi = np.arange(self.ny, dtype=np.int32)
            
            local_indices = []
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                local_indices += [ yi_ for yi_ in yi[cy1:cy2] ]
            
            G = self.comm.gather([ self.rank , local_indices ], root=0)
            G = self.comm.bcast(G, root=0)
            
            all_indices = []
            for G_ in G:
                all_indices += G_[1]
            all_indices = np.array( sorted(all_indices), dtype=np.int32 )
            
            if not np.array_equal( all_indices , yi ):
                raise AssertionError
        
        # ## output filename : pickle (.dat)
        # if (fn_dat_fft is None): ## automatically determine name
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_fft_dat_base = fname_root+'_fft.dat'
        #     fn_dat_fft = str(PurePosixPath(fname_path, fname_fft_dat_base))
        # if (Path(fn_dat_fft).suffix != '.dat'):
        #     raise ValueError(f"fn_dat_fft='{str(fn_dat_fft)}' must end in .dat")
        
        ## output filename : HDF5 (.h5)
        if (fn_h5_fft is None): ## automatically determine name
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_h5_base = fname_root+'_fft.h5'
            fn_h5_fft = str(PurePosixPath(fname_path, fname_fft_h5_base))
        if (Path(fn_h5_fft).suffix != '.h5'):
            raise ValueError(f"fn_h5_fft='{str(fn_h5_fft)}' must end in .h5")
        if os.path.isfile(fn_h5_fft):
            #if (os.path.getsize(fn_h5_fft) > 8*1024**3):
            #    raise ValueError(f"fn_h5_fft='{str(fn_h5_fft)}' exists and is >8 [GB]. exiting for your own safety.")
            if (fn_h5_fft == self.fname):
                raise ValueError(f"fn_h5_fft='{str(fn_h5_fft)}' cannot be same as input filename.")
        
        if verbose: even_print( 'fn_h5'      , self.fname )
        #if verbose: even_print( 'fn_dat_fft' , fn_dat_fft )
        if verbose: even_print( 'fn_h5_fft'  , fn_h5_fft  )
        if verbose: print(72*'-')
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        ## infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx',f'{self.nx:d}')
        if verbose: even_print('ny',f'{self.ny:d}')
        if verbose: even_print('nz',f'{self.nz:d}')
        if verbose: even_print('nt',f'{self.nt:d}')
        if verbose: even_print('ngp',f'{self.ngp/1e6:0.1f} [M]')
        if verbose: even_print('cy',f'{cy:d}')
        if verbose: even_print('n_ranks',f'{self.n_ranks:d}')
        if verbose: even_print('n_threads',f'{n_threads:d}')
        if verbose: print(72*'-')
        
        ## 0D freestream scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays & re-dimensionalize
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        ## assert constant Δz
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-6)):
            raise NotImplementedError('Δz not constant')
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-12, atol=1e-12)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-12, atol=1e-12)
        
        zrange = z.max() - z.min()
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print( 'Δt/tchar'       , f'{dt/self.tchar:0.8f}' )
        if verbose: even_print( 'Δt'             , f'{dt:0.3e} [s]'        )
        if verbose: even_print( 'duration/tchar' , f'{self.duration:0.1f}' )
        if verbose: even_print( 'duration'       , f'{self.duration*self.tchar:0.3e} [s]' )
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('Δt'     , f'{dt    :0.5e} [s]' )
            even_print('t_meas' , f'{t_meas:0.5e} [s]' )
            even_print('Δz'     , f'{dz0   :0.5e} [m]' )
            even_print('zrange' , f'{zrange:0.5e} [m]' )
            print(72*'-')
        
        ## establish [t] windowing
        win_len, overlap = get_overlapping_window_size(nt, n_win, overlap_fac_nom)
        overlap_fac = overlap / win_len
        tw, n_win, n_pad = get_overlapping_windows(t, win_len, overlap)
        t_meas_per_win = (win_len-1)*dt
        data['win_len']        = win_len
        data['overlap_fac']    = overlap_fac
        data['overlap']        = overlap
        data['n_win']          = n_win
        data['t_meas_per_win'] = t_meas_per_win
        
        if verbose:
            even_print('overlap_fac (nominal)' , f'{overlap_fac_nom:0.5f}'  )
            even_print('n_win'                 , f'{n_win:d}'               )
            even_print('win_len'               , f'{win_len:d}'             )
            even_print('overlap'               , f'{overlap:d}'             )
            even_print('overlap_fac'           , f'{overlap_fac:0.5f}'      )
            even_print('n_pad'                 , f'{n_pad:d}'               )
            #even_print('t_win/(δ99/uτ)'        , '%0.3f [-]'%t_eddy_per_win )
            print(72*'-')
        
        ## temporal [t] frequency (f) vector (Short Time Fourier Transform)
        freq_full = sp.fft.fftfreq(n=win_len, d=dt)
        fp        = np.where(freq_full>0) ## indices of positive values
        freq      = np.copy(freq_full[fp])
        df        = freq[1] - freq[0]
        nf        = freq.size
        
        data['freq'] = freq
        data['df']   = df
        data['nf']   = nf
        
        if verbose:
            even_print('freq min',f'{freq.min():0.1f} [Hz]')
            even_print('freq max',f'{freq.max():0.1f} [Hz]')
            even_print('df',f'{df:0.1f} [Hz]')
            even_print('nf',f'{nf:d}')
            #period_eddy = (1/freq) / (d99/u_tau)
            #period_plus = (1/freq) / sc_t_in
            #even_print('min : period+ = (1/f)/(νw/uτ²)'     , '%0.5f [-]'%period_plus.min())
            #even_print('max : period_eddy = (1/f)/(δ99/uτ)' , '%0.5f [-]'%period_eddy.max())
            print(72*'-')
        
        ## spatial [x] wavenumber (kx) and wavelength (λx)
        # λx  = u/f
        # kx  = 2·π·f/u
        # λx+ = λx/(ν/uτ)
        # kx+ = (2·π·f/u)·(ν/uτ)
        na = np.newaxis
        
        ## kx = 2·π·f/u --> [y,f]
        #kx = np.copy( 2*np.pi*freq[na,:] / u_avg[:,na] )
        
        ## λx = u/f --> [y,f]
        #lx = np.copy( u_avg[:,na] / freq[na,:] )
        
        #data['kx'] = kx
        #data['lx'] = lx
        
        ## spatial [z] wavenumber (kz) vector
        kz_full = sp.fft.fftfreq(n=nz, d=dz0) * ( 2 * np.pi )
        kzp     = np.where(kz_full>0) ## indices of positive values
        kz      = np.copy(kz_full[kzp])
        dkz     = kz[1] - kz[0]
        nkz     = kz.shape[0]
        
        ## wavenumber vector should be size nz//2-1
        if (nkz!=nz//2-1):
            raise ValueError
        
        data['kz']  = kz
        data['dkz'] = dkz
        data['nkz'] = nkz
        
        if verbose:
            even_print('kz min',f'{kz.min():0.1f} [1/m]')
            even_print('kz max',f'{kz.max():0.1f} [1/m]')
            even_print('dkz',f'{dkz:0.1f} [1/m]')
            even_print('nkz',f'{nkz:d}')
            print(72*'-')
        
        ## wavelength λz = (2·π)/kz
        lz = np.copy( 2 * np.pi / kz )
        data['lz'] = lz
        
        # ===
        
        ## cospectrum pairs
        ## [ str:var1, str:var2, bool:do_density_weighting]
        fft_combis = [
        [ 'u' , 'u' , True  ], ## FFT[ ρ·u″ , ρ·u″ ]
        [ 'v' , 'v' , True  ], ## FFT[ ρ·v″ , ρ·v″ ]
        [ 'w' , 'w' , True  ], ## FFT[ ρ·w″ , ρ·w″ ]
        [ 'u' , 'v' , True  ], ## FFT[ ρ·u″ , ρ·v″ ]
        [ 'u' , 'w' , True  ], ## FFT[ ρ·u″ , ρ·w″ ]
        [ 'v' , 'w' , True  ], ## FFT[ ρ·v″ , ρ·w″ ]
        [ 'u' , 'u' , False ], ## FFT[ u′ , u′ ]
        [ 'v' , 'v' , False ], ## FFT[ v′ , v′ ]
        [ 'w' , 'w' , False ], ## FFT[ w′ , w′ ]
        [ 'u' , 'v' , False ], ## FFT[ u′ , v′ ]
        [ 'u' , 'w' , False ], ## FFT[ u′ , w′ ]
        [ 'v' , 'w' , False ], ## FFT[ v′ , w′ ]
        ]
        
        ## generate FFT cospectrum scalar names
        scalars = []
        for fft_combi in fft_combis:
            s1,s2,do_density_weighting = fft_combi
            if do_density_weighting:
                scalars.append(f'r{s1}II_r{s2}II')
            else:
                scalars.append(f'{s1}I_{s2}I')
        
        scalars_dtypes = [ np.dtype(np.float64) for s in scalars ]
        
        ## generate avg scalar names
        scalars_Re_avg = []
        scalars_Fv_avg = []
        for fft_combi in fft_combis:
            s1,s2,do_density_weighting = fft_combi
            if do_density_weighting and ('rho' not in scalars_Re_avg):
                scalars_Re_avg.append('rho')
            if do_density_weighting:
                if (s1 not in scalars_Fv_avg):
                    scalars_Fv_avg.append(s1)
                if (s2 not in scalars_Fv_avg):
                    scalars_Fv_avg.append(s2)
            else:
                if (s1 not in scalars_Re_avg):
                    scalars_Re_avg.append(s1)
                if (s2 not in scalars_Re_avg):
                    scalars_Re_avg.append(s2)
        
        ## numpy formatted arrays: buffers for PSD & other data (rank-local)
        Ekz        = np.zeros(shape=(nyr,nkz ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        Ef         = np.zeros(shape=(nyr,nf  ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        covariance = np.zeros(shape=(nyr,    ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        avg_Re     = np.zeros(shape=(nyr,    ) , dtype={'names':scalars_Re_avg, 'formats':[ np.dtype(np.float64) for s in scalars_Re_avg ]})
        avg_Fv     = np.zeros(shape=(nyr,    ) , dtype={'names':scalars_Fv_avg, 'formats':[ np.dtype(np.float64) for s in scalars_Fv_avg ]})
        
        if verbose:
            even_print('n turb spectrum scalar combinations' , '%i'%(len(fft_combis),))
            print(72*'-')
        
        ## no window for [z]
        window_z = np.ones(nz, dtype=np.float64)
        sum_sqrt_win_z = np.sum(np.sqrt(window_z))
        if verbose:
            #even_print('sum(sqrt(window_z))'     , '%0.5f'%(sum_sqrt_win_z,))
            even_print('sum(sqrt(window_z)) / nz', '%0.5f'%(sum_sqrt_win_z/nz,))
        
        ## window function for [t]
        if (window_type=='tukey'):
            window_t = sp.signal.windows.tukey(win_len,alpha=0.5,sym=False) ## α=0:rectangular, α=1:Hann
        elif (window_type=='hann'):
            window_t = sp.signal.windows.hann(win_len,sym=False)
        elif (window_type is None):
            window_t = np.ones(win_len, dtype=np.float64)
        else:
            raise ValueError
        
        if verbose:
            even_print('window type [t]', '\'%s\''%str(window_type))
        
        ## sum of sqrt of window: needed for normalization
        sum_sqrt_win_t = np.sum(np.sqrt(window_t))
        if verbose:
            #even_print('sum(sqrt(window_t))'          , '%0.5f'%(sum_sqrt_win_t,))
            even_print('sum(sqrt(window_t)) / win_len', '%0.5f'%(sum_sqrt_win_t/win_len,))
        
        #if verbose: print(72*'-')
        
        # === main loop
        
        if verbose:
            progress_bar = tqdm(total=len(fft_combis)*cy, ncols=100, desc='fft', leave=False, file=sys.stdout, smoothing=0.)
        
        for cci,cc in enumerate(fft_combis): ## fft pairs
            
            if verbose: tqdm.write(72*'-')
            
            scalar_L, scalar_R, do_density_weighting = cc
            
            if do_density_weighting:
                msg = f'FFT[ρ·{scalar_L}″,ρ·{scalar_R}″]'
            else:
                msg = f'FFT[{scalar_L}′,{scalar_R}′]'
            if verbose:
                tqdm.write(even_print('computing',msg,s=True,))
            
            if verbose: ## report per-rank read shape
                nyc_max = max([ cyl_[1]-cyl_[0] for cyl_ in cyl ])
                data_gb = 8 * nx * nyc_max * nz * nt / 1024**3
                tqdm.write(even_print('read shape', f'[{nx:d},{nyc_max:d},{nz:d},{nt:d}] · 8 [Bytes] --> {data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print('mem per read', f'{self.n_ranks*data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print('mem per read ×4.5 / 245 [GB/Node]', f'{self.n_ranks*data_gb*4.5/245.:0.3f} [Nodes]', s=True))
            
            dset_L   = self[f'data/{scalar_L}']
            dset_R   = self[f'data/{scalar_R}']
            dset_rho = self[f'data/rho']
            
            scalar = scalars[cci]
            
            ## assert scalar name
            if do_density_weighting:
                if (f'r{scalar_L}II_r{scalar_R}II' != scalar ):
                    raise ValueError
            else:
                if (f'{scalar_L}I_{scalar_R}I' != scalar ):
                    raise ValueError
            
            ## [y] loop outer (chunks within rank)
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                nyc = cy2 - cy1
                
                self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## read data L
                n_scalars_read = 1 ## initialize
                scalar_str = scalar_L
                with dset_L.collective:
                    data_L = np.copy( dset_L[:,:,cy1:cy2,:].T ).astype(np.float64)
                
                ## read data R (if != data L)
                if (scalar_L==scalar_R):
                    data_R = np.copy( data_L )
                else:
                    n_scalars_read += 1
                    scalar_str += f',{scalar_R}'
                    with dset_R.collective:
                        data_R = np.copy( dset_R[:,:,cy1:cy2,:].T ).astype(np.float64)
                
                ## read ρ
                if do_density_weighting:
                    n_scalars_read += 1
                    scalar_str += ',ρ'
                    with dset_rho.collective:
                        rho = np.copy( dset_rho[:,:,cy1:cy2,:].T ).astype(np.float64)
                else:
                    rho = None
                
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = n_scalars_read * ( self.nx * ry * (cy2-cy1) * self.nz * self.nt * dset_L.dtype.itemsize ) / 1024**3
                if verbose:
                    tqdm.write(even_print(f'read: {scalar_str}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## data_L and data_R should be [nx,nyc,nz,nt] where nyc is the chunk [y] range
                if ( data_L.shape != (nx,nyc,nz,nt) ) or ( data_R.shape != (nx,nyc,nz,nt) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                if (rho is not None) and ( rho.shape != (nx,nyc,nz,nt) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                # === redimensionalize
                
                if scalar_L in ['u','v','w',]:
                    data_L *= U_inf
                else:
                    raise ValueError
                
                if scalar_R in ['u','v','w',]:
                    data_R *= U_inf
                else:
                    raise ValueError
                
                if (rho is not None): ## i.e. do_density_weighting
                    rho *= rho_inf
                
                # === compute mean-removed data
                
                ## avg(□) or avg(ρ·□)/avg(ρ) in [t]
                if do_density_weighting:
                    rho_avg     = np.mean(        rho , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                    data_L_avg  = np.mean( rho*data_L , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                    data_L_avg /= rho_avg
                    data_R_avg  = np.mean( rho*data_R , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                    data_R_avg /= rho_avg
                else:
                    data_L_avg = np.mean( data_L , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                    data_R_avg = np.mean( data_R , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                
                ## Reynolds prime □′ or Favre prime □″
                data_L -= data_L_avg
                data_R -= data_R_avg
                
                ## assert stationarity / definition averaging
                ## avg(□′)==0 or avg(ρ·□″)==0
                if do_density_weighting:
                    a_ = np.mean(rho*data_L, axis=3, dtype=np.float64, keepdims=True)
                    b_ = np.mean(rho*data_R, axis=3, dtype=np.float64, keepdims=True)
                else:
                    a_ = np.mean(data_L, axis=3, dtype=np.float64, keepdims=True)
                    b_ = np.mean(data_R, axis=3, dtype=np.float64, keepdims=True)
                if not np.allclose( a_, np.zeros_like(a_), atol=1e-6 ) or not np.allclose( b_, np.zeros_like(b_), atol=1e-6 ):
                    print(f'rank {self.rank:d}: avg(□′)!=0 or avg(ρ·□″)!=0')
                    self.comm.Abort(1)
                
                ## covariance: <□′·□′> OR <ρ□″·ρ□″> --> note that this is NOT the typical Favre <ρ·□″□″>
                if do_density_weighting:
                    covariance_ = np.mean( rho*data_L * rho*data_R , axis=3 , dtype=np.float64, keepdims=True)
                else:
                    covariance_ = np.mean( data_L*data_R , axis=3 , dtype=np.float64, keepdims=True)
                
                ## write this chunk/scalar's covariance to covariance buffer
                ## avg over [x,z] : [x,y,z,1] --> [y]
                yiA = cy1 - ry1
                yiB = cy2 - ry1
                covariance[scalar][yiA:yiB] = np.squeeze( np.mean( covariance_ , axis=(0,2,3) , dtype=np.float64) )
                
                ## write (rank-local) 1D [y] averages
                if do_density_weighting:
                    avg_Fv[scalar_L][yiA:yiB] = np.squeeze( np.mean( data_L_avg , axis=(0,2,3) , dtype=np.float64) )
                    avg_Fv[scalar_R][yiA:yiB] = np.squeeze( np.mean( data_R_avg , axis=(0,2,3) , dtype=np.float64) )
                    avg_Re['rho'][yiA:yiB]    = np.squeeze( np.mean( rho_avg    , axis=(0,2,3) , dtype=np.float64) )
                else:
                    avg_Re[scalar_L][yiA:yiB] = np.squeeze( np.mean( data_L_avg , axis=(0,2,3) , dtype=np.float64) )
                    avg_Re[scalar_R][yiA:yiB] = np.squeeze( np.mean( data_R_avg , axis=(0,2,3) , dtype=np.float64) )
                
                # ===============================================================================
                # At this point you have 4D [x,y,z,t] [□′,□′] or [ρ·□″,ρ·□″] data
                # ===============================================================================
                
                def __fft_z_thread_kernel(xi,ti,yii,do_density_weighting):
                    
                    ## 1D [z] □′ or ρ·□″ vectors
                    if do_density_weighting:
                        uL = rho[xi,yii,:,ti] * data_L[xi,yii,:,ti]
                        uR = rho[xi,yii,:,ti] * data_R[xi,yii,:,ti]
                    else:
                        uL = data_L[xi,yii,:,ti]
                        uR = data_R[xi,yii,:,ti]
                    
                    ## amplitude spectra
                    A1 = sp.fft.fft( uL * window_z )[kzp] / sum_sqrt_win_z
                    A2 = sp.fft.fft( uR * window_z )[kzp] / sum_sqrt_win_z
                    
                    ## energy spectral density approximation
                    E_ijl = 2. * np.real(A1*np.conj(A2)) / dkz
                    
                    return xi,ti,E_ijl
                
                def __fft_t_thread_kernel(xi,zi,yii,do_density_weighting):
                    
                    ## 1D [z] □′ or ρ·□″ vectors
                    if do_density_weighting:
                        uL = rho[xi,yii,zi,:] * data_L[xi,yii,zi,:]
                        uR = rho[xi,yii,zi,:] * data_R[xi,yii,zi,:]
                    else:
                        uL = data_L[xi,yii,zi,:]
                        uR = data_R[xi,yii,zi,:]
                    
                    uL_, nw, n_pad = get_overlapping_windows(uL, win_len, overlap)
                    uR_, nw, n_pad = get_overlapping_windows(uR, win_len, overlap)
                    
                    ## STFT buffer
                    E_ijk_buf = np.zeros((nw,nf), dtype=np.float64)
                    
                    ## compute fft for each overlapped window segment
                    for wi in range(nw):
                        A1 = sp.fft.fft( uL_[wi,:] * window_t )[fp] / sum_sqrt_win_t
                        A2 = sp.fft.fft( uR_[wi,:] * window_t )[fp] / sum_sqrt_win_t
                        E_ijk_buf[wi,:] = 2. * np.real(A1*np.conj(A2)) / df
                    
                    ## mean across short time FFT (STFT) segments
                    E_ijk = np.mean(E_ijk_buf, axis=0, dtype=np.float64)
                    
                    return xi,zi,E_ijk
                
                # ===============================================================================
                
                ## [y] loop inner (indices within chunk)
                for yi in range(cy1,cy2):
                    
                    yii  = yi - cy1 ## chunk local
                    yiii = yi - ry1 ## rank local
                    
                    ## PSD buffers for [y] loop inner
                    E_xt = np.zeros((nx,nt,nkz) , dtype=np.float64) ## [x,t] range for FFT(z)
                    E_xz = np.zeros((nx,nz,nf ) , dtype=np.float64) ## [x,z] range for FFT(t)
                    
                    # ===========================================================================
                    # FFT(z) : loop over [x,t]
                    # ===========================================================================
                    
                    ## concurrent/threaded execution for fft(z)
                    tasks = [(xi,ti,yii,do_density_weighting) for xi in range(nx) for ti in range(nt)]
                    with ThreadPoolExecutor(max_workers=n_threads) as executor:
                        results = executor.map(lambda t: __fft_z_thread_kernel(*t,), tasks)
                        for xi,ti,result in results:
                            E_xt[xi,ti,:] = result
                    
                    ## for xi in range(nx):
                    ##     for ti in range(nt):
                    ##         ...
                    
                    ## avg in [x,t] & write in rank context
                    Ekz[scalar][yiii,:] = np.mean(E_xt, axis=(0,1), dtype=np.float64)
                    
                    # ===========================================================================
                    # FFT(t) : loop over [x,z], use windows
                    # ===========================================================================
                    
                    ## concurrent/threaded execution for fft(t)
                    tasks = [(xi,zi,yii,do_density_weighting) for xi in range(nx) for zi in range(nz)]
                    with ThreadPoolExecutor(max_workers=n_threads) as executor:
                        results = executor.map(lambda t: __fft_t_thread_kernel(*t,), tasks)
                        for xi,zi,result in results:
                            E_xz[xi,zi,:] = result
                    
                    ## for xi in range(nx):
                    ##     for zi in range(nz):
                    ##         ...
                    
                    ## avg in [x,z] & write in rank context
                    Ef[scalar][yiii,:] = np.mean(E_xz, axis=(0,1), dtype=np.float64)
                
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print(msg, format_time_string(t_delta), s=True))
                if verbose: progress_bar.update() ## (scalar, [y] chunk) progress
        if verbose: progress_bar.close()
        self.comm.Barrier()
        if verbose: print(72*'-')
        
        # ==============================================================
        # write HDF5 (.h5) file
        # ==============================================================
        
        ## open on rank 0 and write attributes, dimensions, etc.
        if (self.rank==0):
            with h5py.File(fn_h5_fft, 'w') as hfw:
                
                ## write floats,ints as top-level attributes
                for key,val in data.items():
                    if isinstance(data[key], (int,np.int32,np.int64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], (float,np.float32,np.float64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], np.ndarray):
                        pass
                    else:
                        print(f'key {key} is type {str(type(data[key]))}')
                        self.comm.Abort(1)
                
                ## write numpy arrays
                hfw.create_dataset( 'dims/x'    , data=x    ) ## [m]
                hfw.create_dataset( 'dims/y'    , data=y    ) ## [m]
                hfw.create_dataset( 'dims/z'    , data=z    ) ## [m]
                hfw.create_dataset( 'dims/t'    , data=t    ) ## [s]
                hfw.create_dataset( 'dims/freq' , data=freq ) ## [1/s] | [Hz]
                hfw.create_dataset( 'dims/kz'   , data=kz   ) ## [1/m]
                hfw.create_dataset( 'dims/lz'   , data=lz   ) ## [m]
        
        self.comm.Barrier()
        
        with h5py.File(fn_h5_fft, 'a', driver='mpio', comm=self.comm) as hfw:
            
            ## initialize datasets
            for scalar in scalars:
                hfw.create_dataset( f'covariance/{scalar}' , shape=(ny,),    dtype=np.float64, chunks=(1,)    )
                hfw.create_dataset( f'Ekz/{scalar}'        , shape=(ny,nkz), dtype=np.float64, chunks=(1,nkz) )
                hfw.create_dataset( f'Ef/{scalar}'         , shape=(ny,nf ), dtype=np.float64, chunks=(1,nf)  )
            
            ## initialize datasets 1D [y] mean
            for scalar in avg_Re.dtype.names:
                hfw.create_dataset( f'avg/Re/{scalar}', shape=(ny,), dtype=np.float64, chunks=(1,) )
            for scalar in avg_Fv.dtype.names:
                hfw.create_dataset( f'avg/Fv/{scalar}', shape=(ny,), dtype=np.float64, chunks=(1,) )
            
            self.comm.Barrier()
            
            ## collectively write covariance,Ekz,Ef
            for scalar in scalars:
                dset = hfw[f'covariance/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = covariance[scalar][:]
                dset = hfw[f'Ekz/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:] = Ekz[scalar][:,:]
                dset = hfw[f'Ef/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:] = Ef[scalar][:,:]
            
            ## collectively write 1D [y] avgs
            for scalar in avg_Re.dtype.names:
                dset = hfw[f'avg/Re/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = avg_Re[scalar][:]
            for scalar in avg_Fv.dtype.names:
                dset = hfw[f'avg/Fv/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = avg_Fv[scalar][:]
        
        self.comm.Barrier()
        if verbose:
            print('--w-> %s : %0.2f [MB]'%(fn_h5_fft,os.path.getsize(fn_h5_fft)/1024**2))
            print(72*'-')
        
        ## report file contents
        self.comm.Barrier()
        if (self.rank==0):
            with h5py.File(fn_h5_fft,'r') as hfr:
                h5_print_contents(hfr)
        self.comm.Barrier()
        
        self.comm.Barrier()
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_turb_cospectrum_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_ccor_xpln(self, **kwargs):
        '''
        calculate cross-correlation in [z,t] at every [x,y]
        - designed for analyzing unsteady, thin planes in [x]
        - multithreaded, make sure OMP_NUM_THREADS is set
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_ccor_xpln()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## assert that the opened RGD has fsubtype 'unsteady' (i.e. is NOT a prime file)
        if (self.fsubtype!='unsteady'):
            raise ValueError
        if not self.usingmpi:
            raise NotImplementedError('function is not implemented for non-MPI usage')
        
        h5py_is_mpi_build = h5py.h5.get_config().mpi
        if not h5py_is_mpi_build:
            if verbose: print('h5py was not compiled for parallel usage! exiting.')
            sys.exit(1)
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        cy = kwargs.get('cy',1) ## number of chunks per rank [y] range
        
        #n_threads = kwargs.get('n_threads',1)
        try:
            n_threads = int(os.environ.get('OMP_NUM_THREADS'))
        except TypeError: ## not set
            n_threads = os.cpu_count()
        
        #fn_dat_ccor = kwargs.get('fn_dat_ccor',None) ## filename for output pickle (.dat) file
        fn_h5_ccor  = kwargs.get('fn_h5_ccor',None) ## filename for output HDF5 (.h5) file
        
        #overlap_fac_nom = kwargs.get('overlap_fac_nom',0.5)
        #n_win           = kwargs.get('n_win',8)
        
        ## only distribute data across [y]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if not isinstance(cy, (int,np.int32,np.int64)):
            raise TypeError
        
        ## distribute 4D data over ranks --> here only in [y]
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        ## check all [y] ranges have same size (1/2) --> not strictly necessary
        if (self.ny%self.n_ranks!=0):
            raise ValueError('ny%n_ranks!=0')
        
        ## check all [y] ranges have same size (2/2) --> not strictly necessary
        for ryl_ in ryl:
            if not (ryl_[1]-ryl_[0]==nyr):
                raise ValueError('[y] chunks are not even in size')
        
        ## [y] sub chunk range --> cyl = list of ranges in ry1:ry2
        ## cy is the NUMBER of chunks for the rank sub-range
        cyl_ = np.array_split( np.arange(ry1,ry2) , min(cy,nyr) )
        cyl  = [[b[0],b[-1]+1] for b in cyl_ ]
        
        ## check sub ranges
        for nyc_ in [ cyl_[1]-cyl_[0] for cyl_ in cyl ]:
            if (nyc_ < 1):
                #raise ValueError
                print(f'rank {self.rank:d}: sub-range is <1')
                self.comm.Abort(1)
        
        if 1: ## assert that [y] sub-chunk ranges are correct
            
            yi = np.arange(self.ny, dtype=np.int32)
            
            local_indices = []
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                local_indices += [ yi_ for yi_ in yi[cy1:cy2] ]
            
            G = self.comm.gather([ self.rank , local_indices ], root=0)
            G = self.comm.bcast(G, root=0)
            
            all_indices = []
            for G_ in G:
                all_indices += G_[1]
            all_indices = np.array( sorted(all_indices), dtype=np.int32 )
            
            if not np.array_equal( all_indices , yi ):
                raise AssertionError
        
        # ## output filename : pickle (.dat)
        # if (fn_dat_ccor is None): ## automatically determine name
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_ccor_dat_base = fname_root+'_ccor.dat'
        #     fn_dat_ccor = str(PurePosixPath(fname_path, fname_ccor_dat_base))
        # if (Path(fn_dat_ccor).suffix != '.dat'):
        #     raise ValueError(f"fn_dat_ccor='{str(fn_dat_ccor)}' must end in .dat")
        
        ## output filename : HDF5 (.h5)
        if (fn_h5_ccor is None): ## automatically determine name
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_ccor_h5_base = fname_root+'_ccor.h5'
            fn_h5_ccor = str(PurePosixPath(fname_path, fname_ccor_h5_base))
        if (Path(fn_h5_ccor).suffix != '.h5'):
            raise ValueError(f"fn_h5_ccor='{str(fn_h5_ccor)}' must end in .h5")
        if os.path.isfile(fn_h5_ccor):
            #if (os.path.getsize(fn_h5_ccor) > 8*1024**3):
            #    raise ValueError(f"fn_h5_ccor='{str(fn_h5_ccor)}' exists and is >8 [GB]. exiting for your own safety.")
            if (fn_h5_ccor == self.fname):
                raise ValueError(f"fn_h5_ccor='{str(fn_h5_ccor)}' cannot be same as input filename.")
        
        if verbose: even_print( 'fn_h5'       , self.fname  )
        #if verbose: even_print( 'fn_dat_ccor' , fn_dat_ccor )
        if verbose: even_print( 'fn_h5_ccor'  , fn_h5_ccor  )
        if verbose: print(72*'-')
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        ## infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx',f'{self.nx:d}')
        if verbose: even_print('ny',f'{self.ny:d}')
        if verbose: even_print('nz',f'{self.nz:d}')
        if verbose: even_print('nt',f'{self.nt:d}')
        if verbose: even_print('ngp',f'{self.ngp/1e6:0.1f} [M]')
        if verbose: even_print('cy',f'{cy:d}')
        if verbose: even_print('n_ranks',f'{self.n_ranks:d}')
        if verbose: even_print('n_threads',f'{n_threads:d}')
        if verbose: print(72*'-')
        
        ## 0D freestream scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays & re-dimensionalize
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        ## assert constant Δz
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-6)):
            raise NotImplementedError('Δz not constant')
        dz = np.diff(z)[0]
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-12, atol=1e-12)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-12, atol=1e-12)
        
        zrange = z.max() - z.min()
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz'] = dz
        data['zrange'] = zrange
        
        if verbose: even_print( 'Δt/tchar'       , f'{dt/self.tchar:0.8f}' )
        if verbose: even_print( 'Δt'             , f'{dt:0.3e} [s]'        )
        if verbose: even_print( 'duration/tchar' , f'{self.duration:0.1f}' )
        if verbose: even_print( 'duration'       , f'{self.duration*self.tchar:0.3e} [s]' )
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('Δt'     , f'{dt    :0.5e} [s]' )
            even_print('t_meas' , f'{t_meas:0.5e} [s]' )
            even_print('Δz'     , f'{dz0   :0.5e} [m]' )
            even_print('zrange' , f'{zrange:0.5e} [m]' )
            print(72*'-')
        
        # ## establish [t] windowing
        # win_len, overlap = get_overlapping_window_size(nt, n_win, overlap_fac_nom)
        # overlap_fac = overlap / win_len
        # tw, n_win, n_pad = get_overlapping_windows(t, win_len, overlap)
        # t_meas_per_win = (win_len-1)*dt
        # data['win_len']        = win_len
        # data['overlap_fac']    = overlap_fac
        # data['overlap']        = overlap
        # data['n_win']          = n_win
        # data['t_meas_per_win'] = t_meas_per_win
        # 
        # if verbose:
        #     even_print('overlap_fac (nominal)' , f'{overlap_fac_nom:0.5f}'  )
        #     even_print('n_win'                 , f'{n_win:d}'               )
        #     even_print('win_len'               , f'{win_len:d}'             )
        #     even_print('overlap'               , f'{overlap:d}'             )
        #     even_print('overlap_fac'           , f'{overlap_fac:0.5f}'      )
        #     even_print('n_pad'                 , f'{n_pad:d}'               )
        #     #even_print('t_win/(δ99/uτ)'        , '%0.3f [-]'%t_eddy_per_win )
        #     print(72*'-')
        
        ## get lags [t]
        #lags_t,_  = ccor( np.ones(win_len,dtype=np.float32) , np.ones(win_len,dtype=np.float32), get_lags=True )
        #n_lags_t_ = win_len*2-1
        lags_t,_  = ccor( np.ones(nt,dtype=np.float32) , np.ones(nt,dtype=np.float32), get_lags=True )
        n_lags_t_ = nt*2-1
        n_lags_t  = lags_t.shape[0]
        if (n_lags_t!=n_lags_t_):
            raise AssertionError('check lags [t]')
        
        data['lags_t']   = lags_t
        data['n_lags_t'] = n_lags_t
        
        if verbose:
            even_print('n lags (Δt)' , '%i'%(n_lags_t,))
        
        ## get lags [z]
        lags_z,_  = ccor( np.ones(nz,dtype=np.float32) , np.ones(nz,dtype=np.float32), get_lags=True )
        n_lags_z_ = nz*2-1
        n_lags_z  = lags_z.shape[0]
        if (n_lags_z!=n_lags_z_):
            raise AssertionError('check lags [z]')
        
        data['lags_z']   = lags_z
        data['n_lags_z'] = n_lags_z
        
        if verbose:
            even_print('n lags (Δz)' , '%i'%(n_lags_z,))
        
        # ===
        
        ## cross-correlation pairs
        ## [ str:var1, str:var2, bool:do_density_weighting]
        ccor_combis = [
        [ 'u' , 'u' , True  ], ## ccor[ ρ·u″ , ρ·u″ ]
        [ 'v' , 'v' , True  ], ## ccor[ ρ·v″ , ρ·v″ ]
        [ 'w' , 'w' , True  ], ## ccor[ ρ·w″ , ρ·w″ ]
        [ 'u' , 'v' , True  ], ## ccor[ ρ·u″ , ρ·v″ ]
        [ 'u' , 'w' , True  ], ## ccor[ ρ·u″ , ρ·w″ ]
        [ 'v' , 'w' , True  ], ## ccor[ ρ·v″ , ρ·w″ ]
        [ 'u' , 'u' , False ], ## ccor[ u′ , u′ ]
        [ 'v' , 'v' , False ], ## ccor[ v′ , v′ ]
        [ 'w' , 'w' , False ], ## ccor[ w′ , w′ ]
        [ 'u' , 'v' , False ], ## ccor[ u′ , v′ ]
        [ 'u' , 'w' , False ], ## ccor[ u′ , w′ ]
        [ 'v' , 'w' , False ], ## ccor[ v′ , w′ ]
        ]
        
        ## generate cross-correlation scalar names
        scalars = []
        for ccor_combi in ccor_combis:
            s1,s2,do_density_weighting = ccor_combi
            if do_density_weighting:
                scalars.append(f'r{s1}II_r{s2}II')
            else:
                scalars.append(f'{s1}I_{s2}I')
        
        scalars_dtypes = [ np.dtype(np.float64) for s in scalars ]
        
        ## generate avg scalar names
        scalars_Re_avg = []
        scalars_Fv_avg = []
        for ccor_combi in ccor_combis:
            s1,s2,do_density_weighting = ccor_combi
            if do_density_weighting and ('rho' not in scalars_Re_avg):
                scalars_Re_avg.append('rho')
            if do_density_weighting:
                if (s1 not in scalars_Fv_avg):
                    scalars_Fv_avg.append(s1)
                if (s2 not in scalars_Fv_avg):
                    scalars_Fv_avg.append(s2)
            else:
                if (s1 not in scalars_Re_avg):
                    scalars_Re_avg.append(s1)
                if (s2 not in scalars_Re_avg):
                    scalars_Re_avg.append(s2)
        
        ## numpy formatted arrays: buffers for PSD & other data (rank-local)
        Rz         = np.zeros(shape=(nyr,n_lags_z ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        Rt         = np.zeros(shape=(nyr,n_lags_t ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        covariance = np.zeros(shape=(nyr,    ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        avg_Re     = np.zeros(shape=(nyr,    ) , dtype={'names':scalars_Re_avg, 'formats':[ np.dtype(np.float64) for s in scalars_Re_avg ]})
        avg_Fv     = np.zeros(shape=(nyr,    ) , dtype={'names':scalars_Fv_avg, 'formats':[ np.dtype(np.float64) for s in scalars_Fv_avg ]})
        
        if verbose:
            even_print('n cross-correlation scalar combinations' , f'{len(ccor_combis):d}')
        
        # === main loop
        
        if verbose:
            progress_bar = tqdm(total=len(ccor_combis)*cy, ncols=100, desc='ccor', leave=False, file=sys.stdout, smoothing=0.)
        
        for cci,cc in enumerate(ccor_combis): ## ccor pairs
            
            if verbose: tqdm.write(72*'-')
            
            scalar_L, scalar_R, do_density_weighting = cc
            
            if do_density_weighting:
                msg = f'ccor[ρ·{scalar_L}″,ρ·{scalar_R}″]'
            else:
                msg = f'ccor[{scalar_L}′,{scalar_R}′]'
            if verbose:
                tqdm.write(even_print('computing',msg,s=True,))
            
            if verbose: ## report per-rank read shape
                nyc_max = max([ cyl_[1]-cyl_[0] for cyl_ in cyl ])
                data_gb = 8 * nx * nyc_max * nz * nt / 1024**3
                tqdm.write(even_print('read shape', f'[{nx:d},{nyc_max:d},{nz:d},{nt:d}] · 8 [Bytes] --> {data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print('mem per read', f'{self.n_ranks*data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print('mem per read ×5.5 / 245 [GB/Node]', f'{self.n_ranks*data_gb*5.5/245.:0.3f} [Nodes]', s=True))
            
            dset_L   = self[f'data/{scalar_L}']
            dset_R   = self[f'data/{scalar_R}']
            dset_rho = self[f'data/rho']
            
            scalar = scalars[cci]
            
            ## assert scalar name
            if do_density_weighting:
                if (f'r{scalar_L}II_r{scalar_R}II' != scalar ):
                    raise ValueError
            else:
                if (f'{scalar_L}I_{scalar_R}I' != scalar ):
                    raise ValueError
            
            ## [y] loop outer (chunks within rank)
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                nyc = cy2 - cy1
                
                self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## read data L
                n_scalars_read = 1 ## initialize
                scalar_str = scalar_L
                with dset_L.collective:
                    data_L = np.copy( dset_L[:,:,cy1:cy2,:].T ).astype(np.float64)
                
                ## read data R (if != data L)
                if (scalar_L==scalar_R):
                    data_R = np.copy( data_L )
                else:
                    n_scalars_read += 1
                    scalar_str += f',{scalar_R}'
                    with dset_R.collective:
                        data_R = np.copy( dset_R[:,:,cy1:cy2,:].T ).astype(np.float64)
                
                ## read ρ
                if do_density_weighting:
                    n_scalars_read += 1
                    scalar_str += ',ρ'
                    with dset_rho.collective:
                        rho = np.copy( dset_rho[:,:,cy1:cy2,:].T ).astype(np.float64)
                else:
                    rho = None
                
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = n_scalars_read * ( self.nx * ry * (cy2-cy1) * self.nz * self.nt * dset_L.dtype.itemsize ) / 1024**3
                if verbose:
                    tqdm.write(even_print(f'read: {scalar_str}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## data_L and data_R should be [nx,nyc,nz,nt] where nyc is the chunk [y] range
                if ( data_L.shape != (nx,nyc,nz,nt) ) or ( data_R.shape != (nx,nyc,nz,nt) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                if (rho is not None) and ( rho.shape != (nx,nyc,nz,nt) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                # === redimensionalize
                
                if scalar_L in ['u','v','w',]:
                    data_L *= U_inf
                else:
                    raise ValueError
                
                if scalar_R in ['u','v','w',]:
                    data_R *= U_inf
                else:
                    raise ValueError
                
                if (rho is not None): ## i.e. if do_density_weighting
                    rho *= rho_inf
                
                # === compute mean-removed data
                
                ## avg(□) or avg(ρ·□)/avg(ρ) in [t]
                if do_density_weighting:
                    rho_avg     = np.mean(        rho , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                    data_L_avg  = np.mean( rho*data_L , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                    data_L_avg /= rho_avg
                    data_R_avg  = np.mean( rho*data_R , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                    data_R_avg /= rho_avg
                else:
                    data_L_avg = np.mean( data_L , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                    data_R_avg = np.mean( data_R , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                
                ## Reynolds prime □′ or Favre prime □″
                data_L -= data_L_avg
                data_R -= data_R_avg
                
                ## assert stationarity / definition averaging
                ## avg(□′)==0 or avg(ρ·□″)==0
                if do_density_weighting:
                    a_ = np.mean(rho*data_L, axis=3, dtype=np.float64, keepdims=True)
                    b_ = np.mean(rho*data_R, axis=3, dtype=np.float64, keepdims=True)
                else:
                    a_ = np.mean(data_L, axis=3, dtype=np.float64, keepdims=True)
                    b_ = np.mean(data_R, axis=3, dtype=np.float64, keepdims=True)
                if not np.allclose( a_, np.zeros_like(a_), atol=1e-6 ) or not np.allclose( b_, np.zeros_like(b_), atol=1e-6 ):
                    print(f'rank {self.rank:d}: avg(□′)!=0 or avg(ρ·□″)!=0')
                    self.comm.Abort(1)
                
                ## covariance: <□′·□′> OR <ρ□″·ρ□″> --> note that this is NOT the typical Favre <ρ·□″□″>
                if do_density_weighting:
                    covariance_ = np.mean( rho*data_L * rho*data_R , axis=3 , dtype=np.float64, keepdims=True)
                else:
                    covariance_ = np.mean( data_L*data_R , axis=3 , dtype=np.float64, keepdims=True)
                
                ## write this chunk/scalar's covariance to covariance buffer
                ## avg over [x,z] : [x,y,z,1] --> [y]
                yiA = cy1 - ry1
                yiB = cy2 - ry1
                covariance[scalar][yiA:yiB] = np.squeeze( np.mean( covariance_ , axis=(0,2,3) , dtype=np.float64) )
                
                ## write (rank-local) 1D [y] averages
                if do_density_weighting:
                    avg_Fv[scalar_L][yiA:yiB] = np.squeeze( np.mean( data_L_avg , axis=(0,2,3) , dtype=np.float64) )
                    avg_Fv[scalar_R][yiA:yiB] = np.squeeze( np.mean( data_R_avg , axis=(0,2,3) , dtype=np.float64) )
                    avg_Re['rho'][yiA:yiB]    = np.squeeze( np.mean( rho_avg    , axis=(0,2,3) , dtype=np.float64) )
                else:
                    avg_Re[scalar_L][yiA:yiB] = np.squeeze( np.mean( data_L_avg , axis=(0,2,3) , dtype=np.float64) )
                    avg_Re[scalar_R][yiA:yiB] = np.squeeze( np.mean( data_R_avg , axis=(0,2,3) , dtype=np.float64) )
                
                # ===============================================================================
                # At this point you have 4D [x,y,z,t] [□′,□′] or [ρ·□″,ρ·□″] data
                # ===============================================================================
                
                def __ccor_z_thread_kernel(xi,ti,yii,do_density_weighting):
                    if do_density_weighting:
                        uL = rho[xi,yii,:,ti] * data_L[xi,yii,:,ti]
                        uR = rho[xi,yii,:,ti] * data_R[xi,yii,:,ti]
                    else:
                        uL = data_L[xi,yii,:,ti]
                        uR = data_R[xi,yii,:,ti]
                    return xi,ti,ccor(uL,uR)
                
                def __ccor_t_thread_kernel(xi,zi,yii,do_density_weighting):
                    if do_density_weighting:
                        uL = rho[xi,yii,zi,:] * data_L[xi,yii,zi,:]
                        uR = rho[xi,yii,zi,:] * data_R[xi,yii,zi,:]
                    else:
                        uL = data_L[xi,yii,zi,:]
                        uR = data_R[xi,yii,zi,:]
                    return xi,zi,ccor(uL,uR)
                
                self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## [y] loop inner (indices within chunk)
                for yi in range(cy1,cy2):
                    
                    yii  = yi - cy1 ## chunk local
                    yiii = yi - ry1 ## rank local
                    
                    ## ccor buffers for [y] loop inner
                    R_xt = np.zeros((nx,nt,n_lags_z) , dtype=np.float64) ## [x,t] range for ccor(z)
                    R_xz = np.zeros((nx,nz,n_lags_t) , dtype=np.float64) ## [x,z] range for ccor(t)
                    
                    # ===========================================================================
                    # ccor(z) : loop over [x,t]
                    # ===========================================================================
                    
                    ## concurrent/threaded execution for ccor(z)
                    tasks = [(xi,ti,yii,do_density_weighting) for xi in range(nx) for ti in range(nt)]
                    with ThreadPoolExecutor(max_workers=n_threads) as executor:
                        results = executor.map(lambda t: __ccor_z_thread_kernel(*t,), tasks)
                        for xi,ti,result in results:
                            R_xt[xi,ti,:] = result
                    
                    # for xi in range(nx):
                    #     for ti in range(nt):
                    #         
                    #         ## 1D [z] □′ or ρ·□″ vectors
                    #         if do_density_weighting:
                    #             uL = np.copy( rho[xi,yii,:,ti] * data_L[xi,yii,:,ti] )
                    #             uR = np.copy( rho[xi,yii,:,ti] * data_R[xi,yii,:,ti] )
                    #         else:
                    #             uL = np.copy( data_L[xi,yii,:,ti] )
                    #             uR = np.copy( data_R[xi,yii,:,ti] )
                    #         
                    #         R_xt[xi,ti,:] = ccor( uL , uR )
                    
                    ## avg in [x,t] & write in rank context
                    Rz[scalar][yiii,:] = np.mean(R_xt, axis=(0,1), dtype=np.float64)
                    
                    # ===========================================================================
                    # ccor(t) : loop over [x,z]
                    # ===========================================================================
                    
                    ## concurrent/threaded execution for ccor(t)
                    tasks = [(xi,zi,yii,do_density_weighting) for xi in range(nx) for zi in range(nz)]
                    with ThreadPoolExecutor(max_workers=n_threads) as executor:
                        results = executor.map(lambda t: __ccor_t_thread_kernel(*t,), tasks)
                        for xi,zi,result in results:
                            R_xz[xi,zi,:] = result
                    
                    # for xi in range(nx):
                    #     for zi in range(nz):
                    #         
                    #         ## 1D [z] □′ or ρ·□″ vectors
                    #         if do_density_weighting:
                    #             uL = np.copy( rho[xi,yii,zi,:] * data_L[xi,yii,zi,:] )
                    #             uR = np.copy( rho[xi,yii,zi,:] * data_R[xi,yii,zi,:] )
                    #         else:
                    #             uL = np.copy( data_L[xi,yii,zi,:] )
                    #             uR = np.copy( data_R[xi,yii,zi,:] )
                    #         
                    #         R_xz[xi,zi,:] = ccor( uL , uR )
                    
                    ## avg in [x,z] & write in rank context
                    Rt[scalar][yiii,:] = np.mean(R_xz, axis=(0,1), dtype=np.float64)
                
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print(msg, format_time_string(t_delta), s=True))
                if verbose: progress_bar.update() ## (scalar, [y] chunk) progress
        if verbose: progress_bar.close()
        self.comm.Barrier()
        if verbose: print(72*'-')
        
        # ==============================================================
        # write HDF5 (.h5) file
        # ==============================================================
        
        ## open on rank 0 and write attributes, dimensions, etc.
        if (self.rank==0):
            with h5py.File(fn_h5_ccor, 'w') as hfw:
                
                ## write floats,ints as top-level attributes
                for key,val in data.items():
                    if isinstance(data[key], (int,np.int32,np.int64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], (float,np.float32,np.float64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], np.ndarray):
                        pass
                    else:
                        print(f'key {key} is type {str(type(data[key]))}')
                        self.comm.Abort(1)
                
                ## write numpy arrays
                hfw.create_dataset( 'dims/x'      , data=x      ) ## [m]
                hfw.create_dataset( 'dims/y'      , data=y      ) ## [m]
                hfw.create_dataset( 'dims/z'      , data=z      ) ## [m]
                hfw.create_dataset( 'dims/t'      , data=t      ) ## [s]
                hfw.create_dataset( 'dims/lags_z' , data=lags_z ) ## [m]
                hfw.create_dataset( 'dims/lags_t' , data=lags_t ) ## [s]
        
        self.comm.Barrier()
        
        with h5py.File(fn_h5_ccor, 'a', driver='mpio', comm=self.comm) as hfw:
            
            ## initialize datasets
            for scalar in scalars:
                hfw.create_dataset( f'covariance/{scalar}'  , shape=(ny,)         , dtype=np.float64, chunks=(1,)         )
                hfw.create_dataset( f'Rz/{scalar}'          , shape=(ny,n_lags_z) , dtype=np.float64, chunks=(1,n_lags_z) )
                hfw.create_dataset( f'Rt/{scalar}'          , shape=(ny,n_lags_t) , dtype=np.float64, chunks=(1,n_lags_t) )
            
            ## initialize datasets 1D [y] mean
            for scalar in avg_Re.dtype.names:
                hfw.create_dataset( f'avg/Re/{scalar}', shape=(ny,), dtype=np.float64, chunks=(1,) )
            for scalar in avg_Fv.dtype.names:
                hfw.create_dataset( f'avg/Fv/{scalar}', shape=(ny,), dtype=np.float64, chunks=(1,) )
            
            self.comm.Barrier()
            
            ## collectively write covariance,Rz,Rt
            for scalar in scalars:
                dset = hfw[f'covariance/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = covariance[scalar][:]
                dset = hfw[f'Rz/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:] = Rz[scalar][:,:]
                dset = hfw[f'Rt/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:] = Rt[scalar][:,:]
            
            ## collectively write 1D [y] avgs (Reynolds,Favre)
            for scalar in avg_Re.dtype.names:
                dset = hfw[f'avg/Re/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = avg_Re[scalar][:]
            for scalar in avg_Fv.dtype.names:
                dset = hfw[f'avg/Fv/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = avg_Fv[scalar][:]
        
        self.comm.Barrier()
        if verbose:
            print('--w-> %s : %0.2f [MB]'%(fn_h5_ccor,os.path.getsize(fn_h5_ccor)/1024**2))
            print(72*'-')
        
        ## report file contents
        self.comm.Barrier()
        if (self.rank==0):
            with h5py.File(fn_h5_ccor,'r') as hfr:
                h5_print_contents(hfr)
        self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_ccor_xpln() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_turb_budget(self, **kwargs):
        '''
        calculate budget of turbulent kinetic energy (k)
        -----
        - parallelized over time [t] dimension
        - dimensional [SI]
        - designed for analyzing unsteady, thin planes in [x]
        - currently does NOT work in serial mode (parallel only)
        -----
        SI units of terms are [kg/(m·s³)] or [kg m^-1 s^-3]
        normalize with
        * ν / u**4 / ρ --> *[ kg^-1 m s^3]
        or
        / ( u**4 * ρ / ν ) --> /[kg m^-1 s^-3]
        -----
        Pirozzoli Grasso Gatski (2004)
        Direct numerical simulation and analysis of a spatially evolving supersonic turbulent boundary layer at M=2.25
        https://doi.org/10.1063/1.1637604
        -----
        Gaurini Moser Shariff Wray (2000)
        Direct numerical simulation of a supersonic turbulent boundary layer at Mach 2.5
        https://doi.org/10.1017/S0022112000008466
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_turb_budget()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if (self.fsubtype!='unsteady'):
            raise ValueError
        
        if ( not self.usingmpi ):
            raise ValueError('rgd.calc_turb_budget() currently only works in MPI mode')
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        ct = kwargs.get('ct',1)
        
        force = kwargs.get('force',False)
        
        fn_h5_mean        = kwargs.get('fn_h5_mean',None)
        fn_h5_prime       = kwargs.get('fn_h5_prime',None)
        fn_h5_turb_budget = kwargs.get('fn_h5_turb_budget',None)
        fn_dat_turb_budget = kwargs.get('fn_dat_turb_budget',None)
        
        save_unsteady = kwargs.get('save_unsteady',False)
        fn_h5_turb_budget_unsteady = kwargs.get('fn_h5_turb_budget_unsteady',None)
        
        acc          = kwargs.get('acc', 6)
        edge_stencil = kwargs.get('edge_stencil', 'full')
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',None) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',None)
        
        ## for now only distribute data in [t]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (ry!=1):
            raise AssertionError('ry!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        # ===
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,ry,rt], periods=[False,False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        #ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        #ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        rt1,rt2 = rtl[self.rank]; ntr = rt2 - rt1
        
        ## [t] sub chunk range --> ctl = list of ranges in rt1:rt2
        ctl_ = np.array_split( np.arange(rt1,rt2) , min(ct,ntr) )
        ctl = [[b[0],b[-1]+1] for b in ctl_ ]
        
        ## check that no sub ranges are <=1
        for a_ in [ ctl_[1]-ctl_[0] for ctl_ in ctl ]:
            if (a_ <= 1):
                #raise ValueError
                print(f'rank {self.rank:d} has [t] a chunk subrange <= 1')
                self.comm.Abort(1)
        
        ## the average sub [t] chunk size on this rank
        avg_chunk_nt = np.mean( [ ctl_[1]-ctl_[0] for ctl_ in ctl ] )
        
        if 1: ## check that [t] sub-chunk ranges are correct
            
            mytimeindices = []
            for ctl_ in ctl:
                ct1, ct2 = ctl_
                mytimeindices += [ ti_ for ti_ in self.ti[ct1:ct2] ]
            
            G = self.comm.gather([ self.rank , mytimeindices ], root=0)
            G = self.comm.bcast(G, root=0)
            
            alltimeindices = []
            for G_ in G:
                alltimeindices += G_[1]
            alltimeindices = np.array( sorted(alltimeindices), dtype=np.int64 )
            
            if not np.array_equal( alltimeindices , self.ti ):
                raise AssertionError
            if not np.array_equal( alltimeindices , np.arange(self.nt, dtype=np.int64) ):
                raise AssertionError
        
        self.comm.Barrier()
        
        ## mean file name (for reading)
        if (fn_h5_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean.h5'
            #fn_h5_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_h5_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_h5_mean = Path(fname_path, fname_mean_h5_base)
        
        if not os.path.isfile(fn_h5_mean):
            raise FileNotFoundError('%s not found!'%fn_h5_mean)
        
        ## turb_budget .h5 file name (for writing) --> AVERAGED
        if (fn_h5_turb_budget is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fn_h5_turb_budget_base = fname_root+'_turb_budget.h5'
            fn_h5_turb_budget = str(PurePosixPath(fname_path, fn_h5_turb_budget_base))
        
        if os.path.isfile(fn_h5_turb_budget) and (force is False):
            raise ValueError(f'{fn_h5_turb_budget} already present & force=False')
        
        ## turb_budget .h5 file name (for writing) --> UNSTEADY
        if save_unsteady:
            if (fn_h5_turb_budget_unsteady is None):
                fname_path = os.path.dirname(self.fname)
                fname_base = os.path.basename(self.fname)
                fname_root, fname_ext = os.path.splitext(fname_base)
                fn_h5_turb_budget_unsteady_base = fname_root+'_turb_budget_unsteady.h5'
                fn_h5_turb_budget_unsteady = str(PurePosixPath(fname_path, fn_h5_turb_budget_unsteady_base))
            
            if os.path.isfile(fn_h5_turb_budget_unsteady) and (force is False):
                raise ValueError(f'{fn_h5_turb_budget_unsteady} already present & force=False')
        
        ## turb_budget .dat file name (for writing)
        if (fn_dat_turb_budget is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fn_dat_turb_budget_base = fname_root+'_turb_budget.dat'
            fn_dat_turb_budget = str(PurePosixPath(fname_path, fn_dat_turb_budget_base))
        
        if verbose: even_print('fn_h5'              , self.fname         )
        if verbose: even_print('fn_h5_mean'         , fn_h5_mean         )
        if verbose: even_print('fn_h5_turb_budget'  , fn_h5_turb_budget  )
        if verbose: even_print('fn_dat_turb_budget' , fn_dat_turb_budget )
        if verbose: print(72*'-')
        if verbose: even_print('save unsteady' , str(save_unsteady) )
        if verbose and save_unsteady:
            even_print('unsteady file' , fn_h5_turb_budget_unsteady )
        if verbose: print(72*'-')
        
        # ===
        
        if verbose: even_print('nx' , '%i'%self.nx)
        if verbose: even_print('ny' , '%i'%self.ny)
        if verbose: even_print('nz' , '%i'%self.nz)
        if verbose: even_print('nt' , '%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('rt' , '%i'%rt)
        if verbose: even_print('ct' , '%i'%ct)
        if verbose: even_print('ntr' , '%i'%ntr)
        if verbose: even_print('avg [t] chunk nt' , '%0.2f'%avg_chunk_nt)
        if verbose: print(72*'-')
        
        # === init outfiles
        
        #if verbose: print(72*'*')
        if verbose: print(fn_h5_turb_budget)
        if verbose: print(len(fn_h5_turb_budget)*'-')
        
        ## initialize file: turbulent kinetic energy budget, AVERAGED
        with rgd(fn_h5_turb_budget, 'w', force=force, driver='mpio', comm=self.comm) as f1:
            
            f1.init_from_rgd(self.fname, t_info=False)
            
            ## set some top-level attributes
            #f1.attrs['duration_avg'] = duration_avg ## duration of mean
            f1.attrs['duration_avg'] = self.t[-1] - self.t[0]
            #f1_mean.attrs['duration_avg'] = self.duration
            f1.attrs['dt'] = self.t[1] - self.t[0]
            #f1.attrs['fclass'] = 'rgd'
            f1.attrs['fsubtype'] = 'mean'
            
            shape = (1,f1.nz,f1.ny,f1.nx)
            chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=4)
            
            data_gb = 4*1*f1.nz*f1.ny*f1.nx / 1024**3
            
            for dss in ['production','dissipation','transport','diffusion','p_dilatation','p_diffusion']:
                
                if verbose:
                    even_print('initializing data/%s'%(dss,),'%0.1f [GB]'%(data_gb,))
                
                dset = f1.create_dataset(
                            'data/%s'%dss, 
                            shape=shape, 
                            dtype=np.float32,
                            chunks=chunks,
                            )
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            # === replace dims/t array --> take last time of series
            if ('dims/t' in f1):
                del f1['dims/t']
            f1.create_dataset('dims/t', data=np.array([self.t[-1]],dtype=np.float64))
            
            if hasattr(f1, 'duration_avg'):
                if verbose: even_print('duration_avg', '%0.2f'%f1.duration_avg)
        
        if verbose: print(72*'-')
        
        ## initialize file: turbulent kinetic energy budget, UNSTEADY
        if save_unsteady:
            
            if verbose: print(fn_h5_turb_budget_unsteady)
            if verbose: print(len(fn_h5_turb_budget_unsteady)*'-')
            
            with rgd(fn_h5_turb_budget_unsteady, 'w', force=force, driver='mpio', comm=self.comm) as f1:
                
                f1.init_from_rgd(self.fname, t_info=True)
                
                ## set some top-level attributes
                #f1.attrs['duration_avg'] = duration_avg ## duration of mean
                f1.attrs['duration_avg'] = self.t[-1] - self.t[0]
                #f1_mean.attrs['duration_avg'] = self.duration
                f1.attrs['dt'] = self.t[1] - self.t[0]
                #f1.attrs['fclass'] = 'rgd'
                f1.attrs['fsubtype'] = 'unsteady'
                
                shape = (self.nt,f1.nz,f1.ny,f1.nx)
                #chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=4)
                chunks = self['data/u'].chunks ## use chunk pattern of this file
                
                data_gb = 4*self.nt*f1.nz*f1.ny*f1.nx / 1024**3
                
                for dss in ['production','dissipation','transport','diffusion']: #,'p_dilatation','p_diffusion']:
                    
                    if verbose:
                        even_print('initializing data/%s'%(dss,),'%0.1f [GB]'%(data_gb,))
                    
                    dset = f1.create_dataset(
                                'data/%s'%dss, 
                                shape=shape, 
                                dtype=np.float32,
                                chunks=chunks,
                                )
                    
                    chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                    if verbose:
                        even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                        even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if verbose: print(72*'-')
        
        self.comm.Barrier()
        
        # ===
        
        ## OPEN the mean file... remember to close later!
        hf_mean = rgd(fn_h5_mean, 'r', driver='mpio', comm=self.comm)
        
        ## verify contents of the mean file
        np.testing.assert_allclose( hf_mean.x       , self.x        , atol=1e-12, rtol=1e-12 )
        np.testing.assert_allclose( hf_mean.y       , self.y        , atol=1e-12, rtol=1e-12 )
        np.testing.assert_allclose( hf_mean.z       , self.z        , atol=1e-12, rtol=1e-12 )
        np.testing.assert_allclose( hf_mean.lchar   , self.lchar    , atol=1e-12, rtol=1e-12 )
        np.testing.assert_allclose( hf_mean.U_inf   , self.U_inf    , atol=1e-12, rtol=1e-12 )
        np.testing.assert_allclose( hf_mean.T_inf   , self.T_inf    , atol=1e-12, rtol=1e-12 )
        np.testing.assert_allclose( hf_mean.rho_inf , self.rho_inf  , atol=1e-12, rtol=1e-12 )
        if (hf_mean.fsubtype!='mean'):
            raise ValueError
        
        # hf_prime = rgd(fn_h5_prime , 'r', driver='mpio', comm=self.comm) ## TODO
        # if (hf_prime.fsubtype!='prime'):
        #     raise ValueError
        
        # ===
        
        data = {} ## dict to be output to .dat file
        
        if ('data_dim' not in hf_mean):
            raise ValueError('group data_dim not present')
        
        ## put all data from 'data_dim' into the dictionary data which will be pickled at the end
        for dsn in hf_mean['data_dim'].keys():
            d_ = np.copy( hf_mean[f'data_dim/{dsn}'][()] )
            if (d_.ndim == 0):
                d_ = float(d_)
            data[dsn] = d_
        
        ## 1D
        #rho_avg = np.copy( hf_mean['data_dim/rho'][()] )
        #u_avg   = np.copy( hf_mean['data_dim/u'][()]   )
        #v_avg   = np.copy( hf_mean['data_dim/v'][()]   )
        #w_avg   = np.copy( hf_mean['data_dim/w'][()]   )
        #T_avg   = np.copy( hf_mean['data_dim/T'][()]   )
        #p_avg   = np.copy( hf_mean['data_dim/p'][()]   )
        
        ## 0D
        u_tau    = float( hf_mean['data_dim/u_tau'][()]    )
        nu_wall  = float( hf_mean['data_dim/nu_wall'][()]  )
        rho_wall = float( hf_mean['data_dim/rho_wall'][()] )
        T_wall   = float( hf_mean['data_dim/T_wall'][()]   )
        d99      = float( hf_mean['data_dim/d99'][()]      )
        u_99     = float( hf_mean['data_dim/u_99'][()]     )
        Re_tau   = float( hf_mean['data_dim/Re_tau'][()]   )
        Re_theta = float( hf_mean['data_dim/Re_theta'][()] )
        sc_u_in  = float( hf_mean['data_dim/sc_u_in'][()]  )
        sc_l_in  = float( hf_mean['data_dim/sc_l_in'][()]  )
        sc_t_in  = float( hf_mean['data_dim/sc_t_in'][()]  )
        sc_u_out = float( hf_mean['data_dim/sc_u_out'][()] )
        sc_l_out = float( hf_mean['data_dim/sc_l_out'][()] )
        sc_t_out = float( hf_mean['data_dim/sc_t_out'][()] )
        
        ## 0D scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays, then re-dimensionalize [m]
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-14, atol=1e-14)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-14, atol=1e-14)
        
        t_eddy = t_meas / ( d99 / u_tau )
        
        ## check if constant Δz (calculate Δz+ later)
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-7)):
            raise NotImplementedError
        np.testing.assert_allclose(dz0, z[1]-z[0], rtol=1e-14, atol=1e-14)
        
        zrange = z.max() - z.min()
        np.testing.assert_allclose(zrange, z[-1]-z[0], rtol=1e-14, atol=1e-14)
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        ## add data to dict that gets output
        data['x'] = x
        data['y'] = y
        data['z'] = z
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz0'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print('Δt/tchar','%0.8f'%(dt/self.tchar))
        if verbose: even_print('Δt','%0.3e [s]'%(dt,))
        if verbose: even_print('duration/tchar','%0.1f'%(self.duration,))
        if verbose: even_print('duration','%0.3e [s]'%(self.duration*self.tchar,))
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('dt'     , '%0.5e [s]' % dt      )
            even_print('t_meas' , '%0.5e [s]' % t_meas  )
            even_print('dz0'    , '%0.5e [m]' % dz0     )
            even_print('zrange' , '%0.5e [m]' % zrange  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('Re_τ'    , '%0.1f'        % Re_tau    )
            even_print('Re_θ'    , '%0.1f'        % Re_theta  )
            even_print('δ99'     , '%0.5e [m]'    % d99       )
            even_print('δ_ν=(ν_wall/u_τ)' , '%0.5e [m]' % sc_l_in )
            even_print('U_inf'  , '%0.3f [m/s]'   % self.U_inf )
            even_print('u_τ'    , '%0.3f [m/s]'   % u_tau     )
            even_print('ν_wall' , '%0.5e [m²/s]'  % nu_wall   )
            even_print('ρ_wall' , '%0.6f [kg/m³]' % rho_wall  )
            ##
            even_print( 'Δz+'        , '%0.3f'%(dz0/sc_l_in) )
            even_print( 'zrange/δ99' , '%0.3f'%(zrange/d99)  )
            even_print( 'Δt+'        , '%0.3f'%(dt/sc_t_in)  )
            print(72*'-')
        
        ## report
        if verbose:
            even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy)
            even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99/u_99)))
            even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99/u_99)))
            print(72*'-')
        
        # ===
        
        ## copy AVG [u,v,w] into memory... DIMENSIONAL
        u_re = np.copy( U_inf * hf_mean['data/u'][0,:,:,:].T    )
        v_re = np.copy( U_inf * hf_mean['data/v'][0,:,:,:].T    )
        w_re = np.copy( U_inf * hf_mean['data/w'][0,:,:,:].T    )
        u_fv = np.copy( U_inf * hf_mean['data/u_fv'][0,:,:,:].T )
        v_fv = np.copy( U_inf * hf_mean['data/v_fv'][0,:,:,:].T )
        w_fv = np.copy( U_inf * hf_mean['data/w_fv'][0,:,:,:].T )
        
        p_re = np.copy( rho_inf * U_inf**2 * hf_mean['data/p'][0,:,:,:].T )
        
        ## get Reynolds avg strain tensor elements
        dudx_re = gradient( u=u_re , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dudy_re = gradient( u=u_re , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dudz_re = gradient( u=u_re , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dvdx_re = gradient( u=v_re , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dvdy_re = gradient( u=v_re , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dvdz_re = gradient( u=v_re , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dwdx_re = gradient( u=w_re , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dwdy_re = gradient( u=w_re , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dwdz_re = gradient( u=w_re , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        
        ## get Favre avg strain tensor elements
        dudx_fv = gradient( u=u_fv , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dudy_fv = gradient( u=u_fv , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dudz_fv = gradient( u=u_fv , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dvdx_fv = gradient( u=v_fv , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dvdy_fv = gradient( u=v_fv , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dvdz_fv = gradient( u=v_fv , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dwdx_fv = gradient( u=w_fv , x=x , axis=0 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dwdy_fv = gradient( u=w_fv , x=y , axis=1 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        dwdz_fv = gradient( u=w_fv , x=z , axis=2 , acc=acc , edge_stencil=edge_stencil )[:,:,:,np.newaxis]
        
        ## accumulators for per-timestep sum, at end gets multiplied by (1/nt) to get average
        unsteady_production_sum   = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
        unsteady_dissipation_sum  = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
        unsteady_transport_sum    = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
        unsteady_diffusion_sum    = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
        unsteady_p_diffusion_sum  = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
        unsteady_p_dilatation_sum = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
        
        # === main loop (iterate through sub-ranges within rank-distributed [t] range)
        
        if verbose:
            progress_bar = tqdm(total=ct, ncols=100, desc='turb budget', leave=False, file=sys.stdout, smoothing=0.)
        
        ct_counter=0
        for ctl_ in ctl:
            ct_counter += 1
            ct1, ct2 = ctl_
            ntc = ct2 - ct1
            
            if (ct>1):
                if verbose:
                    mesg = f'[t] sub chunk {ct_counter:d}/{ct:d}'
                    tqdm.write( mesg )
                    tqdm.write( '-'*len(mesg) )
            
            # === read unsteady data
            
            ss1 = ['u','v','w','T','rho','p']
            ss2 = ['uI','vI','wI', 'uII','vII','wII', 'pI'] ## 'TI','rhoI'
            
            formats = [ np.float32 for s in ss1+ss2 ]
            #shape = (nx,ny,nz,ntr)
            shape = (nx,ny,nz,ntc)
            dd = np.zeros(shape=shape, dtype={'names':ss1+ss2, 'formats':formats}, order='C')
            
            for ss in ss1:
                dset = self['data/%s'%ss]
                self.comm.Barrier()
                t_start = timeit.default_timer()
                with dset.collective:
                    #dd[ss] = dset[rt1:rt2,:,:,:].T
                    dd[ss] = dset[ct1:ct2,:,:,:].T
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                #data_gb = 4*nx*ny*nz*nt/1024**3
                data_gb = self.n_ranks * 4*nx*ny*nz*ntc/1024**3 ## approximate!
                if verbose:
                    tqdm.write( even_print('read: %s'%ss, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True) )
            
            # for ss in ss2:
            #     dset = hf_prime['data/%s'%ss]
            #     self.comm.Barrier()
            #     t_start = timeit.default_timer()
            #     with dset.collective:
            #         #dd[ss] = dset[rt1:rt2,:,:,:].T
            #         dd[ss] = dset[ct1:ct2,:,:,:].T
            #     self.comm.Barrier()
            #     t_delta = timeit.default_timer() - t_start
            #     #data_gb = 4*nx*ny*nz*nt/1024**3
            #     data_gb = self.n_ranks * 4*nx*ny*nz*ntc/1024**3 ## approximate!
            #     if verbose:
            #         tqdm.write( even_print('read: %s'%ss, '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True) )
            
            ## DIMLESS
            dd['uI'][:,:,:,:]  = np.copy( dd['u'] - (u_re[:,:,:,np.newaxis]/U_inf) )
            dd['vI'][:,:,:,:]  = np.copy( dd['v'] - (v_re[:,:,:,np.newaxis]/U_inf) )
            dd['wI'][:,:,:,:]  = np.copy( dd['w'] - (w_re[:,:,:,np.newaxis]/U_inf) )
            dd['uII'][:,:,:,:] = np.copy( dd['u'] - (u_fv[:,:,:,np.newaxis]/U_inf) )
            dd['vII'][:,:,:,:] = np.copy( dd['v'] - (v_fv[:,:,:,np.newaxis]/U_inf) )
            dd['wII'][:,:,:,:] = np.copy( dd['w'] - (w_fv[:,:,:,np.newaxis]/U_inf) )
            
            dd['pI'][:,:,:,:]  = np.copy( dd['p'] - (p_re[:,:,:,np.newaxis]/(rho_inf*U_inf**2)) )
            
            if verbose: tqdm.write(72*'-')
            
            # ===
            
            data_gb = dd.nbytes / 1024**3
            if verbose:
                tqdm.write(even_print('data_gb (rank)', '%0.1f [GB]'%data_gb, s=True))
            
            mem_avail_gb = psutil.virtual_memory().available/1024**3
            mem_free_gb  = psutil.virtual_memory().free/1024**3
            if verbose:
                tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
            
            # === redimensionalize the unsteady data
            
            u    = np.copy( dd['u']   * U_inf )
            v    = np.copy( dd['v']   * U_inf )
            w    = np.copy( dd['w']   * U_inf )
            uI   = np.copy( dd['uI']  * U_inf )
            vI   = np.copy( dd['vI']  * U_inf )
            wI   = np.copy( dd['wI']  * U_inf )
            uII  = np.copy( dd['uII'] * U_inf )
            vII  = np.copy( dd['vII'] * U_inf )
            wII  = np.copy( dd['wII'] * U_inf )
            
            rho  = np.copy( dd['rho']  * rho_inf              )
            #rhoI = np.copy( dd['rhoI'] * rho_inf              )
            #p    = np.copy( dd['p']    * (rho_inf * U_inf**2) )
            pI   = np.copy( dd['pI']   * (rho_inf * U_inf**2) )
            T    = np.copy( dd['T']    * T_inf                )
            #TI   = np.copy( dd['TI']   * T_inf                )
            
            mu = self.C_Suth * T**(3/2) / (T + self.S_Suth)
            nu = mu / rho
            
            dd = None ; del dd
            
            # === get gradients
            
            t_start = timeit.default_timer()
            
            dudx   = gradient(u, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            dudy   = gradient(u, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            dudz   = gradient(u, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            dvdx   = gradient(v, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            dvdy   = gradient(v, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            dvdz   = gradient(v, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            dwdx   = gradient(w, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            dwdy   = gradient(w, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            dwdz   = gradient(w, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            
            duIdx  = gradient(uI, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            duIdy  = gradient(uI, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            duIdz  = gradient(uI, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            dvIdx  = gradient(vI, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            dvIdy  = gradient(vI, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            dvIdz  = gradient(vI, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            dwIdx  = gradient(wI, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            dwIdy  = gradient(wI, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            dwIdz  = gradient(wI, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            
            duIIdx = gradient(uII, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            duIIdy = gradient(uII, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            duIIdz = gradient(uII, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            dvIIdx = gradient(vII, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            dvIIdy = gradient(vII, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            dvIIdz = gradient(vII, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            dwIIdx = gradient(wII, x, acc=acc, edge_stencil=edge_stencil, axis=0)
            dwIIdy = gradient(wII, y, acc=acc, edge_stencil=edge_stencil, axis=1)
            dwIIdz = gradient(wII, z, acc=acc, edge_stencil=edge_stencil, axis=2)
            
            t_delta = timeit.default_timer() - t_start
            
            if verbose:
                tqdm.write(even_print('get gradients', format_time_string(t_delta), s=True))
            
            self.comm.Barrier()
            mem_avail_gb = psutil.virtual_memory().available/1024**3
            mem_free_gb  = psutil.virtual_memory().free/1024**3
            if verbose:
                tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
            
            # === stack velocities into vectors, fluctuating strains into (Re variance) tensor
            
            t_start = timeit.default_timer()
            
            uI_i  = np.copy(np.stack((uI,  vI,  wI ), axis=-1))
            uII_i = np.copy(np.stack((uII, vII, wII), axis=-1))
            
            duIdx_ij = np.copy( np.stack((np.stack((duIdx, duIdy, duIdz), axis=4),
                                          np.stack((dvIdx, dvIdy, dvIdz), axis=4),
                                          np.stack((dwIdx, dwIdy, dwIdz), axis=4)), axis=5) )
            
            duIIdx_ij = np.copy( np.stack((np.stack((duIIdx, duIIdy, duIIdz), axis=4),
                                           np.stack((dvIIdx, dvIIdy, dvIIdz), axis=4),
                                           np.stack((dwIIdx, dwIIdy, dwIIdz), axis=4)), axis=5) )
            
            t_delta = timeit.default_timer() - t_start
            if verbose:
                tqdm.write(even_print('tensor stacking',format_time_string(t_delta), s=True))
            
            self.comm.Barrier()
            mem_avail_gb = psutil.virtual_memory().available/1024**3
            mem_free_gb  = psutil.virtual_memory().free/1024**3
            if verbose:
                tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
            
            # === production : P
            if True:
                
                if verbose: tqdm.write(72*'-')
                t_start = timeit.default_timer()
                
                r_uII_uII = rho*uII*uII
                r_uII_vII = rho*uII*vII
                r_uII_wII = rho*uII*wII
                
                r_vII_uII = rho*vII*uII
                r_vII_vII = rho*vII*vII
                r_vII_wII = rho*vII*wII
                
                r_wII_uII = rho*wII*uII
                r_wII_vII = rho*wII*vII
                r_wII_wII = rho*wII*wII
                
                ## unsteady_production_ = - ( r_uII_uII * dudx_fv + r_uII_vII * dudy_fv + r_uII_wII * dudz_fv \
                ##                          + r_uII_vII * dvdx_fv + r_vII_vII * dvdy_fv + r_vII_wII * dvdz_fv \
                ##                          + r_uII_wII * dwdx_fv + r_vII_wII * dwdy_fv + r_wII_wII * dwdz_fv )
                
                r_uIIuII_ij = np.stack((np.stack((r_uII_uII, r_uII_vII, r_uII_wII), axis=4),
                                        np.stack((r_vII_uII, r_vII_vII, r_vII_wII), axis=4),
                                        np.stack((r_wII_uII, r_wII_vII, r_wII_wII), axis=4)), axis=5)
                
                dudx_fv_ij = np.stack((np.stack((dudx_fv, dudy_fv, dudz_fv), axis=4),
                                       np.stack((dvdx_fv, dvdy_fv, dvdz_fv), axis=4),
                                       np.stack((dwdx_fv, dwdy_fv, dwdz_fv), axis=4)), axis=5)
                
                unsteady_production = -1*np.einsum('xyztij,xyztij->xyzt', r_uIIuII_ij, dudx_fv_ij)
                
                ## np.testing.assert_allclose(unsteady_production, unsteady_production_, atol=20000)
                ## print('check passed : np.einsum()')
                
                t_delta = timeit.default_timer() - t_start
                if verbose:
                    tqdm.write(even_print('calc production', format_time_string(t_delta), s=True))
                
                self.comm.Barrier()
                
                if save_unsteady: ## write 4D unsteady production
                    
                    ## technically this is an estimate
                    data_gb = self.n_ranks * np.prod(unsteady_production.shape) * unsteady_production.dtype.itemsize / 1024**3
                    
                    with rgd(fn_h5_turb_budget_unsteady, 'a', driver='mpio', comm=self.comm) as f1:
                        dset = f1['data/production']
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            dset[ct1:ct2,:,:,:] = unsteady_production.T
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('write: production', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## combine sums across ranks for this [t] chunk
                unsteady_production_sum_i   = np.sum(unsteady_production, axis=3, keepdims=True, dtype=np.float64)
                unsteady_production_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                self.comm.Reduce([unsteady_production_sum_i, MPI.DOUBLE],
                                 [unsteady_production_sum_buf, MPI.DOUBLE],
                                 op=MPI.SUM,
                                 root=0)
                
                ## add aggregated sum (across ranks) to the total accumulator
                unsteady_production_sum += unsteady_production_sum_buf
                
                # === release mem
                r_uII_uII           = None; del r_uII_uII
                r_uII_vII           = None; del r_uII_vII
                r_uII_wII           = None; del r_uII_wII
                r_vII_uII           = None; del r_vII_uII
                r_vII_vII           = None; del r_vII_vII
                r_vII_wII           = None; del r_vII_wII
                r_wII_uII           = None; del r_wII_uII
                r_wII_vII           = None; del r_wII_vII
                r_wII_wII           = None; del r_wII_wII
                r_uIIuII_ij         = None; del r_uIIuII_ij
                dudx_fv_ij          = None; del dudx_fv_ij
                unsteady_production = None; del unsteady_production
                
                self.comm.Barrier()
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                    tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
            
            # === dissipation : ϕ,ε
            if True:
                
                if verbose: tqdm.write(72*'-')
                t_start = timeit.default_timer()
                
                duIIdx_ij_duIdx_ij = np.einsum('xyztij,xyztij->xyzt', duIIdx_ij, duIdx_ij)
                duIIdx_ij_duIdx_ji = np.einsum('xyztij,xyztji->xyzt', duIIdx_ij, duIdx_ij)
                
                ## # === from pp_turbulent_budget.F90
                ## unsteady_dissipation_ = mu * ( (duIdx + duIdx) * duIIdx  +  (duIdy + dvIdx) * duIIdy  +  (duIdz + dwIdx) * duIIdz \
                ##                              + (dvIdx + duIdy) * dvIIdx  +  (dvIdy + dvIdy) * dvIIdy  +  (dvIdz + dwIdy) * dvIIdz \
                ##                              + (dwIdx + duIdz) * dwIIdx  +  (dwIdy + dvIdz) * dwIIdy  +  (dwIdz + dwIdz) * dwIIdz )
                
                unsteady_dissipation = mu*(duIIdx_ij_duIdx_ij + duIIdx_ij_duIdx_ji)
                
                ## np.testing.assert_allclose(unsteady_dissipation, unsteady_dissipation_, rtol=1e-4)
                ## print('check passed : np.einsum() : dissipation')
                
                t_delta = timeit.default_timer() - t_start
                if verbose:
                    tqdm.write(even_print('calc dissipation', format_time_string(t_delta),s=True))
                
                self.comm.Barrier()
                
                if save_unsteady: ## write 4D unsteady dissipation
                    
                    ## technically this is an estimate
                    data_gb = self.n_ranks * np.prod(unsteady_dissipation.shape) * unsteady_dissipation.dtype.itemsize / 1024**3
                    
                    with rgd(fn_h5_turb_budget_unsteady, 'a', driver='mpio', comm=self.comm) as f1:
                        dset = f1['data/dissipation']
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            dset[ct1:ct2,:,:,:] = unsteady_dissipation.T
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('write: dissipation', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## combine sums across ranks for this [t] chunk
                unsteady_dissipation_sum_i   = np.sum(unsteady_dissipation, axis=3, keepdims=True, dtype=np.float64)
                unsteady_dissipation_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                self.comm.Reduce([unsteady_dissipation_sum_i, MPI.DOUBLE],
                                 [unsteady_dissipation_sum_buf, MPI.DOUBLE],
                                 op=MPI.SUM,
                                 root=0)
                
                ## add aggregated sum (across ranks) to the total accumulator
                unsteady_dissipation_sum += unsteady_dissipation_sum_buf
                
                ## release mem
                duIIdx_ij_duIdx_ij   = None; del duIIdx_ij_duIdx_ij
                duIIdx_ij_duIdx_ji   = None; del duIIdx_ij_duIdx_ji
                unsteady_dissipation = None; del unsteady_dissipation
                
                self.comm.Barrier()
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                    tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
            
            # === transport : T
            if True:
                
                if verbose: tqdm.write(72*'-')
                t_start = timeit.default_timer()
                
                ## triple correlation
                tc = np.einsum('xyzt,xyzti,xyzti,xyztj->xyztj', rho, uII_i, uII_i, uII_i)
                
                #tc_ddx = np.gradient(tc, x, axis=0, edge_order=2)
                #tc_ddy = np.gradient(tc, y, axis=1, edge_order=2)
                #tc_ddz = np.gradient(tc, z, axis=2, edge_order=2)
                
                tc_ddx = gradient(tc, x, axis=0, acc=acc, edge_stencil=edge_stencil)
                tc_ddy = gradient(tc, y, axis=1, acc=acc, edge_stencil=edge_stencil)
                tc_ddz = gradient(tc, z, axis=2, acc=acc, edge_stencil=edge_stencil)
                
                unsteady_transport = -0.5*(tc_ddx[:,:,:,:,0] + tc_ddy[:,:,:,:,1] + tc_ddz[:,:,:,:,2])
                
                t_delta = timeit.default_timer() - t_start
                if verbose:
                    tqdm.write(even_print('calc transport', format_time_string(t_delta),s=True))
                
                self.comm.Barrier()
                
                if save_unsteady: ## write 4D unsteady transport
                    
                    ## technically this is an estimate
                    data_gb = self.n_ranks * np.prod(unsteady_transport.shape) * unsteady_transport.dtype.itemsize / 1024**3
                    
                    with rgd(fn_h5_turb_budget_unsteady, 'a', driver='mpio', comm=self.comm) as f1:
                        dset = f1['data/transport']
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            dset[ct1:ct2,:,:,:] = unsteady_transport.T
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('write: transport', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## combine sums across ranks for this [t] chunk
                unsteady_transport_sum_i   = np.sum(unsteady_transport, axis=3, keepdims=True, dtype=np.float64)
                unsteady_transport_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                self.comm.Reduce([unsteady_transport_sum_i, MPI.DOUBLE],
                                 [unsteady_transport_sum_buf, MPI.DOUBLE],
                                 op=MPI.SUM,
                                 root=0)
                
                ## add aggregated sum (across ranks) to the total accumulator
                unsteady_transport_sum += unsteady_transport_sum_buf
                
                ## release mem
                tc     = None; del tc
                tc_ddx = None; del tc_ddx
                tc_ddy = None; del tc_ddy
                tc_ddz = None; del tc_ddz
                unsteady_transport = None; del unsteady_transport
                
                self.comm.Barrier()
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                    tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
            
            # === viscous diffusion : D
            if True:
                
                if verbose: tqdm.write(72*'-')
                t_start = timeit.default_timer()
                
                omega_ij = duIdx_ij + np.transpose(duIdx_ij, axes=(0,1,2,3,5,4))
                
                if False:
                    
                    omega_ij_2 = np.stack((np.stack(((duIdx+duIdx), (dvIdx+duIdy), (dwIdx+duIdz)), axis=4),
                                           np.stack(((duIdy+dvIdx), (dvIdy+dvIdy), (dwIdy+dvIdz)), axis=4),
                                           np.stack(((duIdz+dwIdx), (dvIdz+dwIdy), (dwIdz+dwIdz)), axis=4)), axis=5)
                    
                    np.testing.assert_allclose(omega_ij, omega_ij_2, rtol=1e-8)
                    
                    if verbose:
                        print('check passed : omega_ij')
                
                ## ## this one is likely wrong
                ## omega_ij = np.stack((np.stack(((duIdx+duIdx), (dvIdx+duIdy), (dwIdx+duIdz)), axis=4),
                ##                      np.stack(((duIdy+duIdx), (dvIdy+duIdy), (dwIdy+duIdz)), axis=4),
                ##                      np.stack(((duIdz+duIdx), (dvIdz+duIdy), (dwIdz+duIdz)), axis=4)), axis=5)
                
                A = np.einsum('xyzt,xyzti,xyztij->xyztj', mu, uI_i, omega_ij)
                
                # A_ddx = np.gradient(A[:,:,:,:,0], x, axis=0, edge_order=2)
                # A_ddy = np.gradient(A[:,:,:,:,1], y, axis=1, edge_order=2)
                # A_ddz = np.gradient(A[:,:,:,:,2], z, axis=2, edge_order=2)
                
                A_ddx = gradient(A[:,:,:,:,0], x, axis=0, acc=acc, edge_stencil=edge_stencil)
                A_ddy = gradient(A[:,:,:,:,1], y, axis=1, acc=acc, edge_stencil=edge_stencil)
                A_ddz = gradient(A[:,:,:,:,2], z, axis=2, acc=acc, edge_stencil=edge_stencil)
                
                unsteady_diffusion = A_ddx + A_ddy + A_ddz
                
                t_delta = timeit.default_timer() - t_start
                if verbose:
                    tqdm.write(even_print('calc diffusion', format_time_string(t_delta),s=True))
                
                self.comm.Barrier()
                
                if save_unsteady: ## write 4D unsteady diffusion
                    
                    ## technically this is an estimate
                    data_gb = self.n_ranks * np.prod(unsteady_diffusion.shape) * unsteady_diffusion.dtype.itemsize / 1024**3
                    
                    with rgd(fn_h5_turb_budget_unsteady, 'a', driver='mpio', comm=self.comm) as f1:
                        dset = f1['data/diffusion']
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            dset[ct1:ct2,:,:,:] = unsteady_diffusion.T
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        if verbose:
                            tqdm.write(even_print('write: diffusion', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## combine sums across ranks for this [t] chunk
                unsteady_diffusion_sum_i   = np.sum(unsteady_diffusion, axis=3, keepdims=True, dtype=np.float64)
                unsteady_diffusion_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                self.comm.Reduce([unsteady_diffusion_sum_i, MPI.DOUBLE],
                                 [unsteady_diffusion_sum_buf, MPI.DOUBLE],
                                 op=MPI.SUM,
                                 root=0)
                
                ## add aggregated sum (across ranks) to the total accumulator
                unsteady_diffusion_sum += unsteady_diffusion_sum_buf
                
                ## release mem
                omega_ij = None; del omega_ij
                A        = None; del A
                A_ddx    = None; del A_ddx
                A_ddy    = None; del A_ddy
                A_ddz    = None; del A_ddz
                unsteady_diffusion = None; del unsteady_diffusion
                
                self.comm.Barrier()
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                    tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
            
            # === pressure diffusion
            if True:
                
                if verbose: tqdm.write(72*'-')
                t_start = timeit.default_timer()
                
                A = np.einsum('xyzti,xyzt->xyzti', uII_i, pI)
                
                #A_ddx = np.gradient(A[:,:,:,:,0], x, axis=0, edge_order=2)
                #A_ddy = np.gradient(A[:,:,:,:,1], y, axis=1, edge_order=2)
                #A_ddz = np.gradient(A[:,:,:,:,2], z, axis=2, edge_order=2)
                
                A_ddx = gradient(A[:,:,:,:,0], x, axis=0, acc=acc, edge_stencil=edge_stencil)
                A_ddy = gradient(A[:,:,:,:,1], y, axis=1, acc=acc, edge_stencil=edge_stencil)
                A_ddz = gradient(A[:,:,:,:,2], z, axis=2, acc=acc, edge_stencil=edge_stencil)
                
                unsteady_p_diffusion = A_ddx + A_ddy + A_ddz
                
                t_delta = timeit.default_timer() - t_start
                if verbose:
                    tqdm.write(even_print('calc p_diffusion', format_time_string(t_delta),s=True))
                
                self.comm.Barrier()
                
                ## combine sums across ranks for this [t] chunk
                unsteady_p_diffusion_sum_i   = np.sum(unsteady_p_diffusion, axis=3, keepdims=True, dtype=np.float64)
                unsteady_p_diffusion_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                self.comm.Reduce([unsteady_p_diffusion_sum_i, MPI.DOUBLE],
                                 [unsteady_p_diffusion_sum_buf, MPI.DOUBLE],
                                 op=MPI.SUM,
                                 root=0)
                
                ## add aggregated sum (across ranks) to the total accumulator
                unsteady_p_diffusion_sum += unsteady_p_diffusion_sum_buf
                
                # === release mem
                A        = None; del A
                A_ddx    = None; del A_ddx
                A_ddy    = None; del A_ddy
                A_ddz    = None; del A_ddz
                unsteady_p_diffusion = None; del unsteady_p_diffusion
                
                self.comm.Barrier()
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                    tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
            
            # === pressure dilatation
            if True:
                
                if verbose: tqdm.write(72*'-')
                t_start = timeit.default_timer()
                
                # A = np.einsum('xyzt,xyzti->xyzti', pI, uII_i)
                # A_ddx = np.gradient(A[:,:,:,:,0], x, axis=0, edge_order=2)
                # A_ddy = np.gradient(A[:,:,:,:,1], y, axis=1, edge_order=2)
                # A_ddz = np.gradient(A[:,:,:,:,2], z, axis=2, edge_order=2)
                # unsteady_p_dilatation = A_ddx + A_ddy + A_ddz
                
                unsteady_p_dilatation = pI * ( duIIdx + dvIIdy + dwIIdz )
                
                t_delta = timeit.default_timer() - t_start
                if verbose:
                    tqdm.write(even_print('calc p_dilatation', format_time_string(t_delta),s=True))
                
                self.comm.Barrier()
                
                ## combine sums across ranks for this [t] chunk
                unsteady_p_dilatation_sum_i   = np.sum(unsteady_p_dilatation, axis=3, keepdims=True, dtype=np.float64)
                unsteady_p_dilatation_sum_buf = np.zeros((nx,ny,nz,1), dtype=np.float64, order='C')
                self.comm.Reduce([unsteady_p_dilatation_sum_i, MPI.DOUBLE],
                                 [unsteady_p_dilatation_sum_buf, MPI.DOUBLE],
                                 op=MPI.SUM,
                                 root=0)
                
                ## add aggregated sum (across ranks) to the total accumulator
                unsteady_p_dilatation_sum += unsteady_p_dilatation_sum_buf
                
                ## release mem
                unsteady_p_dilatation = None
                del unsteady_p_dilatation
                
                self.comm.Barrier()
                mem_avail_gb = psutil.virtual_memory().available/1024**3
                mem_free_gb  = psutil.virtual_memory().free/1024**3
                if verbose:
                    tqdm.write(even_print('mem available', '%0.1f [GB]'%mem_avail_gb, s=True))
                    tqdm.write(even_print('mem free',      '%0.1f [GB]'%mem_free_gb,  s=True))
            
            ## manual garbage collect
            #gc.collect()
            #self.comm.Barrier()
            
            if verbose: progress_bar.update()
            
            if verbose: tqdm.write(72*'-')
        
        if verbose: progress_bar.close()
        
        ## CLOSE the files that were opened
        hf_mean.close()
        #hf_prime.close()
        
        # === multiply accumulators by (1/n) to get [t] avg --> these are 3D [x,y,z]
        
        if (self.rank==0):
            production   = np.copy( ((1/nt) * unsteady_production_sum   ) )
            dissipation  = np.copy( ((1/nt) * unsteady_dissipation_sum  ) )
            transport    = np.copy( ((1/nt) * unsteady_transport_sum    ) )
            diffusion    = np.copy( ((1/nt) * unsteady_diffusion_sum    ) )
            p_diffusion  = np.copy( ((1/nt) * unsteady_p_diffusion_sum  ) )
            p_dilatation = np.copy( ((1/nt) * unsteady_p_dilatation_sum ) )
        self.comm.Barrier()
        
        # === write to HDF5 --> [x,y,z,1]
        
        #with rgd(fn_h5_turb_budget, 'a', driver='mpio', comm=self.comm) as f1:
        if (self.rank==0):
            with rgd(fn_h5_turb_budget, 'a') as f1:
                f1['data/production'][:,:,:,:]   = production.T.astype(np.float32)
                f1['data/dissipation'][:,:,:,:]  = dissipation.T.astype(np.float32)
                f1['data/transport'][:,:,:,:]    = transport.T.astype(np.float32)
                f1['data/diffusion'][:,:,:,:]    = diffusion.T.astype(np.float32)
                f1['data/p_diffusion'][:,:,:,:]  = p_diffusion.T.astype(np.float32)
                f1['data/p_dilatation'][:,:,:,:] = p_dilatation.T.astype(np.float32)
        self.comm.Barrier()
        
        # === write to .dat --> pre-average in [x,z] leaving [y]
        
        if (self.rank==0):
            
            ## avg in [x,z] --> leave [y]
            data['production']   = np.copy( np.squeeze( np.mean( production   , axis=(0,2) , dtype=np.float64 ) ) )
            data['dissipation']  = np.copy( np.squeeze( np.mean( dissipation  , axis=(0,2) , dtype=np.float64 ) ) )
            data['transport']    = np.copy( np.squeeze( np.mean( transport    , axis=(0,2) , dtype=np.float64 ) ) )
            data['diffusion']    = np.copy( np.squeeze( np.mean( diffusion    , axis=(0,2) , dtype=np.float64 ) ) )
            data['p_diffusion']  = np.copy( np.squeeze( np.mean( p_diffusion  , axis=(0,2) , dtype=np.float64 ) ) )
            data['p_dilatation'] = np.copy( np.squeeze( np.mean( p_dilatation , axis=(0,2) , dtype=np.float64 ) ) )
            
            with open(fn_dat_turb_budget,'wb') as f1:
                pickle.dump(data, f1, protocol=4)
            print('--w-> %s : %0.2f [MB]'%(fn_dat_turb_budget,os.path.getsize(fn_dat_turb_budget)/1024**2))
        self.comm.Barrier()
        
        # ===
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_turb_budget() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        ## make .xdmf
        with rgd(fn_h5_turb_budget, 'r', driver='mpio', comm=self.comm) as f1:
            f1.make_xdmf()
        
        ## make .xdmf
        if save_unsteady:
            with rgd(fn_h5_turb_budget_unsteady, 'r', driver='mpio', comm=self.comm) as f1:
                f1.make_xdmf()
        
        return
    
    def calc_turb_budget_spectrum(self, **kwargs):
        '''
        calculate amplitude spectrum in [z,t] at every [x,y] of unsteady turbulent budget
        - designed for analyzing unsteady, thin planes in [x]
        - requires that calc_turb_budget() has been run with save_unsteady=True
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_turb_budget_spectrum()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## assert that the opened RGD has fsubtype 'unsteady' (i.e. is NOT a prime file)
        if (self.fsubtype!='unsteady'):
            raise ValueError
        if not self.usingmpi:
            raise NotImplementedError('function is not implemented for non-MPI usage')
        
        h5py_is_mpi_build = h5py.h5.get_config().mpi
        if not h5py_is_mpi_build:
            if verbose: print('h5py was not compiled for parallel usage! exiting.')
            sys.exit(1)
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        cy = kwargs.get('cy',1) ## number of chunks per rank [y] range
        
        #n_threads = kwargs.get('n_threads',1)
        try:
            n_threads = int(os.environ.get('OMP_NUM_THREADS'))
        except TypeError: ## not set
            n_threads = os.cpu_count()
        
        #fn_dat_fft = kwargs.get('fn_dat_fft',None) ## filename for output pickle (.dat) file
        fn_h5_fft  = kwargs.get('fn_h5_fft',None) ## filename for output HDF5 (.h5) file
        
        overlap_fac_nom = kwargs.get('overlap_fac_nom',0.50)
        n_win           = kwargs.get('n_win',8)
        window_type     = kwargs.get('window_type','hann') ## 'tukey','hann'
        
        ## only distribute data across [y]
        if (rx!=1):
            raise AssertionError('rx!=1')
        if (rz!=1):
            raise AssertionError('rz!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        
        ## check the choice of ranks per dimension
        if (rx*ry*rz*rt != self.n_ranks):
            raise AssertionError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (rt>self.nt):
            raise AssertionError('rt>self.nt')
        
        if not isinstance(cy, (int,np.int32,np.int64)):
            raise TypeError
        
        ## distribute 4D data over ranks --> here only in [y]
        ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        ## check all [y] ranges have same size (1/2) --> not strictly necessary
        if (self.ny%self.n_ranks!=0):
            raise ValueError('ny%n_ranks!=0')
        
        ## check all [y] ranges have same size (2/2) --> not strictly necessary
        for ryl_ in ryl:
            if not (ryl_[1]-ryl_[0]==nyr):
                raise ValueError('[y] chunks are not even in size')
        
        ## [y] sub chunk range --> cyl = list of ranges in ry1:ry2
        ## cy is the NUMBER of chunks for the rank sub-range
        cyl_ = np.array_split( np.arange(ry1,ry2) , min(cy,nyr) )
        cyl  = [[b[0],b[-1]+1] for b in cyl_ ]
        
        ## check sub ranges
        for nyc_ in [ cyl_[1]-cyl_[0] for cyl_ in cyl ]:
            if (nyc_ < 1):
                #raise ValueError
                print(f'rank {self.rank:d}: sub-range is <1')
                self.comm.Abort(1)
        
        if 1: ## assert that [y] sub-chunk ranges are correct
            
            yi = np.arange(self.ny, dtype=np.int32)
            
            local_indices = []
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                local_indices += [ yi_ for yi_ in yi[cy1:cy2] ]
            
            G = self.comm.gather([ self.rank , local_indices ], root=0)
            G = self.comm.bcast(G, root=0)
            
            all_indices = []
            for G_ in G:
                all_indices += G_[1]
            all_indices = np.array( sorted(all_indices), dtype=np.int32 )
            
            if not np.array_equal( all_indices , yi ):
                raise AssertionError
        
        # ## output filename : pickle (.dat)
        # if (fn_dat_fft is None): ## automatically determine name
        #     fname_path = os.path.dirname(self.fname)
        #     fname_base = os.path.basename(self.fname)
        #     fname_root, fname_ext = os.path.splitext(fname_base)
        #     fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
        #     fname_fft_dat_base = fname_root+'_turb_budget_fft.dat'
        #     fn_dat_fft = str(PurePosixPath(fname_path, fname_fft_dat_base))
        # if (Path(fn_dat_fft).suffix != '.dat'):
        #     raise ValueError(f"fn_dat_fft='{str(fn_dat_fft)}' must end in .dat")
        
        ## output filename : HDF5 (.h5)
        if (fn_h5_fft is None): ## automatically determine name
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_h5_base = fname_root+'_turb_budget_fft.h5'
            fn_h5_fft = str(PurePosixPath(fname_path, fname_fft_h5_base))
        if (Path(fn_h5_fft).suffix != '.h5'):
            raise ValueError(f"fn_h5_fft='{str(fn_h5_fft)}' must end in .h5")
        if os.path.isfile(fn_h5_fft):
            #if (os.path.getsize(fn_h5_fft) > 8*1024**3):
            #    raise ValueError(f"fn_h5_fft='{str(fn_h5_fft)}' exists and is >8 [GB]. exiting for your own safety.")
            if (fn_h5_fft == self.fname):
                raise ValueError(f"fn_h5_fft='{str(fn_h5_fft)}' cannot be same as input filename.")
        
        if verbose: even_print( 'fn_h5'      , self.fname )
        #if verbose: even_print( 'fn_dat_fft' , fn_dat_fft )
        if verbose: even_print( 'fn_h5_fft'  , fn_h5_fft  )
        if verbose: print(72*'-')
        self.comm.Barrier()
        
        ## the data dictionary to be pickled later
        data = {}
        
        ## infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),'%0.1f [GB]'%fsize)
        if verbose: even_print('nx',f'{self.nx:d}')
        if verbose: even_print('ny',f'{self.ny:d}')
        if verbose: even_print('nz',f'{self.nz:d}')
        if verbose: even_print('nt',f'{self.nt:d}')
        if verbose: even_print('ngp',f'{self.ngp/1e6:0.1f} [M]')
        if verbose: even_print('cy',f'{cy:d}')
        if verbose: even_print('n_ranks',f'{self.n_ranks:d}')
        if verbose: even_print('n_threads',f'{n_threads:d}')
        if verbose: print(72*'-')
        
        ## 0D freestream scalars
        lchar   = self.lchar   ; data['lchar']   = lchar
        U_inf   = self.U_inf   ; data['U_inf']   = U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = T_inf
        
        #data['M_inf'] = self.M_inf
        data['Ma'] = self.Ma
        data['Pr'] = self.Pr
        
        ## read in 1D coordinate arrays & re-dimensionalize
        x = np.copy( self['dims/x'][()] * self.lchar )
        y = np.copy( self['dims/y'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        
        nx = self.nx ; data['nx'] = nx
        ny = self.ny ; data['ny'] = ny
        nz = self.nz ; data['nz'] = nz
        nt = self.nt ; data['nt'] = nt
        
        ## assert constant Δz
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-6)):
            raise NotImplementedError('Δz not constant')
        dz = np.diff(z)[0]
        
        ## dimensional [s]
        dt = self.dt * self.tchar
        np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-12, atol=1e-12)
        
        t_meas = self.duration * self.tchar
        np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-12, atol=1e-12)
        
        zrange = z.max() - z.min()
        
        data['x'] = x
        data['y'] = y
        data['z'] = z
        
        data['t'] = t
        data['t_meas'] = t_meas
        data['dt'] = dt
        data['dz'] = dz
        data['zrange'] = zrange
        
        if verbose: even_print( 'Δt/tchar'       , f'{dt/self.tchar:0.8f}' )
        if verbose: even_print( 'Δt'             , f'{dt:0.3e} [s]'        )
        if verbose: even_print( 'duration/tchar' , f'{self.duration:0.1f}' )
        if verbose: even_print( 'duration'       , f'{self.duration*self.tchar:0.3e} [s]' )
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('Δt'     , f'{dt    :0.5e} [s]' )
            even_print('t_meas' , f'{t_meas:0.5e} [s]' )
            even_print('Δz'     , f'{dz0   :0.5e} [m]' )
            even_print('zrange' , f'{zrange:0.5e} [m]' )
            print(72*'-')
        
        ## establish [t] windowing
        win_len, overlap = get_overlapping_window_size(nt, n_win, overlap_fac_nom)
        overlap_fac = overlap / win_len
        tw, n_win, n_pad = get_overlapping_windows(t, win_len, overlap)
        t_meas_per_win = (win_len-1)*dt
        data['win_len']        = win_len
        data['overlap_fac']    = overlap_fac
        data['overlap']        = overlap
        data['n_win']          = n_win
        data['t_meas_per_win'] = t_meas_per_win
        
        if verbose:
            even_print('overlap_fac (nominal)' , f'{overlap_fac_nom:0.5f}'  )
            even_print('n_win'                 , f'{n_win:d}'               )
            even_print('win_len'               , f'{win_len:d}'             )
            even_print('overlap'               , f'{overlap:d}'             )
            even_print('overlap_fac'           , f'{overlap_fac:0.5f}'      )
            even_print('n_pad'                 , f'{n_pad:d}'               )
            #even_print('t_win/(δ99/uτ)'        , '%0.3f [-]'%t_eddy_per_win )
            print(72*'-')
        
        ## temporal [t] frequency (f) vector (Short Time Fourier Transform)
        freq_full = sp.fft.fftfreq(n=win_len, d=dt)
        fp        = np.where(freq_full>0) ## indices of positive values
        freq      = np.copy(freq_full[fp])
        df        = freq[1] - freq[0]
        nf        = freq.size
        
        data['freq'] = freq
        data['df']   = df
        data['nf']   = nf
        
        if verbose:
            even_print('freq min',f'{freq.min():0.1f} [Hz]')
            even_print('freq max',f'{freq.max():0.1f} [Hz]')
            even_print('df',f'{df:0.1f} [Hz]')
            even_print('nf',f'{nf:d}')
            #period_eddy = (1/freq) / (d99/u_tau)
            #period_plus = (1/freq) / sc_t_in
            #even_print('min : period+ = (1/f)/(νw/uτ²)'     , '%0.5f [-]'%period_plus.min())
            #even_print('max : period_eddy = (1/f)/(δ99/uτ)' , '%0.5f [-]'%period_eddy.max())
            print(72*'-')
        
        ## spatial [z] wavenumber (kz) vector
        kz_full = sp.fft.fftfreq(n=nz, d=dz0) * ( 2 * np.pi )
        kzp     = np.where(kz_full>0) ## indices of positive values
        kz      = np.copy(kz_full[kzp])
        dkz     = kz[1] - kz[0]
        nkz     = kz.shape[0]
        
        ## wavenumber vector should be size nz//2-1
        if (nkz!=nz//2-1):
            raise ValueError
        
        data['kz']  = kz
        data['dkz'] = dkz
        data['nkz'] = nkz
        
        if verbose:
            even_print('kz min',f'{kz.min():0.1f} [1/m]')
            even_print('kz max',f'{kz.max():0.1f} [1/m]')
            even_print('dkz',f'{dkz:0.1f} [1/m]')
            even_print('nkz',f'{nkz:d}')
            print(72*'-')
        
        ## wavelength λz = (2·π)/kz
        lz = np.copy( 2 * np.pi / kz )
        data['lz'] = lz
        
        # ===
        
        scalars = [ 'production', 'dissipation', 'transport', 'diffusion', ]
        scalars_dtypes  = [ np.dtype(np.float64) for s in scalars ]
        
        ## assert all scalars are in file
        for scalar in scalars:
            if ( f'data/{scalar}' not in self ):
                raise ValueError(f"'data/{scalar} not found. exiting.")
        
        ## generate avg scalar names
        scalars_avg = scalars
        
        ## numpy formatted arrays: buffers for PSD & other data (rank-local)
        Ekz    = np.zeros(shape=(nyr,nkz ) , dtype={'names':scalars,     'formats':scalars_dtypes})
        Ef     = np.zeros(shape=(nyr,nf  ) , dtype={'names':scalars,     'formats':scalars_dtypes})
        avg_Re = np.zeros(shape=(nyr,    ) , dtype={'names':scalars_avg, 'formats':[ np.dtype(np.float64) for s in scalars_avg ]})
        
        if verbose:
            even_print('n fft scalars' , '%i'%(len(scalars),))
            print(72*'-')
        
        ## no window for [z]
        window_z = np.ones(nz, dtype=np.float64)
        sum_win_z = np.sum(window_z)
        #sum_sqrt_win_z = np.sum(np.sqrt(window_z))
        if verbose:
            #even_print('sum(sqrt(window_z))'     , '%0.5f'%(sum_sqrt_win_z,))
            even_print('sum(window_z) / nz', '%0.5f'%(sum_win_z/nz,))
            #even_print('sum(sqrt(window_z)) / nz', '%0.5f'%(sum_sqrt_win_z/nz,))
        
        ## window function for [t]
        if (window_type=='tukey'):
            window_t = sp.signal.windows.tukey(win_len,alpha=0.5,sym=False) ## α=0:rectangular, α=1:Hann
        elif (window_type=='hann'):
            window_t = sp.signal.windows.hann(win_len,sym=False)
        elif (window_type is None):
            window_t = np.ones(win_len, dtype=np.float64)
        else:
            raise ValueError
        
        if verbose:
            even_print('window type [t]', '\'%s\''%str(window_type))
        
        ## sum of sqrt of window: needed for normalization
        sum_win_t = np.sum(window_t)
        #sum_sqrt_win_t = np.sum(np.sqrt(window_t))
        if verbose:
            #even_print('sum(sqrt(window_t))'          , '%0.5f'%(sum_sqrt_win_t,))
            even_print('sum(window_t) / win_len', '%0.5f'%(sum_win_t/win_len,))
            #even_print('sum(sqrt(window_t)) / win_len', '%0.5f'%(sum_sqrt_win_t/win_len,))
        
        #if verbose: print(72*'-')
        
        # === main loop
        
        if verbose:
            progress_bar = tqdm(total=len(scalars)*cy, ncols=100, desc='fft', leave=False, file=sys.stdout, smoothing=0.)
        
        for cci,scalar in enumerate(scalars):
            
            if verbose: tqdm.write(72*'-')
            
            msg = f'FFT[{scalar}]'
            if verbose:
                tqdm.write(even_print('computing',msg,s=True,))
            
            if verbose: ## report per-rank read shape
                nyc_max = max([ cyl_[1]-cyl_[0] for cyl_ in cyl ])
                data_gb = 8 * nx * nyc_max * nz * nt / 1024**3
                tqdm.write(even_print('read shape', f'[{nx:d},{nyc_max:d},{nz:d},{nt:d}] · 8 [Bytes] --> {data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print('mem per read', f'{self.n_ranks*data_gb:0.1f} [GB]', s=True))
                tqdm.write(even_print('mem per read ×3 / 245 [GB/Node]', f'{self.n_ranks*data_gb*3/245.:0.3f} [Nodes]', s=True))
            
            dset = self[f'data/{scalar}']
            
            ## [y] loop outer (chunks within rank)
            for cyl_ in cyl:
                cy1, cy2 = cyl_
                nyc = cy2 - cy1
                
                self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## read scalar data
                with dset.collective:
                    data_LR = np.copy( dset[:,:,cy1:cy2,:].T ).astype(np.float64)
                
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ( self.nx * ry * (cy2-cy1) * self.nz * self.nt * dset.dtype.itemsize ) / 1024**3
                if verbose:
                    tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                
                ## data_LR should be [nx,nyc,nz,nt] where nyc is the chunk [y] range
                if ( data_LR.shape != (nx,nyc,nz,nt) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                ## < here we would re-dimensionalize but turb budget scalars from calc_turb_budget() are already dimensional >
                
                # === compute mean-removed data
                
                ## avg(□) in [t]
                data_avg = np.mean( data_LR , axis=3, dtype=np.float64, keepdims=True) ## [x,y,z,1]
                
                ## < not removing mean □′ here >
                
                yiA = cy1 - ry1
                yiB = cy2 - ry1
                
                ## write (rank-local) 1D [y] averages
                avg_Re[scalar][yiA:yiB] = np.squeeze( np.mean( data_avg , axis=(0,2,3) , dtype=np.float64) )
                
                # ===============================================================================
                # At this point you have 4D [x,y,z,t] data_LR, NOT mean-removed
                # ===============================================================================
                
                def __fft_z_thread_kernel(xi,ti,yii):
                    data_ijl  = data_LR[xi,yii,:,ti]
                    A1 = sp.fft.fft( data_ijl * window_z )[kzp]
                    E_ijl = 2. * np.abs(A1) / ( dkz * sum_win_z )
                    return xi,ti,E_ijl
                
                def __fft_t_thread_kernel(xi,zi,yii):
                    data_ijk = data_LR[xi,yii,zi,:]
                    data_ijk_2D, nw, n_pad = get_overlapping_windows(data_ijk, win_len, overlap)
                    
                    ## STFT buffer
                    E_ijk_buf = np.zeros((nw,nf), dtype=np.float64)
                    
                    ## compute fft for each overlapped window segment
                    for wi in range(nw):
                        data_ijk_segment = data_ijk_2D[wi,:]
                        A1 = sp.fft.fft( data_ijk_segment * window_t )[fp]
                        E_ijk_buf[wi,:] = 2. * np.abs(A1) / ( df * sum_win_t )
                    
                    ## mean across short time FFT (STFT) segments
                    E_ijk = np.mean(E_ijk_buf, axis=0, dtype=np.float64)
                    
                    return xi,zi,E_ijk
                
                ## [y] loop inner (indices within chunk)
                for yi in range(cy1,cy2):
                    
                    yii  = yi - cy1 ## chunk local
                    yiii = yi - ry1 ## rank local
                    
                    ## PSD buffers for [y] loop inner
                    E_xt = np.zeros((nx,nt,nkz) , dtype=np.float64) ## [x,t] range for FFT(z)
                    E_xz = np.zeros((nx,nz,nf ) , dtype=np.float64) ## [x,z] range for FFT(t)
                    
                    # ===========================================================================
                    # FFT(z) : loop over [x,t]
                    # ===========================================================================
                    
                    ## concurrent/threaded execution for fft(z)
                    tasks = [(xi,ti,yii) for xi in range(nx) for ti in range(nt)]
                    with ThreadPoolExecutor(max_workers=n_threads) as executor:
                        results = executor.map(lambda t: __fft_z_thread_kernel(*t,), tasks)
                        for xi,ti,result in results:
                            E_xt[xi,ti,:] = result
                    
                    # for xi in range(nx):
                    #     for ti in range(nt):
                    #         ...
                    
                    ## avg in [x,t] & write in rank context
                    Ekz[scalar][yiii,:] = np.mean(E_xt, axis=(0,1), dtype=np.float64)
                    
                    # ===========================================================================
                    # FFT(t) : loop over [x,z], use windows
                    # ===========================================================================
                    
                    ## concurrent/threaded execution for fft(t)
                    tasks = [(xi,zi,yii) for xi in range(nx) for zi in range(nz)]
                    with ThreadPoolExecutor(max_workers=n_threads) as executor:
                        results = executor.map(lambda t: __fft_t_thread_kernel(*t,), tasks)
                        for xi,zi,result in results:
                            E_xz[xi,zi,:] = result
                    
                    # for xi in range(nx):
                    #     for zi in range(nz):
                    #         ...
                    
                    ## avg in [x,z] & write in rank context
                    Ef[scalar][yiii,:] = np.mean(E_xz, axis=(0,1), dtype=np.float64)
                
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: tqdm.write(even_print(msg, format_time_string(t_delta), s=True))
                if verbose: progress_bar.update() ## (scalar, [y] chunk) progress
        if verbose: progress_bar.close()
        self.comm.Barrier()
        if verbose: print(72*'-')
        
        # ==============================================================
        # write HDF5 (.h5) file
        # ==============================================================
        
        ## open on rank 0 and write attributes, dimensions, etc.
        if (self.rank==0):
            with h5py.File(fn_h5_fft, 'w') as hfw:
                
                ## write floats,ints as top-level attributes
                for key,val in data.items():
                    if isinstance(data[key], (int,np.int32,np.int64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], (float,np.float32,np.float64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], np.ndarray):
                        pass
                    else:
                        print(f'key {key} is type {str(type(data[key]))}')
                        self.comm.Abort(1)
                
                ## write numpy arrays
                hfw.create_dataset( 'dims/x'    , data=x    ) ## [m]
                hfw.create_dataset( 'dims/y'    , data=y    ) ## [m]
                hfw.create_dataset( 'dims/z'    , data=z    ) ## [m]
                hfw.create_dataset( 'dims/t'    , data=t    ) ## [s]
                hfw.create_dataset( 'dims/freq' , data=freq ) ## [1/s] | [Hz]
                hfw.create_dataset( 'dims/kz'   , data=kz   ) ## [1/m]
                hfw.create_dataset( 'dims/lz'   , data=lz   ) ## [m]
        
        self.comm.Barrier()
        
        with h5py.File(fn_h5_fft, 'a', driver='mpio', comm=self.comm) as hfw:
            
            ## initialize datasets
            for scalar in scalars:
                hfw.create_dataset( f'Ekz/{scalar}' , shape=(ny,nkz), dtype=np.float64, chunks=(1,nkz) )
                hfw.create_dataset( f'Ef/{scalar}'  , shape=(ny,nf ), dtype=np.float64, chunks=(1,nf)  )
            
            ## initialize datasets 1D [y] mean
            for scalar in avg_Re.dtype.names:
                hfw.create_dataset( f'avg/Re/{scalar}', shape=(ny,), dtype=np.float64, chunks=(1,) )
            
            self.comm.Barrier()
            
            ## collectively write Ekz,Ef
            for scalar in scalars:
                dset = hfw[f'Ekz/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:] = Ekz[scalar][:,:]
                dset = hfw[f'Ef/{scalar}']
                with dset.collective:
                    dset[ry1:ry2,:] = Ef[scalar][:,:]
            
            ## collectively write 1D [y] avgs
            for scalar in avg_Re.dtype.names:
                dset = hfw[f'avg/Re/{scalar}']
                with dset.collective:
                    dset[ry1:ry2] = avg_Re[scalar][:]
        
        self.comm.Barrier()
        if verbose:
            print('--w-> %s : %0.2f [MB]'%(fn_h5_fft,os.path.getsize(fn_h5_fft)/1024**2))
            print(72*'-')
        
        ## report file contents
        self.comm.Barrier()
        if (self.rank==0):
            with h5py.File(fn_h5_fft,'r') as hfr:
                h5_print_contents(hfr)
        self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_turb_budget_spectrum() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_multi_mean(self, n=3, **kwargs):
        '''
        calculate n temporal means
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.calc_multi_mean()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        fn_rgd_mean  = kwargs.get('fn_rgd_mean',None) ## outfile name
        ti_min       = kwargs.get('ti_min',None)
        favre        = kwargs.get('favre',True)
        reynolds     = kwargs.get('reynolds',True)
        ct           = kwargs.get('ct',1)
        force        = kwargs.get('force',False)
        
        chunk_kb         = kwargs.get('chunk_kb',4*1024) ## h5 chunk size: default 4 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None)) ## the 'constraint' parameter for sizing h5 chunks
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing mean file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        #if not isinstance(n, (int,np.int32,np.int64)):
        if not isinstance(n, int):
            raise ValueError
        if (n<1):
            raise ValueError
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (ry>self.ny):
            raise AssertionError('ry>self.ny')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
        
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
            ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
            rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
            #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        # === mean file name (for writing)
        if (fn_rgd_mean is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_mean_h5_base = fname_root+'_mean_multi.h5'
            #fn_rgd_mean = os.path.join(fname_path, fname_mean_h5_base)
            fn_rgd_mean = str(PurePosixPath(fname_path, fname_mean_h5_base))
            #fn_rgd_mean = Path(fname_path, fname_mean_h5_base)
        
        if verbose: even_print('fn_rgd'       , self.fname   )
        if verbose: even_print('fn_rgd_mean'  , fn_rgd_mean  )
        #if verbose: even_print('fn_rgd_prime' , fn_rgd_prime )
        if verbose: even_print('do Favre avg' , str(favre)   )
        if verbose: even_print('do Reynolds avg' , str(reynolds)   )
        if verbose: print(72*'-')
        if verbose: even_print('nx','%i'%self.nx)
        if verbose: even_print('ny','%i'%self.ny)
        if verbose: even_print('nz','%i'%self.nz)
        if verbose: even_print('nt','%i'%self.nt)
        if verbose: print(72*'-')
        if verbose: even_print('rx','%i'%rx)
        if verbose: even_print('ry','%i'%ry)
        if verbose: even_print('rz','%i'%rz)
        if verbose: even_print('ct','%i'%ct)
        if verbose: print(72*'-')
        
        ## get times to take for avg
        if (ti_min is not None):
            ti_for_avg = np.copy( self.ti[ti_min:] )
        else:
            ti_for_avg = np.copy( self.ti )
        
        nt_avg       = ti_for_avg.shape[0]
        #t_avg_start  = self.t[ti_for_avg[0]]
        #t_avg_end    = self.t[ti_for_avg[-1]]
        #duration_avg = t_avg_end - t_avg_start ## for complete time
        
        #if not isinstance(ct, (int,np.int32,np.int64)):
        if not isinstance(ct, int):
            raise ValueError
        if (ct<1):
            raise ValueError
        
        ## assert constant Δt, later attach dt as attribute to mean file
        dt0 = np.diff(self.t)[0]
        if not np.all(np.isclose(np.diff(self.t), dt0, rtol=1e-7)):
            raise ValueError
        
        if verbose: even_print('n timesteps avg','%i/%i'%(nt_avg,self.nt))
        #if verbose: even_print('t index avg start','%i'%(ti_for_avg[0],))
        #if verbose: even_print('t index avg end','%i'%(ti_for_avg[-1],))
        #if verbose: even_print('t avg start','%0.2f [-]'%(t_avg_start,))
        #if verbose: even_print('t avg end','%0.2f [-]'%(t_avg_end,))
        #if verbose: even_print('duration avg','%0.2f [-]'%(duration_avg,))
        #if verbose: even_print('Δt','%0.2f [-]'%(dt0,))
        #if verbose: print(72*'-')
        
        if verbose: even_print('ti start',f'{ti_for_avg.min():d}')
        if verbose: even_print('ti end',f'{ti_for_avg.max():d}')
        if verbose: even_print('n windows',str(n))
        
        nt_win = int( np.floor( ti_for_avg.shape[0] / n ) ) ## number of timesteps per mean window
        if (nt_win<1):
            raise ValueError
        if verbose: even_print('nt win',f'{nt_win:d}')
        
        if verbose: even_print('Δt per window',f'{((nt_win-1)*dt0):0.2f} [-]')
        
        ## construct array of time indices per window
        ti_for_avg_win = np.zeros((n,nt_win),dtype=np.int64)
        for ni in range(n):
            a_ = ti_for_avg[0] + ni*nt_win + np.arange(nt_win,dtype=np.int64)
            if verbose: even_print(f'win {ni} ti min max',f'{a_.min():d} {a_.max():d}')
            ti_for_avg_win[ni,:] = a_
        
        if verbose: print(72*'-')
        
        ## performance
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        ##data_gb      = 4*self.nx*self.ny*self.nz*self.nt / 1024**3
        #data_gb      = 4*self.nx*self.ny*self.nz*nt_avg / 1024**3
        #data_gb_mean = 4*self.nx*self.ny*self.nz*1      / 1024**3
        
        scalars_re = ['u','v','w','p','T','rho']
        scalars_fv = ['u','v','w','T'] ## 'p','rho'
        
        ## do a loop through to get names
        scalars_mean_names  = []
        scalars_mean_dtypes = []
        for scalar in self.scalars:
            dtype = self.scalars_dtypes_dict[scalar]
            if reynolds:
                if True: ## always
                    sc_name = scalar
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
            if favre:
                if (scalar in scalars_fv):
                    sc_name = f'r_{scalar}'
                    scalars_mean_names.append(sc_name)
                    scalars_mean_dtypes.append(dtype)
        
        #with rgd(fn_rgd_mean, 'w', force=force, driver='mpio', comm=MPI.COMM_WORLD) as hf_mean:
        with rgd(fn_rgd_mean, 'w', force=force, driver=self.driver, comm=self.comm, stripe_count=stripe_count, stripe_size_mb=stripe_size_mb) as hf_mean:
            
            ## initialize the mean file from the opened unsteady rgd file
            hf_mean.init_from_rgd(self.fname)
            
            ## set some top-level attributes
            hf_mean.attrs['duration_avg'] = nt_win * dt0 ## duration of each mean window
            #hf_mean.attrs['duration_avg'] = duration_avg ## duration of mean
            #hf_mean.attrs['duration_avg'] = self.duration
            hf_mean.attrs['dt'] = dt0
            hf_mean.attrs['fclass'] = 'rgd'
            hf_mean.attrs['fsubtype'] = 'mean'
            
            # === initialize mean datasets
            for scalar in self.scalars:
                
                dtype = self.scalars_dtypes_dict[scalar]
                float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                
                data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                
                shape  = (n,self.nz,self.ny,self.nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                
                if reynolds:
                    
                    ## do the Re mean of all scalars in file, regardless whether explicitly in scalars_re or not
                    #if scalar in scalars_re:
                    if True:
                        
                        if ('data/%s'%scalar in hf_mean):
                            del hf_mean['data/%s'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}' , f'{data_gb_mean:0.3f} [GB]' )
                            even_print( f'shape' , str(shape) )
                        dset = hf_mean.create_dataset(f'data/{scalar}',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if favre:
                    
                    if (scalar in scalars_fv):
                        if ('data/%s_fv'%scalar in hf_mean):
                            del hf_mean['data/%s_fv'%scalar]
                        if verbose:
                            even_print( f'initializing data/{scalar}_fv' , f'{data_gb_mean:0.3f} [GB]' )
                            even_print( f'shape' , str(shape) )
                        dset = hf_mean.create_dataset(f'data/{scalar}_fv',
                                                      shape=shape,
                                                      dtype=dtype,
                                                      chunks=chunks,
                                                      )
                        hf_mean.scalars.append('data/%s_fv'%scalar)
                        
                        chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                        if verbose:
                            even_print('chunk shape (t,z,y,x)','%s'%str(dset.chunks))
                            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            if self.usingmpi: self.comm.Barrier()
            if verbose: print(72*'-')
            
            if verbose:
                progress_bar = tqdm(total=n*ct, ncols=100, desc='mean', leave=False, file=sys.stdout, smoothing=0.)
            
            for ni in range(n): ## each window
                
                ti_for_avg = np.copy( ti_for_avg_win[ni,:] ) ## overwrite
                
                if (ct>ti_for_avg.shape[0]):
                    raise ValueError
                
                ## [t] sub chunk range
                ctl_ = np.array_split( ti_for_avg_win[ni,:], ct )
                ctl = [[b[0],b[-1]+1] for b in ctl_ ]
                
                ## check that no sub ranges are <=1
                for a_ in [ ctl_[1]-ctl_[0] for ctl_ in ctl ]:
                    if (a_ <= 1):
                        raise ValueError
                
                ## accumulator array for local rank --> initialize
                data_sum = np.zeros(shape=(nxr,nyr,nzr,1), dtype={'names':scalars_mean_names, 'formats':[np.float64 for _ in scalars_mean_names]})
                
                for ci,ctl_ in enumerate(ctl): ## each chunk in time, read & do sum
                    ct1, ct2 = ctl_
                    ntc = ct2 - ct1
                    
                    if verbose: tqdm.write(f'win {ni:d} chunk {ci:d} --> ts {ct1:d}:{ct2:d}')
                    
                    # === read rho
                    if favre:
                        
                        dset = self['data/rho']
                        
                        dtype = self.scalars_dtypes_dict['rho']
                        if (dtype!=dset.dtype):
                            raise ValueError
                        float_bytes = self.scalars_dtypes_dict[scalar].itemsize
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        
                        if self.usingmpi: 
                            with dset.collective:
                                rho = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                        else:
                            rho = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if verbose:
                            txt = even_print('read: rho', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                    
                    # === read scalar data, do sum
                    for scalar in self.scalars:
                        
                        dset = self[f'data/{scalar}']
                        dtype = self.scalars_dtypes_dict[scalar]
                        if (dtype!=dset.dtype):
                            raise ValueError
                        float_bytes = dtype.itemsize
                        
                        # === collective read
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        
                        if self.usingmpi:
                            with dset.collective:
                                data = np.copy( dset[ct1:ct2,rz1:rz2,ry1:ry2,rx1:rx2].T ).astype(np.float64)
                        else:
                            data = np.copy( dset[ct1:ct2,:,:,:].T ).astype(np.float64)
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        #data_gb = float_bytes*self.nx*self.ny*self.nz*nt_avg / 1024**3
                        data_gb = float_bytes*self.nx*self.ny*self.nz*ntc / 1024**3
                        
                        if verbose:
                            txt = even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        # === do sum, add to accumulator
                        if reynolds:
                            sc_name = scalar
                            data_sum[sc_name] += np.sum(data, axis=-1, dtype=np.float64, keepdims=True)
                        if favre:
                            if (scalar in scalars_fv):
                                sc_name = f'r_{scalar}'
                                data_sum[sc_name] += np.sum(data*rho, axis=-1, dtype=np.float64, keepdims=True)
                        
                        if self.usingmpi: self.comm.Barrier()
                    
                    mem_avail_gb = psutil.virtual_memory().available/1024**3
                    mem_free_gb  = psutil.virtual_memory().free/1024**3
                    if verbose:
                        tqdm.write(even_print('mem free', '%0.1f [GB]'%mem_free_gb, s=True))
                    
                    if verbose: progress_bar.update()
                    if verbose: tqdm.write(72*'-')
                
                # ==========================================================
                # multiply accumulators by (1/nt_win)
                # ==========================================================
                
                for scalar in self.scalars:
                    if reynolds:
                        sc_name = scalar
                        data_sum[sc_name] *= (1./nt_win)
                    if favre:
                        if (scalar in scalars_fv):
                            sc_name = f'r_{scalar}'
                            data_sum[sc_name] *= (1./nt_win)
                
                # ==========================================================
                # 'data_sum' now contains averages, not sums!
                # ==========================================================
                
                ## Favre avg : φ_tilde = avg[ρ·φ]/avg[ρ]
                rho_mean = np.copy( data_sum['rho'] )
                
                # === write
                for scalar in self.scalars:
                    
                    if reynolds:
                        
                        dset = hf_mean[f'data/{scalar}']
                        dtype = dset.dtype
                        float_bytes = dtype.itemsize
                        
                        if (dtype==np.float32):
                            data_out = np.copy( data_sum[scalar].astype(np.float32) )
                        
                        if self.usingmpi: self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                dset[ni,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                        else:
                            dset[ni,:,:,:] = data_out.T
                        if self.usingmpi: self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                        
                        if verbose:
                            txt = even_print(f'write: {scalar}', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                            tqdm.write(txt)
                        
                        t_write       += t_delta
                        data_gb_write += data_gb_mean
                    
                    if favre:
                        if (scalar in scalars_fv):
                            
                            dset = hf_mean[f'data/{scalar}_fv']
                            dtype = dset.dtype
                            float_bytes = dtype.itemsize
                            
                            ## φ_tilde = avg[ρ·φ]/avg[ρ]
                            data_out = np.copy( data_sum[f'r_{scalar}'] / rho_mean )
                            
                            if (dtype==np.float32):
                                data_out = np.copy( data_out.astype(np.float32) )
                            
                            if self.usingmpi: self.comm.Barrier()
                            t_start = timeit.default_timer()
                            if self.usingmpi:
                                with dset.collective:
                                    dset[ni,rz1:rz2,ry1:ry2,rx1:rx2] = data_out.T
                            else:
                                dset[ni,:,:,:] = data_out.T
                            if self.usingmpi: self.comm.Barrier()
                            t_delta = timeit.default_timer() - t_start
                            
                            data_gb_mean = float_bytes*self.nx*self.ny*self.nz*1 / 1024**3
                            
                            if verbose:
                                txt = even_print(f'write: {scalar}_fv', '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb_mean,t_delta,(data_gb_mean/t_delta)), s=True)
                                tqdm.write(txt)
                            
                            t_write       += t_delta
                            data_gb_write += data_gb_mean
                
                if (ni!=n-1):
                    if verbose: tqdm.write(72*'-')
            
            # === replace dims/t array --> take last time of series
            t = np.zeros(n,dtype=np.float64)
            for ni in range(n): ## each window
                a_ = ti_for_avg_win[ni,:].min()
                b_ = ti_for_avg_win[ni,:].max()
                t[ni] = self.t[b_]
            if ('dims/t' in hf_mean):
                del hf_mean['dims/t']
            hf_mean.create_dataset('dims/t', data=t)
            
            # if hasattr(hf_mean, 'duration_avg'):
            #     if verbose: even_print('duration avg', '%0.2f [-]'%hf_mean.duration_avg)
            
            if verbose: progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: even_print('time read',format_time_string(t_read))
        if verbose: even_print('time write',format_time_string(t_write))
        if verbose: even_print(fn_rgd_mean, '%0.2f [GB]'%(os.path.getsize(fn_rgd_mean)/1024**3))
        if (t_read>0.):
            if verbose: even_print('read total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_read,t_read,(data_gb_read/t_read)))
        if (t_write>0.):
            if verbose: even_print('write total avg', '%0.2f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb_write,t_write,(data_gb_write/t_write)))
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.calc_multi_mean() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def export_polydata_ypln_wall(self, fn_spd=None, **kwargs):
        '''
        get 2D [x,z] wall quantities, export structured polydata (SPD) file
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        h5py_is_mpi_build = h5py.h5.get_config().mpi
        if not h5py_is_mpi_build:
            if verbose: print('h5py was not compiled for parallel usage! exiting.')
            sys.exit(1)
        
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        force    = kwargs.get('force',False)
        
        acc = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        ## chunk is set manually... see below
        #chunk_kb         = kwargs.get('chunk_kb',8*1024) ## h5 chunk size: default 8 [MB]
        #chunk_constraint = kwargs.get('chunk_constraint',(None,None,1)) ## the 'constraint' parameter for sizing h5 chunks (i,j,t)
        #chunk_base       = kwargs.get('chunk_base',2)
        
        ## for initializing SPD file
        stripe_count   = kwargs.pop('stripe_count'   , 16 )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        if verbose: print('\n'+'cgd.export_polydata_ypln_wall()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        if (rx>self.nx):
            raise AssertionError('rx>self.nx')
        if (rz>self.nz):
            raise AssertionError('rz>self.nz')
        if (ry!=1):
            raise AssertionError('ry!=1')
        if (rz!=1):
            raise AssertionError('rz!=1, which is currently unsupported')
        
        if (self.nx%rx!=0):
            raise AssertionError('nx currently needs to be divisible by the n ranks in [x]')
        
        if verbose: even_print('rx',f'{rx:d}')
        if verbose: even_print('rz',f'{rz:d}')
        if verbose: print(72*'-')
        
        ## get size of infile
        fsize = os.path.getsize(self.fname)/1024**3
        if verbose: even_print(os.path.basename(self.fname),f'{fsize:0.1f} [GB]')
        if verbose: even_print('nx',f'{self.nx:d}')
        if verbose: even_print('ny',f'{self.ny:d}')
        if verbose: even_print('nz',f'{self.nz:d}')
        if verbose: even_print('nt',f'{self.nt:d}')
        if verbose: even_print('ngp',f'{self.ngp/1e6:0.1f} [M]')
        if verbose: print(72*'-')
        
        ## polydata file name (for writing)
        if (fn_spd is None):
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            fname_spd_h5_base = fname_root+'_polydata.h5'
            fn_spd = str(PurePosixPath(fname_path, fname_spd_h5_base))
        
        if verbose: even_print('fn_spd',fn_spd)
        
        # === ranks
        
        if self.usingmpi:
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            rxl  = [[b[0],b[-1]+1] for b in rxl_ ]
            rx1, rx2 = rxl[self.rank]
            nxr = rx2 - rx1
        else:
            rx1,rx2 = 0,self.nx
            nxr = self.nx
        
        ## read & dimensionalize grid
        x = np.copy( self.x * self.lchar )
        y = np.copy( self.y * self.lchar )
        z = np.copy( self.z * self.lchar )
        
        #comm_spd = MPI.COMM_WORLD
        
        ## if file exists, just open in append m
        if os.path.isfile(fn_spd):
            fn_spd_openmode = 'a'
        else:
            fn_spd_openmode = 'w'
        
        ## open in parallel mode
        with spd(fn_spd, fn_spd_openmode,
                 force=force,
                 driver=self.driver,
                 #comm=comm_spd,
                 comm=self.comm,
                 stripe_count=stripe_count,
                 stripe_size_mb=stripe_size_mb, ) as hfspd:
            
            ## copy over attributes from RGD to SPD
            header_attr_str_list = ['Ma','Re','Pr','kappa','R','p_inf','T_inf','S_Suth','mu_Suth_ref','T_Suth_ref',]
            for key in header_attr_str_list: #for key in self.udef:
                hfspd.attrs[key] = self.udef[key]
            #for key in self.udef_deriv:
            #    hfspd.attrs[key] = self.udef_deriv[key]
            
            ## 1D [t] vector
            dsn = f'dims/t'
            if not (dsn in hfspd):
                ds = hfspd.create_dataset(dsn, data=self.t, chunks=None)
                if verbose: even_print(dsn,str(ds.shape))
            
            ## 3D xyz [x,z,3] array
            dsn = 'dims/xyz'
            if not (dsn in hfspd):
                
                ## 3D polydata grid coordinates : shape (nx,nz,3)
                xyz = np.zeros((self.nx, self.nz, 3), dtype=np.float64)
                xyz[:,:,0] = self.x[:,np.newaxis]
                xyz[:,:,1] = 0.
                xyz[:,:,2] = self.z[np.newaxis,:]
                self.comm.Barrier()
                
                hfspd.create_dataset(
                    name=dsn,
                    chunks=(1,self.nz,1),
                    shape=(self.nx, self.nz, 3),
                    dtype=np.float64,
                    )
                
                ## write dset
                self.comm.Barrier()
                ds = hfspd[dsn]
                with ds.collective:
                    ds[:,:,:] = xyz
                self.comm.Barrier()
                
                hfspd.flush()
                xyz = None; del xyz
                
                #if verbose: print(ds.chunks)
                if verbose: even_print(dsn,str(ds.shape))
            
            ## 1D [x] vector
            dsn = f'dims/x'
            if not (dsn in hfspd):
                ds = hfspd.create_dataset(dsn, data=self.x, chunks=None)
                if verbose: even_print(dsn,str(ds.shape))
            
            ## 1D [y] vector
            dsn = f'dims/y'
            if not (dsn in hfspd):
                ds = hfspd.create_dataset(dsn, data=np.array([0.,],dtype=np.float64), chunks=None)
                if verbose: even_print(dsn,str(ds.shape))
            
            ## 1D [z] vector
            dsn = f'dims/z'
            if not (dsn in hfspd):
                ds = hfspd.create_dataset(dsn, data=self.z, chunks=None)
                if verbose: even_print(dsn,str(ds.shape))
            
            ## add attributes
            hfspd.attrs['ni'] = self.nx
            hfspd.attrs['nj'] = self.nz
            hfspd.attrs['n_quads'] = ( self.nx - 1 ) * ( self.nz - 1 )
            hfspd.attrs['n_pts'] = self.nx * self.nz
            hfspd.attrs['nt'] = self.nt
            
            if verbose: print(72*'-')
            hfspd.get_header(verbose=verbose)
            if verbose: print(72*'-')
            
            ## initialize datasets
            #dtype    = np.dtype(np.float64)
            dtype    = np.dtype(np.float32)
            itemsize = dtype.itemsize
            shape    = (self.nx,self.nz,self.nt)
            data_gb  = np.prod(shape) * itemsize / 1024**3
            
            #chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=itemsize)
            chunks = ( 1, self.nz, min(8,self.nt) )
            
            ## list of dataset names to make in group data/
            scalars_spd = [
                'tau_uy','tau_vy','tau_wy',
                'u_tau','v_tau','w_tau',
                'T','mu','nu','rho','p',
                ]
            
            ## initialize output datasets
            for scalar in scalars_spd:
                
                dsn = f'data/{scalar}'
                if not (dsn in hfspd):
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if verbose:
                        even_print(f'initializing data/{scalar}','%0.1f [GB]'%(data_gb,))
                    
                    dset = hfspd.create_dataset(
                                            dsn,
                                            shape=shape,
                                            dtype=dtype,
                                            chunks=chunks,
                                            )
                    
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    if verbose: even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]')
                    
                    chunk_kb_ = np.prod(dset.chunks)*itemsize / 1024. ## actual
                    if verbose:
                        #even_print('chunk shape (t,z,x)','%s'%str(dset.chunks))
                        even_print('chunk shape (x,z,t)','%s'%str(dset.chunks))
                        even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            # ==========================================================
            # main loop
            # ==========================================================
            
            if verbose:
                print(72*'-')
                progress_bar = tqdm(total=nxr, ncols=100, desc='export polydata', leave=False, file=sys.stdout, smoothing=1.)
            
            ## iterate through rank-local [x] ranges, read, process, write
            for xi in range(rx1,rx2):
                
                names   = [ 'rho','u','v','w','T','p', ]
                #formats = [ np.dtype(np.float64) for s in names ]
                formats = [ np.dtype(np.float32) for s in names ]
                data    = np.zeros(shape=(1,self.ny,self.nz,self.nt), dtype={'names':names, 'formats':formats})
                
                for scalar in data.dtype.names:
                    dset = self[f'data/{scalar}']
                    if self.usingmpi: self.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset.collective:
                        data[scalar][:,:,:,:] = dset[:,:,:,xi].T
                    if self.usingmpi: self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = self.n_ranks * 1 * self.ny * self.nz * self.nt * dset.dtype.itemsize / 1024**3
                    if verbose:
                        tqdm.write( even_print(f'read {scalar}', f'{data_gb:0.3f} [GB]  {t_delta:0.3f} [s]  {data_gb/t_delta:0.3f} [GB/s]', s=True ) )
                
                ## re-dimensionalize
                data['rho'][:,:,:,:] *= self.rho_inf
                data['u'][:,:,:,:]   *= self.U_inf
                data['v'][:,:,:,:]   *= self.U_inf
                data['w'][:,:,:,:]   *= self.U_inf
                data['T'][:,:,:,:]   *= self.T_inf
                data['p'][:,:,:,:]   *= self.rho_inf * self.U_inf**2
                
                # ===
                
                #mu = np.copy( self.mu_Suth_ref * ( T / self.T_Suth_ref )**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(T+self.S_Suth)) )
                mu = np.copy( self.mu_Suth_ref * ( data['T'][:,:,:,:] / self.T_Suth_ref )**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(data['T'][:,:,:,:]+self.S_Suth)) )
                #nu = np.copy( mu / rho )
                nu = np.copy( mu / data['rho'][:,:,:,:] )
                
                if ( data['u'][:,:,:,:].shape != (1,self.ny,self.nz,self.nt) ): ## (1,y,z,t)
                    raise ValueError
                
                # ===
                
                ## dimensional wall strains
                ddy_u      = gradient( data['u'][:,:,:,:], y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_v      = gradient( data['v'][:,:,:,:], y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_w      = gradient( data['w'][:,:,:,:], y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_u_wall = np.copy( ddy_u[:,0,:,:] )
                ddy_v_wall = np.copy( ddy_v[:,0,:,:] )
                ddy_w_wall = np.copy( ddy_w[:,0,:,:] )
                
                if (ddy_u_wall.ndim!=3): ## (x,z,t)
                    raise ValueError
                if (ddy_u_wall.shape!=(1,self.nz,self.nt)): ## (x,z,t)
                    raise ValueError
                
                ## dimensional wall quantities
                rho_wall = np.copy( data['rho'][:,0,:,:] )
                mu_wall  = np.copy( mu[:,0,:,:]          )
                nu_wall  = np.copy( nu[:,0,:,:]          )
                T_wall   = np.copy( data['T'][:,0,:,:]   )
                p_wall   = np.copy( data['p'][:,0,:,:]   )
                
                if (mu_wall.ndim!=3): ## (x,z,t)
                    raise ValueError
                if (mu_wall.shape!=(1,self.nz,self.nt)): ## (x,z,t)
                    raise ValueError
                
                if (rho_wall.ndim!=3): ## (x,z,t)
                    raise ValueError
                if (rho_wall.shape!=(1,self.nz,self.nt)): ## (x,z,t)
                    raise ValueError
                
                tau_uy = np.copy( mu_wall * ddy_u_wall ) ## INSTANTANEOUS τw
                tau_vy = np.copy( mu_wall * ddy_v_wall )
                tau_wy = np.copy( mu_wall * ddy_w_wall )
                
                u_tau = np.copy( np.sign(tau_uy) * np.sqrt( np.abs(tau_uy) / rho_wall ) ) ## INSTANTANEOUS uτ
                v_tau = np.copy( np.sign(tau_vy) * np.sqrt( np.abs(tau_vy) / rho_wall ) ) ## INSTANTANEOUS vτ
                w_tau = np.copy( np.sign(tau_wy) * np.sqrt( np.abs(tau_wy) / rho_wall ) ) ## INSTANTANEOUS wτ
                
                # ===
                
                ## 3D [scalar][x,z,t] numpy structured array
                grp = hfspd['data']
                scalars_spd = list(grp.keys())
                #dtype = np.dtype(np.float64)
                dtype = np.dtype(np.float32)
                dtypes_spd = [ dtype for s in scalars_spd ]
                data4spd = np.zeros(shape=(1,self.nz,self.nt), dtype={'names':scalars_spd, 'formats':dtypes_spd})
                data4spd['tau_uy'][:,:,:] = tau_uy   / ( self.rho_inf * self.U_inf**2 )
                data4spd['tau_vy'][:,:,:] = tau_vy   / ( self.rho_inf * self.U_inf**2 )
                data4spd['tau_wy'][:,:,:] = tau_wy   / ( self.rho_inf * self.U_inf**2 )
                data4spd['T'][:,:,:]      = T_wall   / self.T_inf
                data4spd['rho'][:,:,:]    = rho_wall / self.rho_inf
                data4spd['p'][:,:,:]      = p_wall   / ( self.rho_inf * self.U_inf**2 )
                data4spd['mu'][:,:,:]     = mu_wall  / self.mu_inf
                data4spd['nu'][:,:,:]     = nu_wall  / self.nu_inf
                data4spd['u_tau'][:,:,:]  = u_tau    / self.U_inf
                data4spd['v_tau'][:,:,:]  = v_tau    / self.U_inf
                data4spd['w_tau'][:,:,:]  = w_tau    / self.U_inf
                
                for scalar in data4spd.dtype.names:
                    
                    dset = hfspd[f'data/{scalar}']
                    if hfspd.usingmpi: hfspd.comm.Barrier()
                    t_start = timeit.default_timer()
                    with dset.collective:
                        dset[xi,:,:] = data4spd[scalar][:,:,:]
                    if hfspd.usingmpi: hfspd.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = dset.dtype.itemsize * self.n_ranks * 1 * 1 * self.nz * self.nt / 1024**3
                    
                    if verbose: tqdm.write(even_print(f'write {scalar}',f'{data_gb:0.3f} [GB]  {t_delta:0.3f} [s]  {data_gb/t_delta:0.3f} [GB/s]', s=True))
                
                if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: even_print( fn_spd, f'{os.path.getsize(fn_spd)/1024**3:0.2f} [GB]' )
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.export_polydata_ypln_wall() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # === time integration / particle tracking : generate LPD
    
    def time_integrate(self, **kwargs):
        '''
        do Lagrangian-frame time integration of [u,v,w] field
        -----
        --> output LPD (Lagrangian Particle Data) file / lpd() instance
        '''
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'rgd.time_integrate()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        force    = kwargs.get('force',False)
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        fn_lpd   = kwargs.get('fn_lpd','pts.h5')
        scheme   = kwargs.get('scheme','RK4')
        npts     = kwargs.get('npts',1e4)
        ntc      = kwargs.get('ntc',None)
        
        if (ntc is not None):
            if not isinstance(ntc,int):
                raise ValueError('ntc should be of type int')
            if (ntc > self.nt):
                raise ValueError('more ts requested than exist')
        
        #rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        #rz = kwargs.get('rz',1)
        #rt = kwargs.get('rt',1)
        
        #if (rx*ry*rz != self.n_ranks):
        #    raise AssertionError('rx*ry*rz != self.n_ranks')
        if (ry != self.n_ranks):
            raise AssertionError('ry != self.n_ranks')
        #if (rx>self.nx):
        #    raise AssertionError('rx>self.nx')
        #if (ry>self.ny):
        #    raise AssertionError('ry>self.ny')
        #if (rz>self.nz):
        #    raise AssertionError('rz>self.nz')
        
        #comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
        #t4d = comm4d.Get_coords(self.rank)
        
        # === the 'standard' non-abutting / non-overlapping index split
        
        #rxl_ = np.array_split(np.array(range(self.nx),dtype=np.int64),min(rx,self.nx))
        ryl_ = np.array_split(np.array(range(self.ny),dtype=np.int64),min(ry,self.ny))
        #rzl_ = np.array_split(np.array(range(self.nz),dtype=np.int64),min(rz,self.nz))
        #rtl_ = np.array_split(np.array(range(self.nt),dtype=np.int64),min(rt,self.nt))
        
        #rxl = [[b[0],b[-1]+1] for b in rxl_ ]
        ryl = [[b[0],b[-1]+1] for b in ryl_ ]
        #rzl = [[b[0],b[-1]+1] for b in rzl_ ]
        #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
        
        #rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
        #ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
        #rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
        #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
        
        ry1,ry2 = ryl[self.rank]; nyr = ry2 - ry1
        
        # === here we actually want to overlap points i.e. share abutting pts
        
        if (self.rank!=(self.n_ranks-1)):
            ry2 += 1
            nyr += 1
        
        ## these should be abutting
        y_max = self.y[ry1:ry2].max()
        y_min = self.y[ry1:ry2].min()
        
        ## overlap in middle ranks
        if (self.rank==0):
            ry1 -= 0
            ry2 += 6
        elif (self.rank==self.n_ranks-1):
            ry1 -= 6
            ry2 += 0
        else:
            ry1 -= 6
            ry2 += 6
        
        nyr = ry2 - ry1
        
        ## these should be overlapping / intersecting
        y_max_ov = self.y[ry1:ry2].max()
        y_min_ov = self.y[ry1:ry2].min()
        
        ## check rank / grid distribution
        if False:
            for ri in range(self.n_ranks):
                self.comm.Barrier()
                if (self.rank == ri):
                    print('rank %04d : ry1=%i ry2=%i y_min=%0.8f y_max=%0.8f y_min_ov=%0.8f y_max_ov=%0.8f'%(self.rank,ry1,ry2,y_min,y_max,y_min_ov,y_max_ov))
                    sys.stdout.flush()
            self.comm.Barrier()
        
        t_read = 0.
        t_write = 0.
        data_gb_read = 0.
        data_gb_write = 0.
        
        # === info about the domain
        
        read_dat_mean_dim = False
        
        if read_dat_mean_dim:
            
            fn_dat_mean_dim = None
            
            if (fn_dat_mean_dim is None):
                fname_path = os.path.dirname(self.fname)
                fname_base = os.path.basename(self.fname)
                fname_root, fname_ext = os.path.splitext(fname_base)
                fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
                fname_dat_mean_base = fname_root+'_mean_dim.dat'
                fn_dat_mean_dim = str(PurePosixPath(fname_path, fname_dat_mean_base))
            
            if not os.path.isfile(fn_dat_mean_dim):
                raise FileNotFoundError('%s not found!'%fn_dat_mean_dim)
        
        if verbose: even_print('fn_rgd'              , self.fname      )
        if read_dat_mean_dim:
            if verbose: even_print('fn_dat_mean_dim' , fn_dat_mean_dim )
        if verbose: even_print('fn_lpd'              , fn_lpd          )
        if verbose: even_print('n ts rgd'            , '%i'%self.nt    )
        
        if read_dat_mean_dim: ## mean dimensional data [x,z]
            
            # === read in data (mean dim) --> every rank gets full [x,z]
            with open(fn_dat_mean_dim,'rb') as f:
                data_mean_dim = pickle.load(f)
            fmd = type('foo', (object,), data_mean_dim)
            
            # === 2D dimensional quantities --> [x,z]
            u_tau    = fmd.u_tau
            nu_wall  = fmd.nu_wall
            d99      = fmd.d99
            u99      = fmd.u99
            Re_tau   = fmd.Re_tau
            Re_theta = fmd.Re_theta
            
            u_tau_avg_end    = np.mean(fmd.u_tau[-1,:]    , axis=(0,))
            nu_wall_avg_end  = np.mean(fmd.nu_wall[-1,:]  , axis=(0,))
            d99_avg_end      = np.mean(fmd.d99[-1,:]      , axis=(0,))
            u99_avg_end      = np.mean(fmd.u99[-1,:]      , axis=(0,))
            Re_tau_avg_end   = np.mean(fmd.Re_tau[-1,:]   , axis=(0,))
            Re_theta_avg_end = np.mean(fmd.Re_theta[-1,:] , axis=(0,))
            
            u_tau_avg_begin    = np.mean(fmd.u_tau[0,:]    , axis=(0,))
            nu_wall_avg_begin  = np.mean(fmd.nu_wall[0,:]  , axis=(0,))
            d99_avg_begin      = np.mean(fmd.d99[0,:]      , axis=(0,))
            u99_avg_begin      = np.mean(fmd.u99[0,:]      , axis=(0,))
            Re_tau_avg_begin   = np.mean(fmd.Re_tau[0,:]   , axis=(0,))
            Re_theta_avg_begin = np.mean(fmd.Re_theta[0,:] , axis=(0,))
            
            # === 2D inner scales --> [x,z]
            sc_l_in = nu_wall / u_tau
            sc_u_in = u_tau
            sc_t_in = nu_wall / u_tau**2
            
            # === 2D outer scales --> [x,z]
            sc_l_out = d99
            sc_u_out = u99
            sc_t_out = d99/u99
            
            # === check
            np.testing.assert_allclose(fmd.lchar   , self.lchar   , rtol=1e-8)
            np.testing.assert_allclose(fmd.U_inf   , self.U_inf   , rtol=1e-8)
            np.testing.assert_allclose(fmd.rho_inf , self.rho_inf , rtol=1e-8)
            np.testing.assert_allclose(fmd.T_inf   , self.T_inf   , rtol=1e-8)
            np.testing.assert_allclose(fmd.nx      , self.nx      , rtol=1e-8)
            np.testing.assert_allclose(fmd.ny      , self.ny      , rtol=1e-8)
            np.testing.assert_allclose(fmd.nz      , self.nz      , rtol=1e-8)
            np.testing.assert_allclose(fmd.xs      , self.x       , rtol=1e-8)
            np.testing.assert_allclose(fmd.ys      , self.y       , rtol=1e-8)
            np.testing.assert_allclose(fmd.zs      , self.z       , rtol=1e-8)
            
            lchar   = self.lchar
            U_inf   = self.U_inf
            rho_inf = self.rho_inf
            T_inf   = self.T_inf
            
            nx = self.nx
            ny = self.ny
            nz = self.nz
            nt = self.nt
            
            ## dimless (inlet)
            xd = self.x
            yd = self.y
            zd = self.z
            td = self.t
            
            ## dimensional [m] / [s]
            x = self.x * lchar 
            y = self.y * lchar
            z = self.z * lchar
            t = self.t * (lchar/U_inf)
            
            t_meas = t[-1]-t[0]
            dt     = self.dt * (lchar/U_inf)
            
            np.testing.assert_equal(nx,x.size)
            np.testing.assert_equal(ny,y.size)
            np.testing.assert_equal(nz,z.size)
            np.testing.assert_equal(nt,t.size)
            np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-8)
            
            # === report
            if verbose:
                even_print('nx'     , '%i'        %nx     )
                even_print('ny'     , '%i'        %ny     )
                even_print('nz'     , '%i'        %nz     )
                even_print('nt'     , '%i'        %nt     )
                even_print('dt'     , '%0.5e [s]' %dt     )
                even_print('t_meas' , '%0.5e [s]' %t_meas )
                even_print('domain x' , '%0.5e [m]' % (x.max()-x.min()) )
                ##
                even_print('U_inf'  , '%0.3f [m/s]'  % U_inf        )
                even_print('U_inf·t_meas/(domain x)' , '%0.3f' % (U_inf*t_meas/(x.max()-x.min())) )
                print(72*'-')
            
            if verbose:
                print('begin x'+'\n'+'-------')
                even_print('Re_τ'   , '%0.1f'        % Re_tau_avg_begin   )
                even_print('Re_θ'   , '%0.1f'        % Re_theta_avg_begin )
                even_print('δ99'    , '%0.5e [m]'    % d99_avg_begin      )
                even_print('u_τ'    , '%0.3f [m/s]'  % u_tau_avg_begin    )
                even_print('ν_wall' , '%0.5e [m²/s]' % nu_wall_avg_begin  )
                even_print('dt+'    , '%0.4f'        % (dt/(nu_wall_avg_begin / u_tau_avg_begin**2)) )
                
                t_eddy_begin = t_meas / (d99_avg_begin/u_tau_avg_begin)
                
                even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy_begin)
                even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99_avg_begin/u99_avg_begin)))
                even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99_avg_begin/u99_avg_begin)))
                ##
                print('end x'+'\n'+'-----')
                even_print('Re_τ'   , '%0.1f'        % Re_tau_avg_end   )
                even_print('Re_θ'   , '%0.1f'        % Re_theta_avg_end )
                even_print('δ99'    , '%0.5e [m]'    % d99_avg_end      )
                even_print('u_τ'    , '%0.3f [m/s]'  % u_tau_avg_end    )
                even_print('ν_wall' , '%0.5e [m²/s]' % nu_wall_avg_end  )
                even_print('dt+'    , '%0.4f'        % (dt/(nu_wall_avg_end / u_tau_avg_end**2)) )
                
                t_eddy_end = t_meas / (d99_avg_end/u_tau_avg_end)
                
                even_print('t_meas/(δ99/u_τ) = t_eddy' , '%0.2f'%t_eddy_end)
                even_print('t_meas/(δ99/u99)'          , '%0.2f'%(t_meas/(d99_avg_end/u99_avg_end)))
                even_print('t_meas/(20·δ99/u99)'       , '%0.2f'%(t_meas/(20*d99_avg_end/u99_avg_end)))
                print(72*'-')
        
        # === make initial particle lists
        
        if (ntc is None):
            tc  = np.copy(self.t).astype(np.float64)
            ntc = tc.size
            dtc = tc[1]-tc[0]
        else:
            tc  = np.copy(self.t[:ntc]).astype(np.float64)
            ntc = tc.size
            dtc = tc[1]-tc[0]
        
        if verbose: even_print('n ts for convection' , '%i'%ntc )
        if verbose: print(72*'-')
        
        if False: ## initial points @ grid points
            
            xc = np.copy(self.x)[::5]
            yc = np.copy(self.y[ry1:ry2]) ## [::3]
            zc = np.copy(self.z)[::5]
            
            xxxp, yyyp, zzzp = np.meshgrid(xc, yc, zc, indexing='ij')
            xp = xxxp.ravel(order='F')
            yp = yyyp.ravel(order='F')
            zp = zzzp.ravel(order='F')
            npts = xp.shape[0] ## this rank
            
            ## 4D
            tp = tc[0]*np.ones((npts,), dtype=xp.dtype)
            xyztp = np.stack((xp,yp,zp,tp)).T
            
            ## ## 3D
            ## xyzp = np.stack((xp,yp,zp)).T
            
            ## get total number of particles all ranks
            G = self.comm.gather([npts, self.rank], root=0)
            G = self.comm.bcast(G, root=0)
            npts_total = sum([x[0] for x in G])
            
            ## get the particle ID array for this rank (pnum)
            rp1  = sum([ G[i][0] for i in range(len(G)) if (i<self.rank) ])
            rp2  = rp1 + npts
            pnum = np.arange(rp1,rp2, dtype=np.int64)
            
            ## check particles per rank
            if False:
                for ri in range(self.n_ranks):
                    self.comm.Barrier()
                    if (self.rank == ri):
                        print('rank %04d : %s'%(self.rank,str(pnum)))
                self.comm.Barrier()
            
            if verbose: even_print('n pts','%i'%npts)
            if verbose: even_print('n pts (total)','%i'%npts_total)
            
            npts_total_orig = npts_total
        
        else: ## initial points @ random locations
            
            ## random number generator
            rng = np.random.default_rng(seed=1)
            
            ## pts already in domain at t=0
            if True:
                
                npts_init     = int(round(npts))
                volume_domain = (self.x.max()-self.x.min()) * (self.y.max()-self.y.min()) * (self.z.max()-self.z.min())
                pts_density   = npts_init / volume_domain ## [n pts / m^3]
                area_inlet    = (self.y.max()-self.y.min()) * (self.z.max()-self.z.min())
                volume_flow   = area_inlet * 1 ## still dimless, so U_inf=1 ## [m^2]*[m/s] = [m^3 / s]
                fac           = 0.9500000 ## approximates U(y)dy integral
                volume_flow  *= fac ## since U(y)!=1 across BL
                volume_per_dt = dtc * volume_flow ## [m^3]
                
                pts_per_dt_inlet          = int(round(pts_density*volume_per_dt))
                pts_per_dt_inlet_arr      = pts_per_dt_inlet * np.ones((ntc-1,), dtype=np.int32)
                pts_per_dt_inlet_arr[-5:] = 0 ## dont add any pts in the last N ts
                
                npts_init -= pts_per_dt_inlet ## added back at beginning of loop
                
                if verbose: even_print('volume_domain','%0.5e'%volume_domain)
                if verbose: even_print('pts_density','%0.5e'%pts_density)
                if verbose: even_print('area_inlet','%0.5e'%area_inlet)
                if verbose: even_print('pts_per_dt_inlet','%i'%pts_per_dt_inlet)
                if verbose: print(72*'-')
                
                xp = rng.uniform(self.x.min(), self.x.max(), size=(npts_init,)) ## double precision
                yp = rng.uniform(self.y.min(), self.y.max(), size=(npts_init,))
                zp = rng.uniform(self.z.min(), self.z.max(), size=(npts_init,))
                tp = tc[0]*np.ones((npts_init,), dtype=xp.dtype)
                
                xyztp = np.stack((xp,yp,zp,tp)).T
                
                ## check
                if (xyztp.dtype!=np.float64):
                    raise AssertionError
                
                pnum = np.arange(npts_init, dtype=np.int64)
                
                offset = npts_init
                
                if (self.rank==0):
                    ii = np.where(xyztp[:,1]<=y_max)
                elif (self.rank==self.n_ranks-1):
                    ii = np.where(xyztp[:,1]>y_min)
                else:
                    ii = np.where((xyztp[:,1]>y_min) & (xyztp[:,1]<=y_max))
                
                xyztp = np.copy(xyztp[ii])
                pnum  = np.copy(pnum[ii])
                
                npts_total = npts_init
            
            else: ## no initial particles
                
                xyztp      = None
                pnum       = None
                offset     = 0
                npts_total = 0
                pts_per_dt_inlet = int(10e3)
            
            ## function to replenish pts
            def pts_initializer(rng, npts, tt, offset):
                '''
                the 'new particle' replenishment func
                '''
                zp = rng.uniform(self.z.min(), self.z.max(), size=(npts,))
                yp = rng.uniform(self.y.min(), self.y.max(), size=(npts,))
                xp = 0.5*(self.x[0]+self.x[1]) * np.ones((npts,), dtype=zp.dtype)
                tp = tt * np.ones((npts,), dtype=zp.dtype)
                ##
                xyztp  = np.stack((xp,yp,zp,tp)).T
                
                ## check
                if (xyztp.dtype!=np.float64):
                    raise AssertionError
                
                pnum = int(offset) + np.arange(npts, dtype=np.int64)
                ##
                return xyztp, pnum
        
        ## get the total number of points that will exist for all times
        npts_all_ts = npts_total
        for tci in range(ntc-1):
            xyztp_, pnum_ = pts_initializer(rng, pts_per_dt_inlet_arr[tci], tc[tci], 0)
            npts,_ = xyztp_.shape
            npts_all_ts += npts
        
        # === check if file exists / delete / touch / chmod
        
        ## self.comm.Barrier()
        ## if (self.rank==0):
        ##     if os.path.isfile(fn_lpd):
        ##         os.remove(fn_lpd)
        ##     Path(fn_lpd).touch()
        ##     os.chmod(fn_lpd, int('770', base=8))
        ##     if shutil.which('lfs') is not None:
        ##         return_code = subprocess.call('lfs migrate --stripe-count 16 --stripe-size 8M %s > /dev/null 2>&1'%('particles.h5',), shell=True)
        ## self.comm.Barrier()
        
        # ===
        
        pcomm = MPI.COMM_WORLD
        with lpd(fn_lpd, 'w', force=force, driver='mpio', comm=pcomm, libver='latest') as hf_lpd:
            
            ## 'self' passed here is RGD / h5py.File instance
            ## this copies over all the header info from the RGD: U_inf, lchar, etc
            hf_lpd.init_from_rgd(self, t_info=False)
            
            ## shape & HDF5 chunk scheme for datasets
            shape = (npts_all_ts, ntc-1)
            
            scalars = [ 'x','y','z', 
                        'u','v','w', 
                        't','id'     ]
            
            scalars_dtype = [ np.float64, np.float64, np.float64, 
                              np.float32, np.float32, np.float32, 
                              np.float64, np.int64                 ]
            
            for si in range(len(scalars)):
                
                scalar       = scalars[si]
                scalar_dtype = scalars_dtype[si]
                
                itemsize = np.dtype(scalar_dtype).itemsize
                
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,1), size_kb=chunk_kb, base=2, itemsize=itemsize)
                
                if verbose:
                    even_print('initializing',scalar)
                
                if ('data/%s'%scalar in hf_lpd):
                    del hf_lpd['data/%s'%scalar]
                dset = hf_lpd.create_dataset('data/%s'%scalar, 
                                          shape=shape, 
                                          dtype=scalar_dtype,
                                          fillvalue=np.nan,
                                          #compression='gzip', ## this causes segfaults :( :(
                                          #compression_opts=5,
                                          #shuffle=True,
                                          chunks=chunks,
                                          )
                
                chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (pts,nt)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
            
            ## write time vector
            if ('dims/t' in hf_lpd):
                del hf_lpd['dims/t']
            hf_lpd.create_dataset('dims/t', data=tc[:-1], dtype=tc.dtype, chunks=True)
            
            pcomm.Barrier()
            self.comm.Barrier()
            
            if verbose:
                print(72*'-')
                even_print( os.path.basename(fn_lpd) , '%0.2f [GB]'%(os.path.getsize(fn_lpd)/1024**3))
                print(72*'-')
            
            if True: ## convect fwd
                
                if verbose:
                    progress_bar = tqdm(total=ntc-1, ncols=100, desc='convect fwd', leave=False, file=sys.stdout)
                
                t_write = 0.
                
                if (dtc!=self.dt):
                    raise AssertionError('dtc!=self.dt')
                
                ## the global list of all particle IDs that have left volume
                pnum_nan_global = np.array([],dtype=np.int64)
                
                for tci in range(ntc-1):
                    
                    if verbose:
                        if (tci>0):
                            tqdm.write('---')
                    
                    ## get global new pts this ts
                    xyztp_new, pnum_new = pts_initializer(rng, pts_per_dt_inlet_arr[tci], tc[tci], offset)
                    npts_new, _ = xyztp_new.shape
                    offset += npts_new
                    
                    ## take pts in rank bounds
                    if (self.rank==0):
                        ii = np.where(xyztp_new[:,1]<=y_max)
                    elif (self.rank==self.n_ranks-1):
                        ii = np.where(xyztp_new[:,1]>y_min)
                    else:
                        ii = np.where((xyztp_new[:,1]>y_min) & (xyztp_new[:,1]<=y_max))
                    ##
                    xyztp = np.concatenate((xyztp, xyztp_new[ii]), axis=0, casting='no')
                    pnum  = np.concatenate((pnum,  pnum_new[ii]),  axis=0, casting='no')
                    
                    if verbose: tqdm.write(even_print('tci', '%i'%(tci,), s=True))
                    
                    # ===
                    
                    ti1  = tci
                    ti2  = tci+1+1
                    tseg = tc[ti1:ti2]
                    
                    ## RK4 times
                    tbegin = tseg[0]
                    tend   = tseg[-1]
                    tmid   = 0.5*(tbegin+tend)
                    
                    ## update pts list time
                    xyztp[:,3] = tc[tci]
                    
                    ## assert : tc[tci] == segment begin time
                    if not np.allclose(tc[tci], tbegin, rtol=1e-8):
                        raise AssertionError('tc[tci] != tbegin')
                    
                    # === read RGD rectilinear data
                    
                    scalars = ['u','v','w']
                    scalars_dtypes = [np.float32 for s in scalars]
                    data = np.zeros(shape=(self.nx, nyr, self.nz, 2), dtype={'names':scalars, 'formats':scalars_dtypes})
                    
                    for scalar in scalars:
                        
                        data_gb = 4*self.nx*self.ny*self.nz*2 / 1024**3
                        
                        dset = self['data/%s'%scalar]
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            data[scalar] = dset[ti1:ti2,:,ry1:ry2,:].T
                        
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        
                        if verbose:
                            tqdm.write(even_print('read: %s'%scalar, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)) 
                        pass
                    
                    # ===
                    
                    ## ## make 4D scalar interpolators
                    ## f_u = sp.interpolate.RegularGridInterpolator((self.x,self.y[ry1:ry2],self.z,self.t[ti1:ti2]), data['u'], method='linear', fill_value=np.nan, bounds_error=False)
                    ## f_v = sp.interpolate.RegularGridInterpolator((self.x,self.y[ry1:ry2],self.z,self.t[ti1:ti2]), data['v'], method='linear', fill_value=np.nan, bounds_error=False)
                    ## f_w = sp.interpolate.RegularGridInterpolator((self.x,self.y[ry1:ry2],self.z,self.t[ti1:ti2]), data['w'], method='linear', fill_value=np.nan, bounds_error=False)
                    
                    uvw   = np.stack((data['u'], data['v'], data['w']), axis=4)
                    f_uvw = sp.interpolate.RegularGridInterpolator((self.x,self.y[ry1:ry2],self.z,self.t[ti1:ti2]), uvw, method='linear', fill_value=np.nan, bounds_error=False)
                    
                    # === Trilinear interpolation at beginning of time segment (1/2) --> this may have NaNs
                    
                    xyztp_k1 = np.copy(xyztp)
                    ## u_k1     = f_u(xyztp_k1)
                    ## v_k1     = f_v(xyztp_k1)
                    ## w_k1     = f_w(xyztp_k1)
                    
                    uvw_k1 = f_uvw(xyztp_k1)
                    u_k1 = uvw_k1[:,0]
                    
                    x_ = np.copy( xyztp[:,0] )
                    y_ = np.copy( xyztp[:,1] )
                    z_ = np.copy( xyztp[:,2] )
                    t_ = np.copy( xyztp[:,3] )
                    
                    # === separate out NaN positions / velocities
                    
                    ii_nan    = np.where(  np.isnan(x_) |  np.isnan(u_k1) )
                    ii_notnan = np.where( ~np.isnan(x_) & ~np.isnan(u_k1) )
                    ##
                    pnum_nan  = np.copy(pnum[ii_nan])
                    xyztp_nan = np.copy(xyztp[ii_nan])
                    ##
                    G = self.comm.gather([np.copy(pnum_nan), self.rank], root=0)
                    G = self.comm.bcast(G, root=0)
                    pnum_nan_global_this_ts = np.concatenate( [g[0] for g in G] , casting='no' )
                    pnum_nan_global         = np.concatenate( (pnum_nan_global_this_ts , pnum_nan_global) , casting='no' )
                    ##
                    npts_nan = pnum_nan_global.shape[0]
                    
                    ## take only non-NaN position particles
                    pnum  = np.copy(pnum[ii_notnan])
                    xyztp = np.copy(xyztp[ii_notnan])
                    
                    if True: ## check global pnum (pt id) vector
                        
                        G = self.comm.gather([np.copy(pnum), self.rank], root=0)
                        G = self.comm.bcast(G, root=0)
                        pnum_global = np.sort( np.concatenate( [g[0] for g in G] , casting='no' ) )
                        pnum_global = np.sort( np.concatenate((pnum_global,pnum_nan_global), casting='no') )
                        
                        ## make sure that the union of the current (non-nan) IDs and nan IDs is 
                        ##    equal to the arange of the total number of particles instantiated to
                        ##    this point
                        if not np.array_equal(pnum_global, np.arange(offset, dtype=np.int64)):
                            raise AssertionError('pnum_global!=np.arange(offset, dtype=np.int64)')
                        
                        if verbose: tqdm.write(even_print('pnum check', 'passed', s=True)) 
                    
                    # === Trilinear interpolation at beginning of time segment (2/2) --> again after NaN filter
                    
                    xyztp_k1 = np.copy(xyztp)
                    ## u_k1     = f_u(xyztp_k1)
                    ## v_k1     = f_v(xyztp_k1)
                    ## w_k1     = f_w(xyztp_k1)
                    uvw_k1   = f_uvw(xyztp_k1)
                    u_k1     = uvw_k1[:,0]
                    v_k1     = uvw_k1[:,1]
                    w_k1     = uvw_k1[:,2]
                    
                    x_ = np.copy( xyztp[:,0] )
                    y_ = np.copy( xyztp[:,1] )
                    z_ = np.copy( xyztp[:,2] )
                    t_ = np.copy( xyztp[:,3] )
                    
                    ## this passes
                    if False:
                        if np.isnan(np.min(u_k1)):
                            print('u_k1 has NaNs')
                            pcomm.Abort(1)
                        if np.isnan(np.min(v_k1)):
                            print('v_k1 has NaNs')
                            pcomm.Abort(1)
                        if np.isnan(np.min(w_k1)):
                            print('w_k1 has NaNs')
                            pcomm.Abort(1)
                    
                    # === Gather/Bcast all data
                    
                    pcomm.Barrier()
                    t_start = timeit.default_timer()
                    
                    G = self.comm.gather([self.rank, pnum, x_, y_, z_, t_, u_k1, v_k1, w_k1], root=0)
                    G = self.comm.bcast(G, root=0)
                    
                    pcomm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    if verbose:
                        tqdm.write(even_print('MPI Gather/Bcast', '%0.2f [s]'%(t_delta,), s=True))
                    
                    npts_total = sum([g[1].shape[0] for g in G])
                    pnum_gl    = np.concatenate([g[1] for g in G], axis=0, casting='no',        dtype=np.int64   )
                    x_gl       = np.concatenate([g[2] for g in G], axis=0, casting='no',        dtype=np.float64 )
                    y_gl       = np.concatenate([g[3] for g in G], axis=0, casting='no',        dtype=np.float64 )
                    z_gl       = np.concatenate([g[4] for g in G], axis=0, casting='no',        dtype=np.float64 )
                    t_gl       = np.concatenate([g[5] for g in G], axis=0, casting='no',        dtype=np.float64 )
                    u_gl       = np.concatenate([g[6] for g in G], axis=0, casting='same_kind', dtype=np.float32 )
                    v_gl       = np.concatenate([g[7] for g in G], axis=0, casting='same_kind', dtype=np.float32 )
                    w_gl       = np.concatenate([g[8] for g in G], axis=0, casting='same_kind', dtype=np.float32 )
                    ##
                    if verbose: tqdm.write(even_print('n pts initialized', '%i'%(offset,),      s=True))
                    if verbose: tqdm.write(even_print('n pts in domain',   '%i'%(npts_total,),  s=True))
                    if verbose: tqdm.write(even_print('n pts left domain', '%i'%(npts_nan,),    s=True))
                    if verbose: tqdm.write(even_print('n pts all time',    '%i'%(npts_all_ts,), s=True))
                    
                    # === add NaN IDs, pad scalar vectors, do sort
                    
                    pnum_gl = np.concatenate([pnum_gl,pnum_nan_global], axis=0, casting='no', dtype=np.int64 )
                    npts_total_incl_nan = pnum_gl.shape[0]
                    
                    if (npts_total_incl_nan!=offset):
                        raise AssertionError('npts_total_incl_nan!=offset')
                    
                    nanpad_f32 = np.empty( (npts_nan,), dtype=np.float32 ); nanpad_f32[:] = np.nan
                    nanpad_f64 = np.empty( (npts_nan,), dtype=np.float64 ); nanpad_f64[:] = np.nan
                    #nanpad_i64 = np.empty( (npts_nan,), dtype=np.int64   ); nanpad_i64[:] = np.nan
                    
                    x_gl = np.concatenate( [x_gl,nanpad_f64], axis=0, dtype=np.float64, casting='no' )
                    y_gl = np.concatenate( [y_gl,nanpad_f64], axis=0, dtype=np.float64, casting='no' )
                    z_gl = np.concatenate( [z_gl,nanpad_f64], axis=0, dtype=np.float64, casting='no' )
                    t_gl = np.concatenate( [t_gl,nanpad_f64], axis=0, dtype=np.float64, casting='no' )
                    u_gl = np.concatenate( [u_gl,nanpad_f32], axis=0, dtype=np.float32, casting='no' )
                    v_gl = np.concatenate( [v_gl,nanpad_f32], axis=0, dtype=np.float32, casting='no' )
                    w_gl = np.concatenate( [w_gl,nanpad_f32], axis=0, dtype=np.float32, casting='no' )
                    ##
                    sort_order = np.argsort(pnum_gl, axis=0)
                    pnum_gl    = np.copy(pnum_gl[sort_order])
                    x_gl       = np.copy(x_gl[sort_order])
                    y_gl       = np.copy(y_gl[sort_order])
                    z_gl       = np.copy(z_gl[sort_order])
                    t_gl       = np.copy(t_gl[sort_order])
                    u_gl       = np.copy(u_gl[sort_order])
                    v_gl       = np.copy(v_gl[sort_order])
                    w_gl       = np.copy(w_gl[sort_order])
                    
                    ## check that the global particle number / ID vector is simply arange(total an pts created)
                    if not np.array_equal(pnum_gl, np.arange(offset, dtype=np.int64)):
                        raise AssertionError('pnum_gl!=np.arange(offset, dtype=np.int64)')
                    
                    ## check that all particle times for this ts are equal --> passes
                    if False:
                        ii_notnan = np.where(~np.isnan(t_gl))
                        if not np.all( np.isclose(t_gl[ii_notnan], t_gl[ii_notnan][0], rtol=1e-14) ):
                            raise AssertionError('not all times are the same at this time integration step --> check!!!')
                    
                    # === get collective write bounds
                    
                    rpl_ = np.array_split(np.arange(npts_total_incl_nan,dtype=np.int64) , self.n_ranks)
                    rpl = [[b[0],b[-1]+1] for b in rpl_ ]
                    rp1,rp2 = rpl[self.rank]
                    
                    # === write
                    
                    for key, value in {'id':pnum_gl, 'x':x_gl, 'y':y_gl, 'z':z_gl, 't':t_gl, 'u':u_gl, 'v':v_gl, 'w':w_gl}.items():
                        
                        data_gb = value.itemsize * npts_total_incl_nan / 1024**3
                        
                        dset = hf_lpd['data/%s'%key]
                        pcomm.Barrier()
                        t_start = timeit.default_timer()
                        with dset.collective:
                            dset[rp1:rp2,tci] = value[rp1:rp2]
                        pcomm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        t_write += t_delta
                        if verbose:
                            tqdm.write(even_print('write: %s'%key, '%0.2f [GB]  %0.2f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True)) 
                    
                    # ===
                    
                    ## ## write the IDs of the particles which have left the domain for sort in a second step
                    ## ## --> this is now done in the step above
                    ## if (npts_nan>0):
                    ##     dset = hf_lpd['data/id']
                    ##     with dset.collective:
                    ##         dset[-npts_nan:,tci] = pnum_nan_global.astype(np.float32)
                    
                    # === Time Integration
                    
                    pcomm.Barrier()
                    t_start = timeit.default_timer()
                    
                    if (scheme=='RK4'):
                        
                        xyztp_    = np.copy(xyztp)
                        xyztp_k2  = np.zeros_like(xyztp)
                        xyztp_k3  = np.zeros_like(xyztp)
                        xyztp_k4  = np.zeros_like(xyztp)
                        
                        xyztp_k2[:,0] = xyztp_[:,0] + u_k1*dtc*0.5
                        xyztp_k2[:,1] = xyztp_[:,1] + v_k1*dtc*0.5
                        xyztp_k2[:,2] = xyztp_[:,2] + w_k1*dtc*0.5
                        xyztp_k2[:,3] = tmid
                        ## u_k2          = f_u(xyztp_k2)
                        ## v_k2          = f_v(xyztp_k2)
                        ## w_k2          = f_w(xyztp_k2)
                        uvw_k2 = f_uvw(xyztp_k2)
                        u_k2   = uvw_k2[:,0]
                        v_k2   = uvw_k2[:,1]
                        w_k2   = uvw_k2[:,2]
                        
                        xyztp_k3[:,0] = xyztp_[:,0] + u_k2*dtc*0.5
                        xyztp_k3[:,1] = xyztp_[:,1] + v_k2*dtc*0.5
                        xyztp_k3[:,2] = xyztp_[:,2] + w_k2*dtc*0.5
                        xyztp_k3[:,3] = tmid
                        ## u_k3          = f_u(xyztp_k3)
                        ## v_k3          = f_v(xyztp_k3)
                        ## w_k3          = f_w(xyztp_k3)
                        uvw_k3 = f_uvw(xyztp_k3)
                        u_k3   = uvw_k3[:,0]
                        v_k3   = uvw_k3[:,1]
                        w_k3   = uvw_k3[:,2]
                        
                        xyztp_k4[:,0] = xyztp_[:,0] + u_k3*dtc*1.0
                        xyztp_k4[:,1] = xyztp_[:,1] + v_k3*dtc*1.0
                        xyztp_k4[:,2] = xyztp_[:,2] + w_k3*dtc*1.0
                        xyztp_k4[:,3] = tend
                        ## u_k4          = f_u(xyztp_k4)
                        ## v_k4          = f_v(xyztp_k4)
                        ## w_k4          = f_w(xyztp_k4)
                        uvw_k4 = f_uvw(xyztp_k4)
                        u_k4   = uvw_k4[:,0]
                        v_k4   = uvw_k4[:,1]
                        w_k4   = uvw_k4[:,2]
                        
                        ## the vel components (RK4 weighted avg) for time integration
                        u = (1./6.) * (1.*u_k1 + 2.*u_k2 + 2.*u_k3 + 1.*u_k4)
                        v = (1./6.) * (1.*v_k1 + 2.*v_k2 + 2.*v_k3 + 1.*v_k4)
                        w = (1./6.) * (1.*w_k1 + 2.*w_k2 + 2.*w_k3 + 1.*w_k4)
                    
                    elif (scheme=='Euler Explicit'):
                        
                        u = u_k1
                        v = v_k1
                        w = w_k1
                    
                    else:
                        raise NotImplementedError('integration scheme \'%s\' not valid. options are: \'Euler Explicit\', \'RK4\''%scheme)
                    
                    ## actual time integration with higher order 'average' [u,v,w] over time segment 
                    xyztp[:,0] = xyztp[:,0] + u*dtc
                    xyztp[:,1] = xyztp[:,1] + v*dtc
                    xyztp[:,2] = xyztp[:,2] + w*dtc
                    
                    pcomm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    t_write += t_delta
                    if verbose:
                        tqdm.write(even_print('%s'%scheme, '%0.2f [s]'%(t_delta,), s=True))
                    
                    # === MPI Gather/Bcast points between domains
                    
                    ## get pts that leave rank bounds in dim [y]
                    i_exit_top = np.where(xyztp[:,1]>y_max)
                    i_exit_bot = np.where(xyztp[:,1]<y_min)
                    
                    ## lists for the pts that leave
                    xyztp_out = np.concatenate((np.copy(xyztp[i_exit_top]) , np.copy(xyztp[i_exit_bot])), axis=0, casting='no' )
                    pnum_out  = np.concatenate((np.copy(pnum[i_exit_top])  , np.copy(pnum[i_exit_bot])),  axis=0, casting='no' )
                    
                    ## delete those from local lists
                    i_del = np.concatenate( (i_exit_top[0], i_exit_bot[0]) , axis=0, casting='no' )
                    xyztp = np.delete(xyztp, (i_del,), axis=0)
                    pnum  = np.delete(pnum,  (i_del,), axis=0)
                    
                    # === MPI : Gather/Bcast all inter-domain pts
                    G = self.comm.gather([np.copy(xyztp_out), np.copy(pnum_out), self.rank], root=0)
                    G = self.comm.bcast(G, root=0)
                    xyztpN = np.concatenate([x[0] for x in G], axis=0, casting='no' )
                    pnumN  = np.concatenate([x[1] for x in G], axis=0, casting='no' )
                    
                    ## get indices to 'take' from inter-domain points
                    if (self.rank==0):
                        i_take = np.where(xyztpN[:,1]<=y_max)
                    elif (self.rank==self.n_ranks-1):
                        i_take = np.where(xyztpN[:,1]>=y_min)
                    else:
                        i_take = np.where((xyztpN[:,1]<y_max) & (xyztpN[:,1]>=y_min))
                    ##
                    xyztp = np.concatenate((xyztp, xyztpN[i_take]), axis=0, casting='no' )
                    pnum  = np.concatenate((pnum,  pnumN[i_take]),  axis=0, casting='no' )
                    
                    if verbose:
                        progress_bar.update()
                    
                    self.comm.Barrier()
                    pcomm.Barrier()
                
                if verbose:
                    progress_bar.close()
        
        if verbose: print(72*'-'+'\n')
        
        pcomm.Barrier()
        self.comm.Barrier()
        
        ## make XDMF/XMF2
        with lpd(fn_lpd, 'r', driver='mpio', comm=pcomm) as hf_lpd:
            hf_lpd.make_xdmf()
        
        # ===
        
        if verbose: print('\n'+72*'-')
        if verbose: print('total time : rgd.time_integrate() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # === testing & assertion
    
    def check_grid_validity(self,):
        '''
        check grid validity for RGD file
        - monotonically increasing
        - no Δ=0 
        '''
        
        verbose = True
        
        if verbose: print('\n'+'rgd.check_grid_validity()'+'\n'+72*'-')
        
        x = np.copy(self.x)
        y = np.copy(self.y)
        z = np.copy(self.z)
        t = np.copy(self.t)
        
        dims = {'x':x,'y':y,'z':z,'t':t}
        
        for dn, d in dims.items():
            
            if (d.shape[0]>1):
                
                ## check no zero distance elements
                if (np.diff(d).size - np.count_nonzero(np.diff(d))) != 0.:
                    #raise AssertionError('%s arr has zero-distance elements'%dn)
                    if verbose: even_print('check: Δ%s!=0'%dn,'failed')
                else:
                    if verbose: even_print('check: Δ%s!=0'%dn,'passed')
                
                ## check monotonically increasing
                if not np.all(np.diff(d) > 0.):
                    #raise AssertionError('%s arr not monotonically increasing'%dn)
                    if verbose: even_print('check: %s mono increasing'%dn,'failed')
                else:
                    if verbose: even_print('check: %s mono increasing'%dn,'passed')
            
            else:
                if verbose: print('dim %s has size 1'%dn)
        
        if verbose: print(72*'-')
        
        return
    
    @staticmethod
    def assert_4d_datasets(fn_h5_A, fn_h5_B, **kwargs):
        '''
        assert that 4D datasets data/<scalar> are the same
        '''
        
        verbose = kwargs.get('verbose',True)
        
        try:
            #comm    = MPI.COMM_WORLD
            rank    = MPI.COMM_WORLD.Get_rank()
            n_ranks = MPI.COMM_WORLD.Get_size()
        except:
            rank = 0
            n_ranks = 1
        
        if (rank==0):
            verbose = True
        else:
            verbose = False
        
        with rgd(fn_h5_A,'r') as fA:
            with rgd(fn_h5_B,'r') as fB:
                
                if verbose: print('\n'+'rgd.assert_4d_datasets()'+'\n'+72*'-')
                t_start_func = timeit.default_timer()
                
                if (fA.fclass != fB.fclass):
                    raise ValueError('fclass not same')
                if verbose: even_print('fclass',str(fA.fclass))
                if (fA.nx != fB.nx):
                    raise ValueError('nx not same')
                if verbose: even_print('nx','%i'%fA.nx)
                if (fA.ny != fB.ny):
                    raise ValueError('ny not same')
                if verbose: even_print('ny','%i'%fA.ny)
                if (fA.nz != fB.nz):
                    raise ValueError('nz not same')
                if verbose: even_print('nz','%i'%fA.nz)
                if (fA.nt != fB.nt):
                    raise ValueError('nt not same')
                if verbose: even_print('nt','%i'%fA.nt)
                if verbose: print(72*'-')
                
                scalars_A = sorted( fA.scalars )
                scalars_B = sorted( fB.scalars )
                
                #if (scalars_A != scalars_B):
                #    raise ValueError('scalars not same')
                
                if (scalars_A == scalars_B):
                    if verbose: even_print('scalars same','True')
                    if verbose: even_print('n scalars','%i'%(len(scalars_A),))
                else:
                    if verbose: even_print('scalars same','False')
                    if verbose: even_print('n scalars A','%i'%(len(fA.scalars),))
                    if verbose: even_print('n scalars B','%i'%(len(fB.scalars),))
                if verbose: print(72*'-')
                
                for scalar in fA.scalars:
                    if scalar in fB.scalars:
                        
                        dsA = fA[f'data/{scalar}']
                        dsB = fB[f'data/{scalar}']
                        
                        for ti_ in range(fA.nt):
                            
                            tiA = fA.ti[ti_]
                            tiB = fB.ti[ti_]
                            dA = np.copy( dsA[tiA,:,:,:].T )
                            dB = np.copy( dsB[tiB,:,:,:].T )
                            
                            if (dA.dtype==np.float64) and (dB.dtype==np.float64):
                                tol = 1e-12
                            else:
                                tol = 1e-6
                            
                            ## this might fail if all 0s
                            np.testing.assert_allclose( np.abs(dA).max() , np.abs(dB).max(), rtol=tol )
                            
                            rdiff_abs_max = np.abs(dA-dB).max()
                            if verbose: even_print(f'max(abs(Δ{scalar}))',f'{rdiff_abs_max:0.3e}')
                            
                            np.testing.assert_allclose( rdiff_abs_max, 0., atol=tol )
        
        # self.comm.Barrier()
        
        #if verbose: print('\n'+72*'-')
        if verbose: print(72*'-')
        if verbose: print('total time : rgd.assert_4d_datasets() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from RGD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        makeVectors = kwargs.get('makeVectors',True) ## write vectors (e.g. velocity, vorticity) to XDMF
        makeTensors = kwargs.get('makeTensors',True) ## write 3x3 tensors (e.g. stress, strain) to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if verbose: print('\n'+'rgd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for scalar in ['x','y','z']:
            if ('dims/'+scalar in self):
                data = self['dims/'+scalar]
                dataset_precision_dict[scalar] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[scalar] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[scalar] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        # scalar names dict
        # --> labels for Paraview could be customized (e.g. units could be added) using a dict
        # --> the block below shows one such example dict, though it is currently inactive
        
        if False:
            units = 'dimless'
            if (units=='SI') or (units=='si'): ## m,s,kg,K
                scalar_names = {'x':'x [m]',
                                'y':'y [m]',
                                'z':'z [m]', 
                                'u':'u [m/s]',
                                'v':'v [m/s]',
                                'w':'w [m/s]', 
                                'T':'T [K]',
                                'rho':'rho [kg/m^3]',
                                'p':'p [Pa]'}
            elif (units=='dimless') or (units=='dimensionless'):
                scalar_names = {'x':'x [dimless]',
                                'y':'y [dimless]',
                                'z':'z [dimless]', 
                                'u':'u [dimless]',
                                'v':'v [dimless]',
                                'w':'w [dimless]',
                                'T':'T [dimless]',
                                'rho':'rho [dimless]',
                                'p':'p [dimless]'}
            else:
                raise ValueError('choice of units not recognized : %s --> options are : %s / %s'%(units,'SI','dimless'))
        else:
            scalar_names = {} ## dummy/empty 
        
        ## refresh header
        self.get_header(verbose=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')
        
        # === write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            if not os.path.isfile(fname_xdmf): ## if doesnt exist...
                Path(fname_xdmf).touch() ## touch XDMF file
                perms_h5 = oct(os.stat(self.fname).st_mode)[-3:] ## get permissions of RGD file
                os.chmod(fname_xdmf, int(perms_h5, base=8)) ## change permissions of XDMF file
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                ## Dimensions can also be NumberOfElements
                xdmf_str=f'''
                         <Topology TopologyType="3DRectMesh" NumberOfElements="{self.nz:d} {self.ny:d} {self.nx:d}"/>
                         <Geometry GeometryType="VxVyVz">
                           <DataItem Dimensions="{self.nx:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                             {fname_base}:/dims/{'x'}
                           </DataItem>
                           <DataItem Dimensions="{self.ny:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                             {fname_base}:/dims/{'y'}
                           </DataItem>
                           <DataItem Dimensions="{self.nz:d}" NumberType="{dataset_numbertype_dict['z']}" Precision="{dataset_precision_dict['z']:d}" Format="HDF">
                             {fname_base}:/dims/{'z'}
                           </DataItem>
                         </Geometry>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # =====
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===== .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        
                        dset_hf_path = 'data/%s'%scalar
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        if scalar in scalar_names:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        xdmf_str=f'''
                                 <!-- ===== scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                     <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                       {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                       {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                       {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                     </DataItem>
                                     <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                       {fname_base}:/{dset_hf_path}
                                     </DataItem>
                                   </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    if makeVectors:
                        
                        # === .xdmf : <Grid> per vector : velocity vector
                        
                        if ('u' in self.scalars) and ('v' in self.scalars) and ('w' in self.scalars):
                            
                            scalar_name    = 'velocity'
                            dset_hf_path_i = 'data/u'
                            dset_hf_path_j = 'data/v'
                            dset_hf_path_k = 'data/w'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['u']}" Precision="{dataset_precision_dict['u']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['v']}" Precision="{dataset_precision_dict['v']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['w']}" Precision="{dataset_precision_dict['w']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                        
                        # === .xdmf : <Grid> per vector : vorticity vector
                        
                        if ('vort_x' in self.scalars) and ('vort_y' in self.scalars) and ('vort_z' in self.scalars):
                            
                            scalar_name    = 'vorticity'
                            dset_hf_path_i = 'data/vort_x'
                            dset_hf_path_j = 'data/vort_y'
                            dset_hf_path_k = 'data/vort_z'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_x']}" Precision="{dataset_precision_dict['vort_x']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_y']}" Precision="{dataset_precision_dict['vort_y']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_z']}" Precision="{dataset_precision_dict['vort_z']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    if makeTensors:
                        if all([('dudx' in self.scalars),('dvdx' in self.scalars),('dwdx' in self.scalars),
                                ('dudy' in self.scalars),('dvdy' in self.scalars),('dwdy' in self.scalars),
                                ('dudz' in self.scalars),('dvdz' in self.scalars),('dwdz' in self.scalars)]):
                            pass
                            pass ## TODO
                            pass
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if self.usingmpi: self.comm.Barrier()
        if verbose: print('--w-> %s'%fname_xdmf_base)
        return

class eas4(h5py.File):
    '''
    Interface class for EAS4 files
    ------------------------------
    - super()'ed h5py.File class
    - EAS4 is the HDF5-based output format from the flow solver NS3D
    - 3D dataset storage ([x,y,z] per [t])
    '''
    
    def __init__(self, *args, **kwargs):
        
        ## if grid MUST be read as 3D, i.e. the GMODE=(5,5,5), only read if this is TRUE
        ## this can lead to huge RAM usage in MPI mode, so OFF by default
        ##
        ## if a single dimension has GMODE=5 but not ALL, i.e. (5,5,2), this allows for
        ## grid to be read as some combination of 2D&1D
        self.read_3d_grid = kwargs.pop('read_3d_grid', False)
        
        self.fname, openMode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        if (openMode!='r'):
            raise ValueError('turbx.eas4(): opening EAS4 in anything but read mode \'r\' is not allowed!')
        
        ## catch possible user error --> user tries to open non-EAS4 with turbx.eas4()
        if (self.fname_ext!='.eas'):
            raise ValueError('turbx.eas4() should not be used to open non-EAS4 files')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
            if ('comm' in kwargs):
                del kwargs['comm']
        
        ## set library version to latest (if not otherwise set)
        if ('libver' not in kwargs):
            kwargs['libver']='latest'
        
        ## determine MPI info / hints
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(8*1024**2))) ) ## 8 [MB]
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        self.domainName = 'DOMAIN_000000' ## turbx only handles one domain for now
        
        ## eas4() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop('verbose',False)
        self.verbose = verbose
        ## force = kwargs.pop('force',False) ## --> dont need, always read-only!
        
        ## call actual h5py.File.__init__()
        super(eas4, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(eas4, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed EAS4 HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(eas4, self).__exit__()
    
    def get_header(self, **kwargs):
        
        EAS4=1
        IEEES=1; IEEED=2
        EAS4_NO_G=1; EAS4_X0DX_G=2; EAS4_UDEF_G=3; EAS4_ALL_G=4; EAS4_FULL_G=5
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        # === characteristic values
        
        if self.verbose: print(72*'-')
        Ma    = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['Ma'][0]
        Re    = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['Re'][0]
        Pr    = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['Pr'][0]
        kappa = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['kappa'][0]
        R     = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['R'][0]
        p_inf = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['p_inf'][0]
        T_inf = self['Kennsatz']['FLOWFIELD_PROPERTIES'].attrs['T_inf'][0]
        
        ## !!! import what is called 'C_Suth' in NS3D as 'S_Suth' !!!
        S_Suth = self['Kennsatz']['VISCOUS_PROPERTIES'].attrs['C_Suth'][0]
        
        mu_Suth_ref = self['Kennsatz']['VISCOUS_PROPERTIES'].attrs['mu_Suth_ref'][0]
        T_Suth_ref  = self['Kennsatz']['VISCOUS_PROPERTIES'].attrs['T_Suth_ref'][0]
        
        C_Suth = mu_Suth_ref/(T_Suth_ref**(3/2))*(T_Suth_ref + S_Suth) ## [kg/(m·s·√K)]
        
        if self.verbose: even_print('Ma'          , '%0.2f [-]'           % Ma          )
        if self.verbose: even_print('Re'          , '%0.1f [-]'           % Re          )
        if self.verbose: even_print('Pr'          , '%0.3f [-]'           % Pr          )
        if self.verbose: even_print('T_inf'       , '%0.3f [K]'           % T_inf       )
        if self.verbose: even_print('p_inf'       , '%0.1f [Pa]'          % p_inf       )
        if self.verbose: even_print('kappa'       , '%0.3f [-]'           % kappa       )
        if self.verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % R           )
        if self.verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % mu_Suth_ref )
        if self.verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % T_Suth_ref  )
        if self.verbose: even_print('S_Suth'      , '%0.2f [K]'           % S_Suth      )
        if self.verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % C_Suth      ) ## actually derived
        
        # === characteristic values : derived
        
        # mu_inf_1 = 14.58e-7*T_inf**1.5/(T_inf+110.4)
        # mu_inf_2 = mu_Suth_ref*(T_inf/T_Suth_ref)**(3/2) * ((T_Suth_ref+S_Suth)/(T_inf+S_Suth))
        # mu_inf_3 = C_Suth*T_inf**(3/2)/(T_inf+S_Suth)
        # if not np.isclose(mu_inf_1, mu_inf_2, rtol=1e-14):
        #     raise AssertionError('inconsistency in Sutherland calc --> check')
        # if not np.isclose(mu_inf_2, mu_inf_3, rtol=1e-14):
        #     raise AssertionError('inconsistency in Sutherland calc --> check')
        # mu_inf    = mu_inf_3
        
        if self.verbose: print(72*'-')
        mu_inf    = mu_Suth_ref*(T_inf/T_Suth_ref)**(3/2) * ((T_Suth_ref+S_Suth)/(T_inf+S_Suth))
        rho_inf   = p_inf / ( R * T_inf )
        nu_inf    = mu_inf/rho_inf
        a_inf     = np.sqrt(kappa*R*T_inf)
        U_inf     = Ma*a_inf
        cp        = R*kappa/(kappa-1.)
        cv        = cp/kappa
        recov_fac = Pr**(1/3)
        Taw       = T_inf + recov_fac*U_inf**2/(2*cp)
        lchar     = Re * nu_inf / U_inf
        
        if self.verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % rho_inf   )
        if self.verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % mu_inf    )
        if self.verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % nu_inf    )
        if self.verbose: even_print('a_inf'           , '%0.6f [m/s]'      % a_inf     )
        if self.verbose: even_print('U_inf'           , '%0.6f [m/s]'      % U_inf     )
        if self.verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % cp        )
        if self.verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % cv        )
        if self.verbose: even_print('recovery factor' , '%0.6f [-]'        % recov_fac )
        if self.verbose: even_print('Taw'             , '%0.3f [K]'        % Taw       )
        if self.verbose: even_print('lchar'           , '%0.6E [m]'        % lchar     )
        if self.verbose: print(72*'-'+'\n')
        
        # ===
        
        self.Ma           = Ma
        self.Re           = Re
        self.Pr           = Pr
        self.kappa        = kappa
        self.R            = R
        self.p_inf        = p_inf
        self.T_inf        = T_inf
        self.C_Suth       = C_Suth
        self.S_Suth       = S_Suth
        self.mu_Suth_ref  = mu_Suth_ref
        self.T_Suth_ref   = T_Suth_ref
        
        self.rho_inf      = rho_inf
        self.mu_inf       = mu_inf
        self.nu_inf       = nu_inf
        self.a_inf        = a_inf
        self.U_inf        = U_inf
        self.cp           = cp
        self.cv           = cv
        self.recov_fac    = recov_fac
        self.Taw          = Taw
        self.lchar        = lchar

        # === check if this a 2D average file like 'mean_flow_mpi.eas'
        
        if self.verbose: print(72*'-')
        if ('/Kennsatz/AUXILIARY/AVERAGING' in self):
            self.total_avg_time       = self['/Kennsatz/AUXILIARY/AVERAGING'].attrs['total_avg_time'][0]
            self.total_avg_iter_count = self['/Kennsatz/AUXILIARY/AVERAGING'].attrs['total_avg_iter_count'][0]
            if self.verbose: even_print('total_avg_time', '%0.2f'%self.total_avg_time)
            if self.verbose: even_print('total_avg_iter_count', '%i'%self.total_avg_iter_count)
            self.measType = 'mean'
        else:
            self.measType = 'unsteady'
        if self.verbose: even_print('meas type', '\'%s\''%self.measType)
        if self.verbose: print(72*'-'+'\n')
        
        # === grid info
        
        ndim1 = int( self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_SIZE'][0] )
        ndim2 = int( self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_SIZE'][1] )
        ndim3 = int( self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_SIZE'][2] )
        
        nx = self.nx = ndim1
        ny = self.ny = ndim2
        nz = self.nz = ndim3
        
        if (self.measType=='mean'):
            nz = self.nz = 1
        
        ngp = self.ngp = nx*ny*nz
        
        if self.verbose: print('grid info\n'+72*'-')
        if self.verbose: even_print('nx',  '%i'%nx  )
        if self.verbose: even_print('ny',  '%i'%ny  )
        if self.verbose: even_print('nz',  '%i'%nz  )
        if self.verbose: even_print('ngp', '%i'%ngp )
        
        gmode_dim1 = self.gmode_dim1 = self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_GMODE'][0]
        gmode_dim2 = self.gmode_dim2 = self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_GMODE'][1]
        gmode_dim3 = self.gmode_dim3 = self['Kennsatz/GEOMETRY/%s'%self.domainName].attrs['DOMAIN_GMODE'][2]
        
        ## the original gmode (pre-conversion)
        gmode_dim1_orig = self.gmode_dim1_orig = gmode_dim1
        gmode_dim2_orig = self.gmode_dim2_orig = gmode_dim2
        gmode_dim3_orig = self.gmode_dim3_orig = gmode_dim3
        
        if self.verbose: even_print( 'gmode dim1' , '%i / %s'%(gmode_dim1,gmode_dict[gmode_dim1]) )
        if self.verbose: even_print( 'gmode dim2' , '%i / %s'%(gmode_dim2,gmode_dict[gmode_dim2]) )
        if self.verbose: even_print( 'gmode dim3' , '%i / %s'%(gmode_dim3,gmode_dict[gmode_dim3]) )
        if self.verbose: print(72*'-')
        
        # === (automatic) grid read
        
        ## can fail if >2[GB] and using driver=='mpio' and using one process
        ## https://github.com/h5py/h5py/issues/1052
        
        if True:
            
            ## dset object handles, no read yet
            dset1 = self[f'Kennsatz/GEOMETRY/{self.domainName}/dim01']
            dset2 = self[f'Kennsatz/GEOMETRY/{self.domainName}/dim02']
            dset3 = self[f'Kennsatz/GEOMETRY/{self.domainName}/dim03']
            
            float_bytes1  = dset1.dtype.itemsize
            float_bytes2  = dset2.dtype.itemsize
            float_bytes3  = dset3.dtype.itemsize
            
            data_gb_dim1 = float_bytes1*np.prod(dset1.shape) / 1024**3
            data_gb_dim2 = float_bytes2*np.prod(dset2.shape) / 1024**3
            data_gb_dim3 = float_bytes3*np.prod(dset3.shape) / 1024**3
            
            ## no 3D datasets, because no dim's GMODE is 5
            ## in this case, the dsets are just 0D/1D, so just read completely to all ranks
            if not any([ (gmode_dim1==5) , (gmode_dim2==5) , (gmode_dim3==5) ]):
                
                self.is_curvilinear = False
                self.is_rectilinear = True
                
                ## check just to be sure that the coord dsets in the HDF5 are indeed not 3D
                if (dset1.ndim>2):
                    raise ValueError
                if (dset2.ndim>2):
                    raise ValueError
                if (dset2.ndim>2):
                    raise ValueError
                
                ## read 1D datasets completely... this should never be a problem for RAM in MPI mode
                dim1_data = np.copy(dset1[()])
                dim2_data = np.copy(dset2[()])
                dim3_data = np.copy(dset3[()])
            
            else:
                
                '''
                at least one dim has GMODE=5, so 3D datasets exist, BUT unless GMODE==(5,5,5) 
                it is still possible to read as combo of 1D/2D, which is safe for MPI
                '''
                
                self.is_curvilinear = True
                self.is_rectilinear = False
                
                if (gmode_dim1==5) and (gmode_dim2==5) and (gmode_dim3==5): ## MUST read as 3D... only do here if read_3d_grid=True explicitly set
                    
                    ## this becomes very RAM intensive in MPI mode (all rannks are reading full 3D coord data)
                    if self.read_3d_grid:
                        dim1_data = np.copy(dset1[()])
                        dim2_data = np.copy(dset2[()])
                        dim3_data = np.copy(dset3[()])
                    else:
                        dim1_data = None
                        dim2_data = None
                        dim3_data = None
                
                elif (gmode_dim1==5) and (gmode_dim2==5) and (gmode_dim3==1): ## 551
                    dim1_data = np.copy(dset1[:,:,0][:,:,np.newaxis]) ## [x] is (nx,ny,1)
                    dim2_data = np.copy(dset2[:,:,0][:,:,np.newaxis]) ## [y] is (nx,ny,1)
                    dim3_data = np.copy(dset3[()])                    ## [z]
                
                elif (gmode_dim1==5) and (gmode_dim2==5) and (gmode_dim3==2): ## 552
                    dim1_data = np.copy(dset1[:,:,0][:,:,np.newaxis]) ## [x] is (nx,ny,1)
                    dim2_data = np.copy(dset2[:,:,0][:,:,np.newaxis]) ## [y] is (nx,ny,1)
                    dim3_data = np.copy(dset3[()])                    ## [z]
                
                elif (gmode_dim1==5) and (gmode_dim2==5) and (gmode_dim3==4): ## 554
                    dim1_data = np.copy(dset1[:,:,0][:,:,np.newaxis]) ## [x] is (nx,ny,1)
                    dim2_data = np.copy(dset2[:,:,0][:,:,np.newaxis]) ## [y] is (nx,ny,1)
                    dim3_data = np.copy(dset3[()])                    ## [z]
                
                else:
                    
                    print(f'gmode combo ({gmode_dim1:d},{gmode_dim2:d},{gmode_dim3:d}) does not yet have instructions in eas4.get_header()')
                    raise NotImplementedError
        
        ## old snippet for getting around HDF5 / h5py bug if >2[GB] and using driver=='mpio' and using one process
        ## keeping here for now
        ## https://github.com/h5py/h5py/issues/1052
        #except OSError:
        if False:
            
            if (gmode_dim1 == EAS4_FULL_G):
                dim1_data = np.zeros((nx,ny,nz), dtype = self['Kennsatz/GEOMETRY/%s/dim01'%self.domainName].dtype)
                for i in range(nx):
                    dim1_data[i,:,:] = self['Kennsatz/GEOMETRY/%s/dim01'%self.domainName][i,:,:]
            else:
                dim1_data = self['Kennsatz/GEOMETRY/%s/dim01'%self.domainName][:]
            
            if (gmode_dim2 == EAS4_FULL_G):
                dim2_data = np.zeros((nx,ny,nz), dtype = self['Kennsatz/GEOMETRY/%s/dim02'%self.domainName].dtype)
                for i in range(nx):
                    dim2_data[i,:,:] = self['Kennsatz/GEOMETRY/%s/dim02'%self.domainName][i,:,:]
            else:
                dim2_data = self['Kennsatz/GEOMETRY/%s/dim02'%self.domainName][:]
            
            if (gmode_dim3 == EAS4_FULL_G):
                dim3_data = np.zeros((nx,ny,nz), dtype = self['Kennsatz/GEOMETRY/%s/dim03'%self.domainName].dtype)
                for i in range(nx):
                    dim3_data[i,:,:] = self['Kennsatz/GEOMETRY/%s/dim03'%self.domainName][i,:,:]
            else:
                dim3_data = self['Kennsatz/GEOMETRY/%s/dim03'%self.domainName][:]
        
        ## check grid for span avg
        if False:
            if (self.measType == 'mean'):
                if (gmode_dim1 == EAS4_FULL_G):
                    if not np.allclose(dim1_data[:,:,0], dim1_data[:,:,1], rtol=1e-08):
                        raise AssertionError('check')
                if (gmode_dim2 == EAS4_FULL_G):
                    if not np.allclose(dim2_data[:,:,0], dim2_data[:,:,1], rtol=1e-08):
                        raise AssertionError('check')
        
        # === expand gmodes 1,2 --> 4 (always ok, even for MPI mode)
        
        ## convert EAS4_NO_G to EAS4_ALL_G (1 --> 4) --> always do this
        ## dim_n_data are already numpy arrays of shape (1,) --> no conversion necessary, just update 'gmode_dimn' attr
        if (gmode_dim1 == EAS4_NO_G):
            gmode_dim1 = self.gmode_dim1 = EAS4_ALL_G
        if (gmode_dim2 == EAS4_NO_G):
            gmode_dim2 = self.gmode_dim2 = EAS4_ALL_G
        if (gmode_dim3 == EAS4_NO_G):
            gmode_dim3 = self.gmode_dim3 = EAS4_ALL_G
        
        ## convert EAS4_X0DX_G to EAS4_ALL_G (2 --> 4) --> always do this
        if (gmode_dim1 == EAS4_X0DX_G):
            dim1_data  = np.linspace(dim1_data[0],dim1_data[0]+dim1_data[1]*(ndim1-1), ndim1, dtype=np.float64)
            gmode_dim1 = self.gmode_dim1 = EAS4_ALL_G
        if (gmode_dim2 == EAS4_X0DX_G):
            dim2_data  = np.linspace(dim2_data[0],dim2_data[0]+dim2_data[1]*(ndim2-1), ndim2, dtype=np.float64)
            gmode_dim2 = self.gmode_dim2 = EAS4_ALL_G
        if (gmode_dim3 == EAS4_X0DX_G):
            dim3_data  = np.linspace(dim3_data[0],dim3_data[0]+dim3_data[1]*(ndim3-1), ndim3, dtype=np.float64)
            gmode_dim3 = self.gmode_dim3 = EAS4_ALL_G
        
        # ===
        
        x = self.x = np.copy(dim1_data)
        y = self.y = np.copy(dim2_data)
        z = self.z = np.copy(dim3_data)
        
        # ## bug check
        # if (z.size > 1):
        #     if np.all(np.isclose(z,z[0],rtol=1e-12)):
        #         raise AssertionError('z has size > 1 but all grid coords are identical!')
        
        if self.verbose: even_print('x_min', '%0.2f'%x.min())
        if self.verbose: even_print('x_max', '%0.2f'%x.max())
        if self.is_rectilinear:
            if (self.nx>2):
                if self.verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (x[1]-x[0]), (x[-1]-x[-2]) ))
        if self.verbose: even_print('y_min', '%0.2f'%y.min())
        if self.verbose: even_print('y_max', '%0.2f'%y.max())
        if self.is_rectilinear:
            if (self.ny>2):
                if self.verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (y[1]-y[0]), (y[-1]-y[-2]) ))
        if self.verbose: even_print('z_min', '%0.2f'%z.min())
        if self.verbose: even_print('z_max', '%0.2f'%z.max())
        if self.is_rectilinear:
            if (self.nz>2):
                if self.verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (z[1]-z[0]), (z[-1]-z[-2]) ))
        if self.verbose: print(72*'-'+'\n')
        
        # === time & scalar info
        
        if self.verbose: print('time & scalar info\n'+72*'-')
        
        n_scalars = self['Kennsatz/PARAMETER'].attrs['PARAMETER_SIZE'][0]
        
        if ('Kennsatz/PARAMETER/PARAMETERS_ATTRS' in self):
            scalars =  [ s.decode('utf-8').strip() for s in self['Kennsatz/PARAMETER/PARAMETERS_ATTRS'][()] ]
        else:
            ## this is the older gen structure
            scalars = [ self['Kennsatz/PARAMETER'].attrs['PARAMETERS_ATTR_%06d'%i][0].decode('utf-8').strip() for i in range(n_scalars) ]
        
        scalar_n_map = dict(zip(scalars, range(n_scalars)))
        
        self.scalars_dtypes = []
        for scalar in scalars:
            dset_path = 'Data/%s/ts_%06d/par_%06d'%(self.domainName,0,scalar_n_map[scalar])
            if (dset_path in self):
                self.scalars_dtypes.append(self[dset_path].dtype)
            else:
                #self.scalars_dtypes.append(np.float32)
                raise AssertionError('dset not found: %s'%dset_path)
        
        self.scalars_dtypes_dict = dict(zip(scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        
        nt         = self['Kennsatz/TIMESTEP'].attrs['TIMESTEP_SIZE'][0] 
        gmode_time = self['Kennsatz/TIMESTEP'].attrs['TIMESTEP_MODE'][0]
        
        ## a baseflow will not have a TIMEGRID
        if ('Kennsatz/TIMESTEP/TIMEGRID' in self):
            t = self['Kennsatz/TIMESTEP/TIMEGRID'][:]
        else:
            t = np.array( [0.] , dtype=np.float64 )
        
        if (gmode_time==EAS4_X0DX_G): ## =2 --> i.e. more than one timestep
            t = np.linspace(t[0],t[0]+t[1]*(nt - 1), nt  )
            gmode_time = EAS4_ALL_G
        else:
            #print('gmode_time : '+str(gmode_time))
            pass
        
        if (t.size>1):
            dt = t[1] - t[0]
            duration = t[-1] - t[0]
        else:
            dt = 0.
            duration = 0.
        
        if self.verbose: even_print('nt', '%i'%nt )
        if self.verbose: even_print('dt', '%0.6f'%dt)
        if self.verbose: even_print('duration', '%0.2f'%duration )
        
        # === attach to instance
        
        self.n_scalars    = n_scalars
        self.scalars      = scalars
        self.scalar_n_map = scalar_n_map
        self.t            = t
        self.dt           = dt
        self.nt           = nt
        self.duration     = duration
        
        self.ti           = np.arange(self.nt, dtype=np.float64)
        
        if self.verbose: print(72*'-'+'\n')
        
        # === udef dictionary attached to instance
        
        # udef_char = [     'Ma',     'Re',     'Pr',      'kappa',    'R',    'p_inf',    'T_inf',    'C_Suth',    'S_Suth',    'mu_Suth_ref',    'T_Suth_ref' ]
        # udef_real = [ self.Ma , self.Re , self.Pr ,  self.kappa, self.R, self.p_inf, self.T_inf, self.C_Suth, self.S_Suth, self.mu_Suth_ref, self.T_Suth_ref  ]
        # self.udef = dict(zip(udef_char, udef_real))
        
        self.udef = { 'Ma':self.Ma,
                      'Re':self.Re,
                      'Pr':self.Pr,
                      'kappa':self.kappa,
                      'R':self.R,
                      'p_inf':self.p_inf,
                      'T_inf':self.T_inf,
                      #'C_Suth':self.C_Suth, ## technically a derived variable
                      'S_Suth':self.S_Suth,
                      'mu_Suth_ref':self.mu_Suth_ref,
                      'T_Suth_ref':self.T_Suth_ref,
                      }
        
        return
    
    # ===
    
    def get_mean(self, **kwargs):
        '''
        get spanwise mean of 2D EAS4 file
        '''
        axis = kwargs.get('axis',(2,)) ## default: avg over [z]
        
        if (self.measType!='mean'):
            raise NotImplementedError('get_mean() not yet valid for measType=\'%s\''%self.measType)
        
        ## numpy structured array
        data_mean = np.zeros(shape=(self.nx,self.ny), dtype={'names':self.scalars, 'formats':self.scalars_dtypes})
        
        for si, scalar in enumerate(self.scalars):
            scalar_dtype = self.scalars_dtypes[si]
            dset_path = 'Data/%s/ts_%06d/par_%06d'%(self.domainName,0,self.scalar_n_map[scalar])
            data = np.copy(self[dset_path][()])
            ## perform np.mean() with float64 accumulator!
            scalar_mean = np.mean(data, axis=axis, dtype=np.float64).astype(scalar_dtype)
            data_mean[scalar] = scalar_mean
        
        return data_mean
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        print('make_xdmf() not yet implemented for turbx class EAS4')
        raise NotImplementedError
        return

class ztmd(h5py.File):
    '''
    span (z) & temporal (t) mean data (md)
    -----
    --> mean_flow_mpi.eas
    --> favre_mean_flow_mpi.eas
    --> ext_rms_fluctuation_mpi.eas
    --> ext_favre_fluctuation_mpi.eas
    --> turbulent_budget_mpi.eas
    -----
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.ztmd()')
        
        ## mpio driver for ZTMD currently not supported
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            raise ValueError('ZTMD class is currently not set up to be used with MPI')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## ztmd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 16    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 8     )
        perms          = kwargs.pop('perms'          , '640' )
        
        if not isinstance(stripe_count, int):
            raise ValueError
        if not isinstance(stripe_size_mb, int):
            raise ValueError
        if not isinstance(perms, str):
            raise ValueError
        if not len(perms)==3:
            raise ValueError
        if not re.fullmatch(r'\d{3}',perms):
            raise ValueError
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using romio --> causes crash
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        
        ## set ROMIO hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                ##
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                mpi_info.Set('cb_buffer_size' , str(int(round(16*1024**2))) ) ## 16 [MB]
                ##
                #mpi_info.Set('romio_no_indep_rw' , 'true' ) ## deferred open + only collective I/O 
                #mpi_info.Set('cb_nodes' , str(int(round(1*self.n_ranks))) )
                ##
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        ## | rdcc_nbytes:
        ## | ------------
        ## | Integer setting the total size of the raw data chunk cache for this dataset in bytes.
        ## | In most cases increasing this number will improve performance, as long as you have 
        ## | enough free memory. The default size is 1 MB
        
        ## --> gets passed to H5Pset_chunk_cache
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
        
        ## | rdcc_nslots:
        ## | ------------
        ## | Integer defining the number of chunk slots in the raw data chunk cache for this dataset.
        
        ## if ('rdcc_nslots' not in kwargs):
        ##     kwargs['rdcc_nslots'] = 521
        
        ## ztmd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with ztmd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(0.1)
                Path(self.fname).touch() ## touch a new file
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        self.mod_avail_tqdm = ('tqdm' in sys.modules)
        
        ## call actual h5py.File.__init__()
        super(ztmd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(ztmd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed ZTMD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(ztmd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of ZTMD class instance
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if (self.rank!=0):
            verbose=False
        
        # === udef (header vector dset based) --> the 'old' way but still present in RGD,CGD
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## make dict where keys are udef_char and values are udef_real
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.M_inf       = self.Ma
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            self.S_Suth      = self.udef['S_Suth']
            #self.C_Suth      = self.udef['C_Suth']
            
            self.C_Suth = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## [kg/(m·s·√K)]
            self.udef['C_Suth'] = self.C_Suth
            
            if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            
            # === characteristic values : derived
            
            self.mu_inf    = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            self.rho_inf   = self.p_inf/(self.R*self.T_inf)
            self.nu_inf    = self.mu_inf/self.rho_inf
            self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            self.U_inf     = self.Ma*self.a_inf
            self.cp        = self.R*self.kappa/(self.kappa-1.)
            self.cv        = self.cp/self.kappa
            self.recov_fac = self.Pr**(1/3)
            self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            self.tchar = self.lchar / self.U_inf
            self.uchar = self.U_inf
            
            #self.p_tot_inf = self.p_inf * (1 + (self.kappa-1)/2 * self.U_inf**2 / (self.kappa*self.R*self.T_inf))**(self.kappa/(self.kappa-1))
            self.p_tot_inf   = self.p_inf   * (1 + (self.kappa-1)/2 * self.M_inf**2)**(self.kappa/(self.kappa-1))
            self.T_tot_inf   = self.T_inf   * (1 + (self.kappa-1)/2 * self.M_inf**2)
            self.rho_tot_inf = self.rho_inf * (1 + (self.kappa-1)/2 * self.M_inf**2)**(1/(self.kappa-1))
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            if verbose: even_print('tchar'           , '%0.6E [s]'        % self.tchar     )
            if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the CGD instance
            self.udef_deriv = { 'rho_inf':self.rho_inf,
                                'mu_inf':self.mu_inf,
                                'nu_inf':self.nu_inf,
                                'a_inf':self.a_inf,
                                'U_inf':self.U_inf,
                                'cp':self.cp,
                                'cv':self.cv,
                                'recov_fac':self.recov_fac,
                                'Taw':self.Taw,
                                'lchar':self.lchar,
                              }
        
        else:
            #print("dset 'header' not in ZTMD")
            pass
        
        # === udef (attr based)
        
        header_attr_str_list = ['Ma','Re','Pr','kappa','R','p_inf','T_inf','S_Suth','mu_Suth_ref','T_Suth_ref'] ## ,'C_Suth'
        if all([ attr_str in self.attrs.keys() for attr_str in header_attr_str_list ]):
            header_attr_based = True
        else:
            header_attr_based = False
        
        if header_attr_based:
            
            ## set all attributes
            for attr_str in header_attr_str_list:
                setattr( self, attr_str, self.attrs[attr_str] )
            
            if hasattr(self,'Ma') and not hasattr(self,'M_inf'):
                setattr(self,'M_inf',self.Ma)
            
            self.C_Suth = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## [kg/(m·s·√K)]
            #self.udef['C_Suth'] = self.C_Suth
            
            if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            
            # === characteristic values : derived
            
            # mu_inf_1 = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            # mu_inf_2 = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            # mu_inf_3 = self.C_Suth*self.T_inf**(3/2)/(self.T_inf+self.S_Suth)
            
            self.mu_inf    = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            self.rho_inf   = self.p_inf/(self.R*self.T_inf)
            self.nu_inf    = self.mu_inf/self.rho_inf
            self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            self.U_inf     = self.Ma*self.a_inf
            self.cp        = self.R*self.kappa/(self.kappa-1.)
            self.cv        = self.cp/self.kappa
            self.recov_fac = self.Pr**(1/3)
            self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            self.tchar = self.lchar / self.U_inf
            self.uchar = self.U_inf
            
            #self.p_tot_inf = self.p_inf * (1 + (self.kappa-1)/2 * self.U_inf**2 / (self.kappa*self.R*self.T_inf))**(self.kappa/(self.kappa-1))
            self.p_tot_inf   = self.p_inf   * (1 + (self.kappa-1)/2 * self.M_inf**2)**(self.kappa/(self.kappa-1))
            self.T_tot_inf   = self.T_inf   * (1 + (self.kappa-1)/2 * self.M_inf**2)
            self.rho_tot_inf = self.rho_inf * (1 + (self.kappa-1)/2 * self.M_inf**2)**(1/(self.kappa-1))
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            if verbose: even_print('tchar'           , '%0.6E [s]'        % self.tchar     )
            #if verbose: print(72*'-'+'\n')
            
            # === write the 'derived' udef variables to a dict attribute of the RGD instance
            self.udef_deriv = { 'rho_inf':self.rho_inf,
                                'mu_inf':self.mu_inf,
                                'nu_inf':self.nu_inf,
                                'a_inf':self.a_inf,
                                'U_inf':self.U_inf,
                                'cp':self.cp,
                                'cv':self.cv,
                                'recov_fac':self.recov_fac,
                                'Taw':self.Taw,
                                'lchar':self.lchar,
                              }
        
        if ('duration_avg' in self.attrs.keys()):
            self.duration_avg = self.attrs['duration_avg']
        if ('nx' in self.attrs.keys()):
            self.nx = self.attrs['nx']
        if ('ny' in self.attrs.keys()):
            self.ny = self.attrs['ny']
        
        # if ('p_inf' in self.attrs.keys()):
        #     self.p_inf = self.attrs['p_inf']
        # if ('lchar' in self.attrs.keys()):
        #     self.lchar = self.attrs['lchar']
        # if ('U_inf' in self.attrs.keys()):
        #     self.U_inf = self.attrs['U_inf']
        # if ('Re' in self.attrs.keys()):
        #     self.Re = self.attrs['Re']
        # if ('T_inf' in self.attrs.keys()):
        #     self.T_inf = self.attrs['T_inf']
        # if ('rho_inf' in self.attrs.keys()):
        #     self.rho_inf = self.attrs['rho_inf']
        
        if ('dims/x' in self):
            self.x = np.copy( self['dims/x'][()] ) ## dont transpose yet
        if ('dims/y' in self):
            self.y = np.copy( self['dims/y'][()] ) ## dont transpose yet
        if ('dims/t' in self):
            self.t = t = np.copy( self['dims/t'][()] )
        
        if hasattr(self,'x') and hasattr(self,'y'):
            if (self.x.ndim==1) and (self.y.ndim==1):
                self.xx, self.yy = np.meshgrid( self.x, self.y, indexing='ij' )
            elif (self.x.ndim==2) and (self.y.ndim==2):
                self.x  = np.copy( self.x.T )
                self.y  = np.copy( self.y.T )
                self.xx = np.copy( self.x   )
                self.yy = np.copy( self.y   )
            else:
                raise ValueError
        
        if ('dz' in self.attrs.keys()):
            self.dz = self.attrs['dz']
        
        if ('dims/stang' in self):
            self.stang = np.copy( self['dims/stang'][()] )
        if ('dims/snorm' in self):
            self.snorm = np.copy( self['dims/snorm'][()] )
        
        if ('csys/vtang' in self):
            self.vtang = np.copy( self['csys/vtang'][()] )
        if ('csys/vnorm' in self):
            self.vnorm = np.copy( self['csys/vnorm'][()] )
        
        if ('dims/crv_R' in self):
            self.crv_R = np.copy( self['dims/crv_R'][()] )
        if ('dims/R_min' in self):
            self.R_min = self['dims/R_min'][()]
        
        if verbose: print(72*'-')
        if verbose and hasattr(self,'duration_avg'): even_print('duration_avg', '%0.5f'%self.duration_avg)
        if verbose: even_print('nx', '%i'%self.nx)
        if verbose: even_print('ny', '%i'%self.ny)
        #if verbose: print(72*'-')
        
        # ===
        
        if ('rectilinear' in self.attrs.keys()):
            self.rectilinear = self.attrs['rectilinear']
        
        if ('curvilinear' in self.attrs.keys()):
            self.curvilinear = self.attrs['curvilinear']
        
        ## check
        if hasattr(self,'rectilinear') and not hasattr(self,'curvilinear'):
            raise ValueError
        if hasattr(self,'curvilinear') and not hasattr(self,'rectilinear'):
            raise ValueError
        if hasattr(self,'rectilinear') or hasattr(self,'curvilinear'):
            if self.rectilinear and self.curvilinear:
                raise ValueError
            if not self.rectilinear and not self.curvilinear:
                raise ValueError
        
        if ('requires_wall_norm_interp' in self.attrs.keys()):
            self.requires_wall_norm_interp = self.attrs['requires_wall_norm_interp']
        else:
            self.requires_wall_norm_interp = False
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys())
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self['data/%s'%scalar].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    # === I/O funcs
    
    def compile_data(self, path, **kwargs):
        '''
        Copy data from 2D EAS4 containers (output from NS3D) to a ZTMD container
        -----
        
        The 'path' directory should contain one or more of the following files:
        
        --> mean_flow_mpi.eas
        --> favre_mean_flow_mpi.eas
        --> ext_rms_fluctuation_mpi.eas
        --> ext_favre_fluctuation_mpi.eas
        --> turbulent_budget_mpi.eas
        
        /dims : 2D dimension datasets (x,y,..) and possibly 1D dimension datasets (s_wall,..)
        /data : 2D datasets (u,uIuI,..)
        
        Datasets are dimensionalized to SI units upon import!
        
        /dimless : copy the dimless datasets as a reference
        
        Curvilinear cases may have the following additional HDF5 groups
        
        /data_1Dx : 1D datsets in streamwise (x/s1) direction (μ_wall,ρ_wall,u_τ,..)
        /csys     : coordinate system transformation arrays (projection vectors, transform tensors, etc.)
        -----
        /dims_2Dw : alternate grid (e.g. wall-normal projected/interpolation grid)
        /data_2Dw : data interpolated onto alternate grid
        '''
        
        verbose   = kwargs.get( 'verbose', True)
        recalc_mu = kwargs.get( 'recalc_mu', False)
        
        if verbose: print('\n'+'turbx.ztmd.compile_data()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        even_print('ztmd',str(self.fname))
        
        ## dz,dt should be input as dimless (characteristic/inlet) (output from tgg)
        ## --> dz & dt get re-dimensionalized during this func!
        dz = kwargs.get('dz',None)
        nz = kwargs.get('nz',None)
        dt = kwargs.get('dt',None)
        
        path_ztmean = Path(path)
        if not path_ztmean.is_dir():
            raise FileNotFoundError('%s does not exist.'%str(path_ztmean))
        fn_Re_mean     = Path(path_ztmean, 'mean_flow_mpi.eas')
        fn_Fv_mean     = Path(path_ztmean, 'favre_mean_flow_mpi.eas')
        fn_Re_fluct    = Path(path_ztmean, 'ext_rms_fluctuation_mpi.eas')
        fn_Fv_fluct    = Path(path_ztmean, 'ext_favre_fluctuation_mpi.eas')
        fn_turb_budget = Path(path_ztmean, 'turbulent_budget_mpi.eas')
        
        self.attrs['fn_Re_mean']     = str( fn_Re_mean.relative_to(Path())     )
        self.attrs['fn_Fv_mean']     = str( fn_Fv_mean.relative_to(Path())     )
        self.attrs['fn_Re_fluct']    = str( fn_Re_fluct.relative_to(Path())    )
        self.attrs['fn_Fv_fluct']    = str( fn_Fv_fluct.relative_to(Path())    )
        self.attrs['fn_turb_budget'] = str( fn_turb_budget.relative_to(Path()) )
        
        ## the simulation timestep dt is not known from the averaged files
        if (dt is not None):
            self.attrs['dt'] = dt
        if (nz is not None):
            self.attrs['nz'] = nz
        if (dz is not None):
            self.attrs['dz'] = dz
        
        if verbose:
            if (nz is not None):
                even_print('nz' , '%i'%nz )
            if (dz is not None):
                even_print('dz' , '%0.6e'%dz )
            if (dt is not None):
                even_print('dt' , '%0.6e'%dt )
            print(72*'-')
        
        # ===
        
        if fn_Re_mean.exists():
            even_print('eas4 Re mean',str(fn_Re_mean.relative_to(Path())))
            with eas4(str(fn_Re_mean),'r',verbose=False) as f1:
                
                ## the EAS4 data is still organized by rank in [z], so perform average across ranks
                data_mean = f1.get_mean()
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                nx = f1.nx ; self.attrs['nx'] = nx
                ny = f1.ny ; self.attrs['ny'] = ny
                
                Ma          = f1.Ma          ; self.attrs['Ma']          = Ma
                Re          = f1.Re          ; self.attrs['Re']          = Re
                Pr          = f1.Pr          ; self.attrs['Pr']          = Pr
                T_inf       = f1.T_inf       ; self.attrs['T_inf']       = T_inf
                p_inf       = f1.p_inf       ; self.attrs['p_inf']       = p_inf
                kappa       = f1.kappa       ; self.attrs['kappa']       = kappa
                R           = f1.R           ; self.attrs['R']           = R
                mu_Suth_ref = f1.mu_Suth_ref ; self.attrs['mu_Suth_ref'] = mu_Suth_ref
                T_Suth_ref  = f1.T_Suth_ref  ; self.attrs['T_Suth_ref']  = T_Suth_ref
                C_Suth      = f1.C_Suth      ; self.attrs['C_Suth']      = C_Suth
                S_Suth      = f1.S_Suth      ; self.attrs['S_Suth']      = S_Suth
                
                rho_inf   = f1.rho_inf   # ; self.attrs['rho_inf']   = rho_inf
                mu_inf    = f1.mu_inf    # ; self.attrs['mu_inf']    = mu_inf
                nu_inf    = f1.nu_inf    # ; self.attrs['nu_inf']    = nu_inf
                a_inf     = f1.a_inf     # ; self.attrs['a_inf']     = a_inf
                U_inf     = f1.U_inf     # ; self.attrs['U_inf']     = U_inf
                cp        = f1.cp        # ; self.attrs['cp']        = cp
                cv        = f1.cv        # ; self.attrs['cv']        = cv
                recov_fac = f1.recov_fac # ; self.attrs['recov_fac'] = recov_fac
                Taw       = f1.Taw       # ; self.attrs['Taw']       = Taw
                lchar     = f1.lchar     # ; self.attrs['lchar']     = lchar
                
                tchar = f1.lchar/f1.U_inf # ; self.attrs['tchar'] = tchar
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                Re_mean_total_avg_time       = f1.total_avg_time * tchar
                Re_mean_total_avg_iter_count = f1.total_avg_iter_count
                Re_mean_dt                   = Re_mean_total_avg_time / Re_mean_total_avg_iter_count
                
                self.attrs['Re_mean_total_avg_time'] = Re_mean_total_avg_time
                self.attrs['Re_mean_total_avg_iter_count'] = Re_mean_total_avg_iter_count
                self.attrs['Re_mean_dt'] = Re_mean_dt
                
                t_meas = f1.total_avg_time * (f1.lchar/f1.U_inf) ## dimensional [s]
                #t_meas = f1.total_avg_time ## dimless (char)
                self.attrs['t_meas'] = t_meas
                dset = self.create_dataset('dims/t', data=np.array([t_meas],dtype=np.float64), chunks=None)
                
                # ===
                
                ## from EAS4, dimless (char)
                x = np.copy(f1.x)
                y = np.copy(f1.y)
                
                if (f1.x.ndim==1) and (f1.y.ndim==1): ## rectilinear in [x,y]
                    
                    self.attrs['rectilinear'] = True
                    self.attrs['curvilinear'] = False
                    
                    ## dimensionalize & write
                    x  *= lchar
                    y  *= lchar
                    dz *= lchar
                    dt *= tchar
                    self.create_dataset('dims/x', data=x, chunks=None, dtype=np.float64)
                    self.create_dataset('dims/y', data=y, chunks=None, dtype=np.float64)
                    self.attrs['dz'] = dz
                    self.attrs['dt'] = dt
                    
                    self.attrs['nx'] = nx
                    self.attrs['ny'] = ny
                    #self.attrs['nz'] = 1 ## NO
                    if verbose:
                        even_print('nx' , '%i'%nx )
                        even_print('ny' , '%i'%ny )
                
                elif (f1.x.ndim==3) and (f1.y.ndim==3): ## curvilinear in [x,y]
                    
                    self.attrs['rectilinear'] = False
                    self.attrs['curvilinear'] = True
                    
                    ## 3D coords: confirm that x,y coords are same in [z] direction
                    np.testing.assert_allclose( x[-1,-1,:] , x[-1,-1,0] , rtol=1e-14 , atol=1e-14 )
                    np.testing.assert_allclose( y[-1,-1,:] , y[-1,-1,0] , rtol=1e-14 , atol=1e-14 )
                    
                    ## 3D coords: take only 1 layer in [z]
                    x = np.squeeze( np.copy( x[:,:,0] ) ) ## dimless (char)
                    y = np.squeeze( np.copy( y[:,:,0] ) )
                    
                    if True: ## check against tgg data wall distance file (if it exists)
                        fn_dat = '../tgg/wall_distance.dat'
                        if os.path.isfile(fn_dat):
                            with open(fn_dat,'rb') as f:
                                d_ = pickle.load(f)
                                xy2d_tmp = d_['xy2d']
                                np.testing.assert_allclose(xy2d_tmp[:,:,0], x[:,:,0], rtol=1e-14, atol=1e-14)
                                np.testing.assert_allclose(xy2d_tmp[:,:,1], y[:,:,0], rtol=1e-14, atol=1e-14)
                                if verbose: even_print('check passed' , 'x grid' )
                                if verbose: even_print('check passed' , 'y grid' )
                                d_ = None; del d_
                                xy2d_tmp = None; del xy2d_tmp
                    
                    # ## backup non-dimensional coordinate arrays
                    # dset = self.create_dataset('/dimless/dims/x', data=x.T, chunks=None)
                    # dset = self.create_dataset('/dimless/dims/y', data=y.T, chunks=None)
                    
                    ## dimensionalize & write
                    x  *= lchar
                    y  *= lchar
                    dz *= lchar
                    dt *= tchar
                    
                    self.create_dataset('dims/x', data=x.T, chunks=None, dtype=np.float64)
                    self.create_dataset('dims/y', data=y.T, chunks=None, dtype=np.float64)
                    self.attrs['dz'] = dz
                    self.attrs['dt'] = dt
                    
                    self.attrs['nx'] = nx
                    self.attrs['ny'] = ny
                    #self.attrs['nz'] = 1 ## NO
                    if verbose:
                        even_print('nx' , '%i'%nx )
                        even_print('ny' , '%i'%ny )
                
                else:
                    raise ValueError('case x.ndim=%i , y.ndim=%i not yet accounted for'%(f1.x.ndim,f1.y.ndim))
                
                # === redimensionalize quantities (by sim characteristic quantities)
                
                u   = np.copy( data_mean['u']   ) * U_inf 
                v   = np.copy( data_mean['v']   ) * U_inf
                w   = np.copy( data_mean['w']   ) * U_inf
                rho = np.copy( data_mean['rho'] ) * rho_inf
                p   = np.copy( data_mean['p']   ) * (rho_inf * U_inf**2)
                T   = np.copy( data_mean['T']   ) * T_inf
                mu  = np.copy( data_mean['mu']  ) * mu_inf
                
                # === check μ
                #
                # the O(1%) discrepancies are due to [z,t] averaging
                # μ=f(T) BUT this no longer holds exactly once averaged
                # the user should decide if <μ> should be re-calculated from <T>
                
                T_NS3D = np.copy( data_mean['T'] ) ## dimless
                
                ## Non-dimensional Suth Temp 'Ts' 
                ## 'equations.F90', subroutines calc_viscosity() & initialize_viscosity()
                Ts = S_Suth/T_inf
                mu_NS3D = np.copy( T_NS3D**1.5 * ( 1 + Ts ) / ( T_NS3D + Ts ) ) ## dimless
                np.testing.assert_allclose(mu/mu_inf, mu_NS3D, rtol=0.003)
                
                mu_A = np.copy( mu_Suth_ref*(T/T_Suth_ref)**(3/2) * ((T_Suth_ref+S_Suth)/(T+S_Suth)) )
                mu_B = np.copy( C_Suth * T**(3/2) / (T + S_Suth) )
                np.testing.assert_allclose(mu_A, mu_B, rtol=1e-6) ## single precision
                
                np.testing.assert_allclose(mu, mu_A, rtol=0.003)
                np.testing.assert_allclose(mu, mu_B, rtol=0.003)
                
                ## !!! replace μ !!!
                if recalc_mu:
                    mu = np.copy( C_Suth * T**(3/2) / (T + S_Suth) )
                
                ## clear structured array from memory
                data_mean = None; del data_mean
                
                ## derived values from base scalars
                a     = np.copy( np.sqrt( kappa * R * T )      )
                nu    = np.copy( mu / rho                      )
                umag  = np.copy( np.sqrt( u**2 + v**2 + w**2 ) )
                M     = np.copy( umag / np.sqrt(kappa * R * T) )
                
                T_tot   = np.copy( T   * (1 + (kappa-1)/2 * M**2)                    )
                p_tot   = np.copy( p   * (1 + (kappa-1)/2 * M**2)**(kappa/(kappa-1)) )
                rho_tot = np.copy( rho * (1 + (kappa-1)/2 * M**2)**(1/(kappa-1))     )
                
                ## base scalars [u,v,w,ρ,p,T]
                self.create_dataset('data/u'   , data=u.T   , chunks=None)
                self.create_dataset('data/v'   , data=v.T   , chunks=None)
                self.create_dataset('data/w'   , data=w.T   , chunks=None)
                self.create_dataset('data/rho' , data=rho.T , chunks=None)
                self.create_dataset('data/p'   , data=p.T   , chunks=None)
                self.create_dataset('data/T'   , data=T.T   , chunks=None)
                
                ## derived fields
                self.create_dataset('data/a'       , data=a.T       , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[m/s]'
                self.create_dataset('data/mu'      , data=mu.T      , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[kg/(m·s)]'
                self.create_dataset('data/nu'      , data=nu.T      , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[m²/s]'
                self.create_dataset('data/umag'    , data=umag.T    , chunks=None) #; dset.attrs['dimensional'] = True  ; dset.attrs['unit'] = '[m/s]'
                self.create_dataset('data/M'       , data=M.T       , chunks=None) #; dset.attrs['dimensional'] = False
                self.create_dataset('data/T_tot'   , data=T_tot.T   , chunks=None)
                self.create_dataset('data/p_tot'   , data=p_tot.T   , chunks=None)
                self.create_dataset('data/rho_tot' , data=rho_tot.T , chunks=None)
        
        if fn_Re_fluct.exists():
            even_print('eas4 Re fluct',str(fn_Re_fluct.relative_to(Path())))
            with eas4(str(fn_Re_fluct),'r',verbose=False) as f1:
                
                data_mean = f1.get_mean()
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                Re_fluct_total_avg_time       = f1.total_avg_time
                Re_fluct_total_avg_iter_count = f1.total_avg_iter_count
                Re_fluct_dt                   = Re_fluct_total_avg_time/Re_fluct_total_avg_iter_count
                
                self.attrs['Re_fluct_total_avg_time'] = Re_fluct_total_avg_time
                self.attrs['Re_fluct_total_avg_iter_count'] = Re_fluct_total_avg_iter_count
                self.attrs['Re_fluct_dt'] = Re_fluct_dt
                
                uI_uI = data_mean["u'u'"] * U_inf**2
                vI_vI = data_mean["v'v'"] * U_inf**2
                wI_wI = data_mean["w'w'"] * U_inf**2
                uI_vI = data_mean["u'v'"] * U_inf**2
                uI_wI = data_mean["u'w'"] * U_inf**2
                vI_wI = data_mean["v'w'"] * U_inf**2
                
                self.create_dataset('data/uI_uI', data=uI_uI.T, chunks=None)
                self.create_dataset('data/vI_vI', data=vI_vI.T, chunks=None)
                self.create_dataset('data/wI_wI', data=wI_wI.T, chunks=None)
                self.create_dataset('data/uI_vI', data=uI_vI.T, chunks=None)
                self.create_dataset('data/uI_wI', data=uI_wI.T, chunks=None)
                self.create_dataset('data/vI_wI', data=vI_wI.T, chunks=None)
                
                uI_TI = data_mean["u'T'"] * (U_inf*T_inf)
                vI_TI = data_mean["v'T'"] * (U_inf*T_inf)
                wI_TI = data_mean["w'T'"] * (U_inf*T_inf)
                
                self.create_dataset('data/uI_TI', data=uI_TI.T, chunks=None)
                self.create_dataset('data/vI_TI', data=vI_TI.T, chunks=None)
                self.create_dataset('data/wI_TI', data=wI_TI.T, chunks=None)
                
                TI_TI = data_mean["T'T'"] * T_inf**2
                pI_pI = data_mean["p'p'"] * (rho_inf * U_inf**2)**2
                rI_rI = data_mean["r'r'"] * rho_inf**2
                muI_muI = data_mean["mu'mu'"] * mu_inf**2
                
                self.create_dataset('data/TI_TI',   data=TI_TI.T,   chunks=None)
                self.create_dataset('data/pI_pI',   data=pI_pI.T,   chunks=None)
                self.create_dataset('data/rI_rI',   data=rI_rI.T,   chunks=None)
                self.create_dataset('data/muI_muI', data=muI_muI.T, chunks=None)
                
                tauI_xx = data_mean["tau'_xx"] * mu_inf * U_inf / lchar
                tauI_yy = data_mean["tau'_yy"] * mu_inf * U_inf / lchar
                tauI_zz = data_mean["tau'_zz"] * mu_inf * U_inf / lchar
                tauI_xy = data_mean["tau'_xy"] * mu_inf * U_inf / lchar
                tauI_xz = data_mean["tau'_xz"] * mu_inf * U_inf / lchar
                tauI_yz = data_mean["tau'_yz"] * mu_inf * U_inf / lchar
                
                self.create_dataset('data/tauI_xx', data=tauI_xx.T, chunks=None)
                self.create_dataset('data/tauI_yy', data=tauI_yy.T, chunks=None)
                self.create_dataset('data/tauI_zz', data=tauI_zz.T, chunks=None)
                self.create_dataset('data/tauI_xy', data=tauI_xy.T, chunks=None)
                self.create_dataset('data/tauI_xz', data=tauI_xz.T, chunks=None)
                self.create_dataset('data/tauI_yz', data=tauI_yz.T, chunks=None)
                
                # === RMS values
                
                if True: ## dimensional
                    
                    uI_uI_rms = np.sqrt(       data_mean["u'u'"]  * U_inf**2 )
                    vI_vI_rms = np.sqrt(       data_mean["v'v'"]  * U_inf**2 )
                    wI_wI_rms = np.sqrt(       data_mean["w'w'"]  * U_inf**2 )
                    uI_vI_rms = np.sqrt(np.abs(data_mean["u'v'"]) * U_inf**2 ) * np.sign(data_mean["u'v'"]) 
                    uI_wI_rms = np.sqrt(np.abs(data_mean["u'w'"]) * U_inf**2 ) * np.sign(data_mean["u'w'"])
                    vI_wI_rms = np.sqrt(np.abs(data_mean["v'w'"]) * U_inf**2 ) * np.sign(data_mean["v'w'"])
                    
                    uI_TI_rms = np.sqrt(np.abs(data_mean["u'T'"]) * U_inf*T_inf) * np.sign(data_mean["u'T'"])
                    vI_TI_rms = np.sqrt(np.abs(data_mean["v'T'"]) * U_inf*T_inf) * np.sign(data_mean["v'T'"])
                    wI_TI_rms = np.sqrt(np.abs(data_mean["w'T'"]) * U_inf*T_inf) * np.sign(data_mean["w'T'"])
                    
                    rI_rI_rms   = np.sqrt( data_mean["r'r'"]   * rho_inf**2              )
                    TI_TI_rms   = np.sqrt( data_mean["T'T'"]   * T_inf**2                )
                    pI_pI_rms   = np.sqrt( data_mean["p'p'"]   * (rho_inf * U_inf**2)**2 )
                    muI_muI_rms = np.sqrt( data_mean["mu'mu'"] * mu_inf**2               )
                    
                    M_rms = uI_uI_rms / np.sqrt(kappa * R * T)
                
                # if False: ## dimless
                #     
                #     uI_uI_rms = np.sqrt(        data_mean["u'u'"]  )
                #     vI_vI_rms = np.sqrt(        data_mean["v'v'"]  )
                #     wI_wI_rms = np.sqrt(        data_mean["w'w'"]  )
                #     uI_vI_rms = np.sqrt( np.abs(data_mean["u'v'"]) ) * np.sign(data_mean["u'v'"]) 
                #     uI_wI_rms = np.sqrt( np.abs(data_mean["u'w'"]) ) * np.sign(data_mean["u'w'"])
                #     vI_wI_rms = np.sqrt( np.abs(data_mean["v'w'"]) ) * np.sign(data_mean["v'w'"])
                #     
                #     uI_TI_rms = np.sqrt( np.abs(data_mean["u'T'"]) ) * np.sign(data_mean["u'T'"])
                #     vI_TI_rms = np.sqrt( np.abs(data_mean["v'T'"]) ) * np.sign(data_mean["v'T'"])
                #     wI_TI_rms = np.sqrt( np.abs(data_mean["w'T'"]) ) * np.sign(data_mean["w'T'"])
                #     
                #     rI_rI_rms   = np.sqrt( data_mean["r'r'"]   )
                #     TI_TI_rms   = np.sqrt( data_mean["T'T'"]   )
                #     pI_pI_rms   = np.sqrt( data_mean["p'p'"]   )
                #     muI_muI_rms = np.sqrt( data_mean["mu'mu'"] )
                #     
                #     # ...
                #     M_rms = np.sqrt( data_mean["u'u'"] * U_inf**2 ) / np.sqrt(kappa * R * (T*T_inf) )
                
                # ===
                
                self.create_dataset( 'data/uI_uI_rms' , data=uI_uI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_vI_rms' , data=vI_vI_rms.T , chunks=None )
                self.create_dataset( 'data/wI_wI_rms' , data=wI_wI_rms.T , chunks=None )
                self.create_dataset( 'data/uI_vI_rms' , data=uI_vI_rms.T , chunks=None )
                self.create_dataset( 'data/uI_wI_rms' , data=uI_wI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_wI_rms' , data=vI_wI_rms.T , chunks=None )
                ##
                self.create_dataset( 'data/uI_TI_rms' , data=uI_TI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_TI_rms' , data=vI_TI_rms.T , chunks=None )
                self.create_dataset( 'data/wI_TI_rms' , data=wI_TI_rms.T , chunks=None )
                ##
                self.create_dataset( 'data/rI_rI_rms'   , data=rI_rI_rms.T   , chunks=None )
                self.create_dataset( 'data/TI_TI_rms'   , data=TI_TI_rms.T   , chunks=None )
                self.create_dataset( 'data/pI_pI_rms'   , data=pI_pI_rms.T   , chunks=None )
                self.create_dataset( 'data/muI_muI_rms' , data=muI_muI_rms.T , chunks=None )
                ##
                self.create_dataset( 'data/M_rms' , data=M_rms.T , chunks=None )
        
        if fn_Fv_mean.exists():
            #print('--r-> %s'%fn_Fv_mean.relative_to(Path()) )
            even_print('eas4 Fv mean',str(fn_Fv_mean.relative_to(Path())))
            with eas4(str(fn_Fv_mean),'r',verbose=False) as f1:
                
                ## the EAS4 data is still organized by rank in [z], so perform average across ranks
                data_mean = f1.get_mean()
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                Fv_mean_total_avg_time       = f1.total_avg_time * tchar
                Fv_mean_total_avg_iter_count = f1.total_avg_iter_count
                Fv_mean_dt                   = Fv_mean_total_avg_time / Fv_mean_total_avg_iter_count
                
                self.attrs['Fv_mean_total_avg_time'] = Fv_mean_total_avg_time
                self.attrs['Fv_mean_total_avg_iter_count'] = Fv_mean_total_avg_iter_count
                self.attrs['Fv_mean_dt'] = Fv_mean_dt
                
                u_Fv   = np.copy( data_mean['u']   ) * U_inf 
                v_Fv   = np.copy( data_mean['v']   ) * U_inf
                w_Fv   = np.copy( data_mean['w']   ) * U_inf
                rho_Fv = np.copy( data_mean['rho'] ) * rho_inf
                p_Fv   = np.copy( data_mean['p']   ) * (rho_inf * U_inf**2)
                T_Fv   = np.copy( data_mean['T']   ) * T_inf
                mu_Fv  = np.copy( data_mean['mu']  ) * mu_inf
                
                uu_Fv  = np.copy( data_mean['uu'] ) * U_inf**2
                uv_Fv  = np.copy( data_mean['uv'] ) * U_inf**2
                
                data_mean = None; del data_mean
                
                self.create_dataset('data/u_Fv'   , data=u_Fv.T   , chunks=None)
                self.create_dataset('data/v_Fv'   , data=v_Fv.T   , chunks=None)
                self.create_dataset('data/w_Fv'   , data=w_Fv.T   , chunks=None)
                self.create_dataset('data/rho_Fv' , data=rho_Fv.T , chunks=None)
                self.create_dataset('data/p_Fv'   , data=p_Fv.T   , chunks=None)
                self.create_dataset('data/T_Fv'   , data=T_Fv.T   , chunks=None)
                self.create_dataset('data/mu_Fv'  , data=mu_Fv.T  , chunks=None)
                
                self.create_dataset('data/uu_Fv' , data=uu_Fv.T   , chunks=None)
                self.create_dataset('data/uv_Fv' , data=uv_Fv.T   , chunks=None)
        
        if fn_Fv_fluct.exists():
            #print('--r-> %s'%fn_Fv_fluct.relative_to(Path()) )
            even_print('eas4 Fv fluct',str(fn_Fv_fluct.relative_to(Path())))
            with eas4(str(fn_Fv_fluct),'r',verbose=False) as f1:
                
                data_mean = f1.get_mean()
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                Fv_fluct_total_avg_time       = f1.total_avg_time
                Fv_fluct_total_avg_iter_count = f1.total_avg_iter_count
                Fv_fluct_dt                   = Fv_fluct_total_avg_time/Fv_fluct_total_avg_iter_count
                
                self.attrs['Fv_fluct_total_avg_time'] = Fv_fluct_total_avg_time
                self.attrs['Fv_fluct_total_avg_iter_count'] = Fv_fluct_total_avg_iter_count
                self.attrs['Fv_fluct_dt'] = Fv_fluct_dt
                
                r_uII_uII = data_mean["r u''u''"]  * rho_inf * U_inf**2
                r_vII_vII = data_mean["r v''v''"]  * rho_inf * U_inf**2
                r_wII_wII = data_mean["r w''_w''"] * rho_inf * U_inf**2
                r_uII_vII = data_mean["r u''v''"]  * rho_inf * U_inf**2
                r_uII_wII = data_mean["r u''w''"]  * rho_inf * U_inf**2
                r_vII_wII = data_mean["r w''v''"]  * rho_inf * U_inf**2
                
                self.create_dataset('data/r_uII_uII', data=r_uII_uII.T, chunks=None)
                self.create_dataset('data/r_vII_vII', data=r_vII_vII.T, chunks=None)
                self.create_dataset('data/r_wII_wII', data=r_wII_wII.T, chunks=None)
                self.create_dataset('data/r_uII_vII', data=r_uII_vII.T, chunks=None)
                self.create_dataset('data/r_uII_wII', data=r_uII_wII.T, chunks=None)
                self.create_dataset('data/r_vII_wII', data=r_vII_wII.T, chunks=None)
                
                r_uII_TII = data_mean["r u''T''"] * rho_inf * U_inf * T_inf
                r_vII_TII = data_mean["r v''T''"] * rho_inf * U_inf * T_inf
                r_wII_TII = data_mean["r w''T''"] * rho_inf * U_inf * T_inf
                
                self.create_dataset('data/r_uII_TII', data=r_uII_TII.T, chunks=None)
                self.create_dataset('data/r_vII_TII', data=r_vII_TII.T, chunks=None)
                self.create_dataset('data/r_wII_TII', data=r_wII_TII.T, chunks=None)
                
                r_TII_TII   = data_mean["r T''T''"]   * rho_inf * T_inf**2
                r_pII_pII   = data_mean["r p''p''"]   * rho_inf * (rho_inf * U_inf**2)**2
                r_rII_rII   = data_mean["r r''r''"]   * rho_inf * rho_inf**2
                r_muII_muII = data_mean["r mu''mu''"] * rho_inf * mu_inf**2
                
                self.create_dataset('data/r_TII_TII',   data=r_TII_TII.T,   chunks=None)
                self.create_dataset('data/r_pII_pII',   data=r_pII_pII.T,   chunks=None)
                self.create_dataset('data/r_rII_rII',   data=r_rII_rII.T,   chunks=None)
                self.create_dataset('data/r_muII_muII', data=r_muII_muII.T, chunks=None)
                
                # === RMS
                
                if True:
                    
                    r_uII_uII_rms = np.sqrt(       data_mean["r u''u''"]  * rho_inf * U_inf**2 )
                    r_vII_vII_rms = np.sqrt(       data_mean["r v''v''"]  * rho_inf * U_inf**2 )
                    r_wII_wII_rms = np.sqrt(       data_mean["r w''_w''"] * rho_inf * U_inf**2 )
                    r_uII_vII_rms = np.sqrt(np.abs(data_mean["r u''v''"]) * rho_inf * U_inf**2 ) * np.sign(data_mean["r u''v''"]) 
                    r_uII_wII_rms = np.sqrt(np.abs(data_mean["r u''w''"]) * rho_inf * U_inf**2 ) * np.sign(data_mean["r u''w''"])
                    r_vII_wII_rms = np.sqrt(np.abs(data_mean["r w''v''"]) * rho_inf * U_inf**2 ) * np.sign(data_mean["r w''v''"])
                    ## ... ρ·u″T″
                    
                    self.create_dataset( 'data/r_uII_uII_rms' , data=r_uII_uII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_vII_vII_rms' , data=r_vII_vII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_wII_wII_rms' , data=r_wII_wII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_uII_vII_rms' , data=r_uII_vII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_uII_wII_rms' , data=r_uII_wII_rms.T , chunks=None )
                    self.create_dataset( 'data/r_vII_wII_rms' , data=r_vII_wII_rms.T , chunks=None )
                    ## ... ρ·u″T″
        
        if fn_turb_budget.exists():
            #print('--r-> %s'%fn_turb_budget.relative_to(Path()) )
            even_print('eas4 turb budget',str(fn_turb_budget.relative_to(Path())))
            with eas4(str(fn_turb_budget),'r',verbose=False) as f1:
                
                data_mean = f1.get_mean() ## numpy structured array
                
                ## assert mean data shape
                for i, key in enumerate(data_mean.dtype.names):
                    if (data_mean[key].shape[0]!=f1.nx):
                        raise AssertionError('mean data dim1 shape != nx')
                    if (data_mean[key].shape[1]!=f1.ny):
                        raise AssertionError('mean data dim2 shape != ny')
                    if (data_mean[key].ndim!=2):
                        raise AssertionError('mean data ndim != 2')
                
                turb_budget_total_avg_time       = f1.total_avg_time
                turb_budget_total_avg_iter_count = f1.total_avg_iter_count
                turb_budget_dt                   = turb_budget_total_avg_time/turb_budget_total_avg_iter_count
                
                self.attrs['turb_budget_total_avg_time'] = turb_budget_total_avg_time
                self.attrs['turb_budget_total_avg_iter_count'] = turb_budget_total_avg_iter_count
                self.attrs['turb_budget_dt'] = turb_budget_dt
                
                production     = data_mean['prod.']     * U_inf**3 * rho_inf / lchar
                dissipation    = data_mean['dis.']      * U_inf**2 * mu_inf  / lchar**2
                turb_transport = data_mean['t-transp.'] * U_inf**3 * rho_inf / lchar
                visc_diffusion = data_mean['v-diff.']   * U_inf**2 * mu_inf  / lchar**2
                p_diffusion    = data_mean['p-diff.']   * U_inf**3 * rho_inf / lchar
                p_dilatation   = data_mean['p-dilat.']  * U_inf**3 * rho_inf / lchar
                rho_terms      = data_mean['rho-terms'] * U_inf**3 * rho_inf / lchar
                ##
                dset = self.create_dataset('data/production'     , data=production.T     , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/dissipation'    , data=dissipation.T    , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/turb_transport' , data=turb_transport.T , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/visc_diffusion' , data=visc_diffusion.T , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/p_diffusion'    , data=p_diffusion.T    , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/p_dilatation'   , data=p_dilatation.T   , chunks=None) #; dset.attrs['dimensional'] = True
                dset = self.create_dataset('data/rho_terms'      , data=rho_terms.T      , chunks=None) #; dset.attrs['dimensional'] = True
                
                if 'dissipation' in locals():
                    
                    #if not self.get('data/nu').attrs['dimensional']:
                    #    raise ValueError('nu is not dimensional')
                    
                    Kolm_len = (nu**3 / np.abs(dissipation))**(1/4)
                    self.create_dataset('data/Kolm_len', data=Kolm_len.T, chunks=None) # ; dset.attrs['dimensional'] = True
        
        # ===
        
        self.get_header(verbose=True)
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.ztmd.compile_data() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def compile_data_eas3(self, path, **kwargs):
        '''
        Copy data from legacy EAS3 containers (output from NS3D) to a ZTMD container
        -----
        
        The 'path' directory should contain one or more of the following files:
        
        --> mean_flow_all_mittel-z.eas
        --> favre_mean_flow_all_mittel-z.eas
        --> ext_rms_fluctuation_all_mittel-z.eas
        --> ext_favre_fluctuation_all_mittel-z.eas
        --> turbulent_budget_all_mittel-z.eas
        
        /dims : 2D dimension datasets (x,y,..)
        /data : 2D datasets (u,uIuI,..)
        
        Datasets are dimensionalized to SI units upon import!
        '''
        
        verbose   = kwargs.get('verbose',True)
        recalc_mu = kwargs.get( 'recalc_mu', False)
        
        if verbose: print('\n'+'turbx.ztmd.compile_data_eas3()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        even_print('ztmd',str(self.fname))
        
        ## dz,dt should be input as dimless (characteristic/inlet) (output from tgg)
        ## --> dz & dt get re-dimensionalized during this func!
        dz = kwargs.get('dz',None)
        nz = kwargs.get('nz',None)
        dt = kwargs.get('dt',None)
        
        path_ztmean = Path(path)
        if not path_ztmean.is_dir():
            raise FileNotFoundError('%s does not exist.'%str(path_ztmean))
        fn_Re_mean     = Path(path_ztmean, 'mean_flow_all_mittel-z.eas')
        fn_Fv_mean     = Path(path_ztmean, 'favre_mean_flow_all_mittel-z.eas')
        fn_Re_fluct    = Path(path_ztmean, 'ext_rms_fluctuation_all_mittel-z.eas')
        fn_Fv_fluct    = Path(path_ztmean, 'ext_favre_fluctuation_all_mittel-z.eas')
        fn_turb_budget = Path(path_ztmean, 'turbulent_budget_all_mittel-z.eas')
        
        self.attrs['fn_Re_mean']     = str( fn_Re_mean.relative_to(Path())     )
        self.attrs['fn_Fv_mean']     = str( fn_Fv_mean.relative_to(Path())     )
        self.attrs['fn_Re_fluct']    = str( fn_Re_fluct.relative_to(Path())    )
        self.attrs['fn_Fv_fluct']    = str( fn_Fv_fluct.relative_to(Path())    )
        self.attrs['fn_turb_budget'] = str( fn_turb_budget.relative_to(Path()) )
        
        ## the simulation timestep dt is not known from the averaged files
        if (dt is not None):
            self.attrs['dt'] = dt
            setattr(self,'dt',dt)
        if (nz is not None):
            self.attrs['nz'] = nz
            setattr(self,'nz',nz)
        if (dz is not None):
            self.attrs['dz'] = dz
            setattr(self,'dz',dz)
        
        if verbose:
            if (nz is not None):
                even_print('nz' , '%i'%nz )
            if (dz is not None):
                even_print('dz' , '%0.6e'%dz )
            if (dt is not None):
                even_print('dt' , '%0.6e'%dt )
            print(72*'-')
        
        # ===
        
        ## this function currently requires that the Reynolds mean file exists
        if not fn_Re_mean.exists():
            raise FileNotFoundError(str(fn_Re_mean))
        
        if fn_Re_mean.exists(): ## Reynolds mean (must exist!)
            even_print('eas3 Re mean',str(fn_Re_mean.relative_to(Path())))
            with eas3(fname=str(fn_Re_mean),verbose=False) as f1:
                
                nx = f1.nx ; self.attrs['nx'] = nx
                ny = f1.ny ; self.attrs['ny'] = ny
                
                Ma          = f1.Ma          ; self.attrs['Ma']          = Ma
                Re          = f1.Re          ; self.attrs['Re']          = Re
                Pr          = f1.Pr          ; self.attrs['Pr']          = Pr
                T_inf       = f1.T_inf       ; self.attrs['T_inf']       = T_inf
                p_inf       = f1.p_inf       ; self.attrs['p_inf']       = p_inf
                kappa       = f1.kappa       ; self.attrs['kappa']       = kappa
                R           = f1.R           ; self.attrs['R']           = R
                mu_Suth_ref = f1.mu_Suth_ref ; self.attrs['mu_Suth_ref'] = mu_Suth_ref
                T_Suth_ref  = f1.T_Suth_ref  ; self.attrs['T_Suth_ref']  = T_Suth_ref
                C_Suth      = f1.C_Suth      ; self.attrs['C_Suth']      = C_Suth
                S_Suth      = f1.S_Suth      ; self.attrs['S_Suth']      = S_Suth
                
                rho_inf   = f1.rho_inf   # ; self.attrs['rho_inf']   = rho_inf
                mu_inf    = f1.mu_inf    # ; self.attrs['mu_inf']    = mu_inf
                nu_inf    = f1.nu_inf    # ; self.attrs['nu_inf']    = nu_inf
                a_inf     = f1.a_inf     # ; self.attrs['a_inf']     = a_inf
                U_inf     = f1.U_inf     # ; self.attrs['U_inf']     = U_inf
                cp        = f1.cp        # ; self.attrs['cp']        = cp
                cv        = f1.cv        # ; self.attrs['cv']        = cv
                recov_fac = f1.recov_fac # ; self.attrs['recov_fac'] = recov_fac
                Taw       = f1.Taw       # ; self.attrs['Taw']       = Taw
                lchar     = f1.lchar     # ; self.attrs['lchar']     = lchar
                
                tchar = f1.lchar/f1.U_inf # ; self.attrs['tchar'] = tchar
                
                setattr(self,'lchar',lchar)
                setattr(self,'tchar',tchar)
                setattr(self,'U_inf',U_inf)
                
                # ===
                
                if (f1.t.shape[0]==1):
                    f1.total_avg_time = float(f1.t[0])
                else:
                    raise NotImplementedError
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                Re_mean_total_avg_time       = f1.total_avg_time * tchar
                Re_mean_total_avg_iter_count = f1.total_avg_iter_count
                Re_mean_dt                   = Re_mean_total_avg_time / Re_mean_total_avg_iter_count
                
                self.attrs['Re_mean_total_avg_time'] = Re_mean_total_avg_time
                self.attrs['Re_mean_total_avg_iter_count'] = Re_mean_total_avg_iter_count
                self.attrs['Re_mean_dt'] = Re_mean_dt
                
                t_meas = f1.total_avg_time * (f1.lchar/f1.U_inf) ## dimensional [s]
                #t_meas = f1.total_avg_time ## dimless (char)
                self.attrs['t_meas'] = t_meas
                dset = self.create_dataset('dims/t', data=np.array([t_meas,],dtype=np.float64), chunks=None)
                
                # ===
                
                ## from EAS3, dimless (char)
                x = np.copy(f1.x)
                y = np.copy(f1.y)
                
                if (f1.x.ndim==1) and (f1.y.ndim==1): ## rectilinear in [x,y]
                    
                    self.attrs['rectilinear'] = True
                    self.attrs['curvilinear'] = False
                    
                    ## dimensionalize
                    x  *= lchar
                    y  *= lchar
                    dz *= lchar
                    dt *= tchar
                    
                    ## write
                    self.create_dataset('dims/x', data=x, chunks=None, dtype=np.float64)
                    self.create_dataset('dims/y', data=y, chunks=None, dtype=np.float64)
                    self.attrs['dz'] = dz
                    self.attrs['dt'] = dt
                    self.attrs['nx'] = nx
                    self.attrs['ny'] = ny
                    #self.attrs['nz'] = 1 ## NO
                    
                    if verbose:
                        even_print('nx' , '%i'%nx )
                        even_print('ny' , '%i'%ny )
                    
                    setattr(self,'x',x)
                    setattr(self,'y',y)
                    
                    setattr(self,'dz',dz)
                    setattr(self,'dt',dt)
                    setattr(self,'nx',nx)
                    setattr(self,'ny',ny)
                
                else:
                    raise NotImplementedError
                
                # ===
                
                if (f1.scalars != f1.attr_param):
                    raise AssertionError
                if (f1.ndim3!=1):
                    raise AssertionError
                
                if (f1.accuracy == f1.IEEES):
                    dtypes = [ np.float32 for _ in f1.scalars ]
                if (f1.accuracy == f1.IEEED):
                    dtypes = [ np.float64 for _ in f1.scalars ]
                else:
                    raise ValueError
                
                ## numpy structured array
                data = np.zeros( shape=(nx,ny), dtype={'names':f1.scalars,'formats':dtypes} )
                
                ## populate structured array from EAS3 binary data file
                progress_bar = tqdm(total=f1.nt*f1.npar, ncols=100, desc='import eas3 Re mean', leave=False)
                for scalar in f1.attr_param:
                    tqdm.write(even_print(scalar,f'({nx},{ny})',s=True))
                    for jj in range(f1.ndim2):
                        if f1.accuracy == f1.IEEES:
                            packet = struct.unpack('!'+str(f1.ndim1)+'f',f1.f.read(4*f1.ndim1))[:]
                        else:
                            packet = struct.unpack('!'+str(f1.ndim1)+'d',f1.f.read(8*f1.ndim1))[:]
                        data[scalar][:,jj] = packet
                    progress_bar.update()
                progress_bar.close()
                
                ## re-dimensionalize by characteristic freestream quantities
                for scalar in data.dtype.names:
                    if scalar in ['u','v','w', 'uI','vI','wI', 'uII','vII','wII',]:
                        data[scalar] *= U_inf
                    elif scalar in ['r_uII','r_vII','r_wII']:
                        data[scalar] *= (U_inf*rho_inf)
                    elif scalar in ['T','TI','TII']:
                        data[scalar] *= T_inf
                    elif scalar in ['r_TII']:
                        data[scalar] *= (T_inf*rho_inf)
                    elif scalar in ['rho','rhoI']:
                        data[scalar] *= rho_inf
                    elif scalar in ['p','pI','pII']:
                        data[scalar] *= (rho_inf * U_inf**2)
                    elif scalar in ['mu']:
                        data[scalar] *= mu_inf
                    else:
                        raise ValueError(f"condition needed for redimensionalizing '{scalar}'")
                
                ## write structured array to ZTMD
                for scalar in data.dtype.names:
                    self.create_dataset(f'data/{scalar}', data=data[scalar].T, chunks=None)
                
                ## already made dimensional
                u   = np.copy( data['u']   )
                v   = np.copy( data['v']   )
                w   = np.copy( data['w']   )
                rho = np.copy( data['rho'] )
                p   = np.copy( data['p']   )
                T   = np.copy( data['T']   )
                mu  = np.copy( data['mu']  )
                
                # === check μ
                #
                # the O(1%) discrepancies are due to [z,t] averaging
                # μ=f(T) BUT this no longer holds exactly once averaged
                # the user should decide if <μ> should be re-calculated from <T>
                
                T_NS3D = np.copy( data['T']/T_inf ) ## dimless
                
                ## Non-dimensional Suth Temp 'Ts'
                ## 'equations.F90', subroutine initialize_viscosity()
                Ts = S_Suth/T_inf
                mu_NS3D = np.copy( T_NS3D**1.5 * ( 1 + Ts ) / ( T_NS3D + Ts ) ) ## dimless
                np.testing.assert_allclose(mu/mu_inf, mu_NS3D, rtol=0.003)
                
                mu_A = np.copy( mu_Suth_ref*(T/T_Suth_ref)**(3/2) * ((T_Suth_ref+S_Suth)/(T+S_Suth)) )
                mu_B = np.copy( C_Suth * T**(3/2) / (T + S_Suth) )
                np.testing.assert_allclose(mu_A, mu_B, rtol=1e-6) ## single precision
                
                np.testing.assert_allclose(mu, mu_A, rtol=0.003)
                np.testing.assert_allclose(mu, mu_B, rtol=0.003)
                
                ## !!! replace μ !!!
                if recalc_mu:
                    mu = np.copy( C_Suth * T**(3/2) / (T + S_Suth) )
                    if ('data/mu' in self):
                        del self['data/mu']
                    self.create_dataset('data/mu', data=mu.T, chunks=None)
                
                ## clear structured array from memory
                data = None ; del data
                
                ## derived values from base scalars
                a     = np.copy( np.sqrt( kappa * R * T )      )
                nu    = np.copy( mu / rho                      )
                umag  = np.copy( np.sqrt( u**2 + v**2 + w**2 ) )
                M     = np.copy( umag / np.sqrt(kappa * R * T) )
                
                T_tot   = np.copy( T   * (1 + (kappa-1)/2 * M**2)                    )
                p_tot   = np.copy( p   * (1 + (kappa-1)/2 * M**2)**(kappa/(kappa-1)) )
                rho_tot = np.copy( rho * (1 + (kappa-1)/2 * M**2)**(1/(kappa-1))     )
                
                ## write derived scalars
                self.create_dataset('data/a'       , data=a.T       , chunks=None)
                self.create_dataset('data/nu'      , data=nu.T      , chunks=None)
                self.create_dataset('data/umag'    , data=umag.T    , chunks=None)
                self.create_dataset('data/M'       , data=M.T       , chunks=None)
                self.create_dataset('data/T_tot'   , data=T_tot.T   , chunks=None)
                self.create_dataset('data/p_tot'   , data=p_tot.T   , chunks=None)
                self.create_dataset('data/rho_tot' , data=rho_tot.T , chunks=None)
        
        if fn_Re_fluct.exists(): ## Reynolds turbulent
            even_print('eas3 Re fluct',str(fn_Re_fluct.relative_to(Path())))
            with eas3(str(fn_Re_fluct),verbose=False) as f1:
                
                if (f1.t.shape[0]==1):
                    f1.total_avg_time = float(f1.t[0])
                else:
                    raise NotImplementedError
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                Re_fluct_total_avg_time       = f1.total_avg_time
                Re_fluct_total_avg_iter_count = f1.total_avg_iter_count
                Re_fluct_dt                   = Re_fluct_total_avg_time/Re_fluct_total_avg_iter_count
                
                self.attrs['Re_fluct_total_avg_time'] = Re_fluct_total_avg_time
                self.attrs['Re_fluct_total_avg_iter_count'] = Re_fluct_total_avg_iter_count
                self.attrs['Re_fluct_dt'] = Re_fluct_dt
                
                ## assert grid,lchar are same
                if hasattr(self,'lchar') and hasattr(f1,'lchar'):
                    np.testing.assert_allclose(self.lchar, f1.lchar, rtol=1e-12)
                if hasattr(self,'x') and hasattr(f1,'x'):
                    np.testing.assert_allclose(self.x, f1.x*lchar, rtol=1e-12)
                if hasattr(self,'y') and hasattr(f1,'y'):
                    np.testing.assert_allclose(self.y, f1.y*lchar, rtol=1e-12)
                
                # ===
                
                if (f1.scalars != f1.attr_param):
                    raise AssertionError
                if (f1.ndim3!=1):
                    raise AssertionError
                
                if (f1.accuracy == f1.IEEES):
                    dtypes = [ np.float32 for _ in f1.scalars ]
                if (f1.accuracy == f1.IEEED):
                    dtypes = [ np.float64 for _ in f1.scalars ]
                else:
                    raise ValueError
                
                ## numpy structured array
                data = np.zeros( shape=(nx,ny), dtype={'names':f1.scalars,'formats':dtypes} )
                
                ## dict for name change
                dnc = {
                "r'r'":'rI_rI',
                "u'u'":'uI_uI',
                "v'v'":'vI_vI',
                "w'w'":'wI_wI',
                "T'T'":'TI_TI',
                "p'p'":'pI_pI',
                "mu'mu'":'muI_muI',
                "u'v'":'uI_vI',
                "u'w'":'uI_wI',
                "v'w'":'vI_wI',
                "u'T'":'uI_TI',
                "v'T'":'vI_TI',
                "w'T'":'wI_TI',
                "tau'_xx":'tauI_xx',
                "tau'_xy":'tauI_xy',
                "tau'_xz":'tauI_xz',
                "tau'_yy":'tauI_yy',
                "tau'_yz":'tauI_yz',
                "tau'_zz":'tauI_zz',
                }
                
                ## populate structured array from EAS3 binary data file
                progress_bar = tqdm(total=f1.nt*f1.npar, ncols=100, desc='import eas3 Re fluct', leave=False)
                for scalar in f1.attr_param:
                    if (scalar in dnc):
                        s_ = dnc[scalar]
                    else:
                        s_ = scalar
                    tqdm.write(even_print(f"{scalar} --> {s_}",f'({nx},{ny})',s=True))
                    for jj in range(f1.ndim2):
                        if f1.accuracy == f1.IEEES:
                            packet = struct.unpack('!'+str(f1.ndim1)+'f',f1.f.read(4*f1.ndim1))[:]
                        else:
                            packet = struct.unpack('!'+str(f1.ndim1)+'d',f1.f.read(8*f1.ndim1))[:]
                        data[scalar][:,jj] = packet
                    progress_bar.update()
                progress_bar.close()
                
                ## re-dimensionalize by characteristic freestream quantities
                for scalar in data.dtype.names:
                    if scalar in ["u'u'","v'v'","w'w'","u'v'","u'w'","v'w'",]:
                        data[scalar] *= U_inf**2
                    elif scalar in ["r'r'",]:
                        data[scalar] *= rho_inf**2
                    elif scalar in ["T'T'",]:
                        data[scalar] *= T_inf**2
                    elif scalar in ["p'p'",]:
                        data[scalar] *= rho_inf**2 * U_inf**4
                    elif scalar in ["mu'mu'",]:
                        data[scalar] *= mu_inf**2
                    elif scalar in ["u'T'","v'T'","w'T'",]:
                        data[scalar] *= U_inf * T_inf
                    elif scalar in ["tau'_xx","tau'_xy","tau'_xz","tau'_yy","tau'_yz","tau'_zz",]:
                        data[scalar] *= mu_inf * U_inf / lchar
                    else:
                        raise ValueError(f"condition needed for redimensionalizing '{scalar}'")
                
                ## write structured array to ZTMD
                for scalar in data.dtype.names:
                    self.create_dataset(f'data/{dnc[scalar]}', data=data[scalar].T, chunks=None)
                
                ## derived
                uI_uI_rms = np.copy( np.sqrt(       data["u'u'"]  )                         )
                vI_vI_rms = np.copy( np.sqrt(       data["v'v'"]  )                         )
                wI_wI_rms = np.copy( np.sqrt(       data["w'w'"]  )                         )
                uI_vI_rms = np.copy( np.sqrt(np.abs(data["u'v'"]) ) * np.sign(data["u'v'"]) )
                uI_wI_rms = np.copy( np.sqrt(np.abs(data["u'w'"]) ) * np.sign(data["u'w'"]) )
                vI_wI_rms = np.copy( np.sqrt(np.abs(data["v'w'"]) ) * np.sign(data["v'w'"]) )
                
                uI_TI_rms = np.copy( np.sqrt(np.abs(data["u'T'"]) ) * np.sign(data["u'T'"]) )
                vI_TI_rms = np.copy( np.sqrt(np.abs(data["v'T'"]) ) * np.sign(data["v'T'"]) )
                wI_TI_rms = np.copy( np.sqrt(np.abs(data["w'T'"]) ) * np.sign(data["w'T'"]) )
                
                rI_rI_rms   = np.copy( np.sqrt( data["r'r'"]   ) )
                TI_TI_rms   = np.copy( np.sqrt( data["T'T'"]   ) )
                pI_pI_rms   = np.copy( np.sqrt( data["p'p'"]   ) )
                muI_muI_rms = np.copy( np.sqrt( data["mu'mu'"] ) )
                
                M_rms = np.copy( uI_uI_rms / np.sqrt(kappa * R * T) )
                
                ## clear structured array from memory
                data = None ; del data
                
                self.create_dataset( 'data/uI_uI_rms' , data=uI_uI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_vI_rms' , data=vI_vI_rms.T , chunks=None )
                self.create_dataset( 'data/wI_wI_rms' , data=wI_wI_rms.T , chunks=None )
                self.create_dataset( 'data/uI_vI_rms' , data=uI_vI_rms.T , chunks=None )
                self.create_dataset( 'data/uI_wI_rms' , data=uI_wI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_wI_rms' , data=vI_wI_rms.T , chunks=None )
                
                self.create_dataset( 'data/uI_TI_rms' , data=uI_TI_rms.T , chunks=None )
                self.create_dataset( 'data/vI_TI_rms' , data=vI_TI_rms.T , chunks=None )
                self.create_dataset( 'data/wI_TI_rms' , data=wI_TI_rms.T , chunks=None )
                
                self.create_dataset( 'data/rI_rI_rms'   , data=rI_rI_rms.T   , chunks=None )
                self.create_dataset( 'data/TI_TI_rms'   , data=TI_TI_rms.T   , chunks=None )
                self.create_dataset( 'data/pI_pI_rms'   , data=pI_pI_rms.T   , chunks=None )
                self.create_dataset( 'data/muI_muI_rms' , data=muI_muI_rms.T , chunks=None )
                
                self.create_dataset( 'data/M_rms' , data=M_rms.T , chunks=None )
        
        if fn_Fv_mean.exists(): ## Favre mean
            even_print('eas3 Fv mean',str(fn_Fv_mean.relative_to(Path())))
            with eas3(str(fn_Fv_mean),verbose=False) as f1:
                
                if (f1.t.shape[0]==1):
                    f1.total_avg_time = float(f1.t[0])
                else:
                    raise NotImplementedError
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                Fv_mean_total_avg_time       = f1.total_avg_time
                Fv_mean_total_avg_iter_count = f1.total_avg_iter_count
                Fv_mean_dt                   = Fv_mean_total_avg_time/Fv_mean_total_avg_iter_count
                
                self.attrs['Fv_mean_total_avg_time'] = Fv_mean_total_avg_time
                self.attrs['Fv_mean_total_avg_iter_count'] = Fv_mean_total_avg_iter_count
                self.attrs['Fv_mean_dt'] = Fv_mean_dt
                
                ## assert grid,lchar are same
                if hasattr(self,'lchar') and hasattr(f1,'lchar'):
                    np.testing.assert_allclose(self.lchar, f1.lchar, rtol=1e-12)
                if hasattr(self,'x') and hasattr(f1,'x'):
                    np.testing.assert_allclose(self.x, f1.x*lchar, rtol=1e-12)
                if hasattr(self,'y') and hasattr(f1,'y'):
                    np.testing.assert_allclose(self.y, f1.y*lchar, rtol=1e-12)
                
                # ===
                
                if (f1.scalars != f1.attr_param):
                    raise AssertionError
                if (f1.ndim3!=1):
                    raise AssertionError
                
                if (f1.accuracy == f1.IEEES):
                    dtypes = [ np.float32 for _ in f1.scalars ]
                if (f1.accuracy == f1.IEEED):
                    dtypes = [ np.float64 for _ in f1.scalars ]
                else:
                    raise ValueError
                
                ## numpy structured array
                data = np.zeros( shape=(nx,ny), dtype={'names':f1.scalars,'formats':dtypes} )
                
                ## dict for name change
                dnc = {
                'rho':'rho_Fv',
                'u':'u_Fv',
                'v':'v_Fv',
                'w':'w_Fv',
                'T':'T_Fv',
                'p':'p_Fv',
                'mu':'mu_Fv',
                'uu':'uu_Fv',
                'uv':'uv_Fv',
                }
                
                ## populate structured array from EAS3 binary data file
                progress_bar = tqdm(total=f1.nt*f1.npar, ncols=100, desc='import eas3 Fv mean', leave=False)
                for scalar in f1.attr_param:
                    if (scalar in dnc):
                        s_ = dnc[scalar]
                    else:
                        s_ = scalar
                    tqdm.write(even_print(f"{scalar} --> {s_}",f'({nx},{ny})',s=True))
                    for jj in range(f1.ndim2):
                        if f1.accuracy == f1.IEEES:
                            packet = struct.unpack('!'+str(f1.ndim1)+'f',f1.f.read(4*f1.ndim1))[:]
                        else:
                            packet = struct.unpack('!'+str(f1.ndim1)+'d',f1.f.read(8*f1.ndim1))[:]
                        data[scalar][:,jj] = packet
                    progress_bar.update()
                progress_bar.close()
                
                ## re-dimensionalize by characteristic freestream quantities
                for scalar in data.dtype.names:
                    if scalar in ["u","v","w",]:
                        data[scalar] *= U_inf
                    elif scalar in ["uu","uv",]:
                        data[scalar] *= U_inf**2
                    elif scalar in ["rho",]:
                        data[scalar] *= rho_inf
                    elif scalar in ["T",]:
                        data[scalar] *= T_inf
                    elif scalar in ["p",]:
                        data[scalar] *= rho_inf * U_inf**2
                    elif scalar in ["mu",]:
                        data[scalar] *= mu_inf
                    else:
                        raise ValueError(f"condition needed for redimensionalizing '{scalar}'")
                
                ## write
                for scalar in f1.attr_param:
                    self.create_dataset( f'data/{dnc[scalar]}' , data=data[scalar].T , chunks=None )
        
        if fn_Fv_fluct.exists(): ## Favre turbulent
            even_print('eas3 Fv fluct',str(fn_Fv_fluct.relative_to(Path())))
            with eas3(str(fn_Fv_fluct),verbose=False) as f1:
                
                if (f1.t.shape[0]==1):
                    f1.total_avg_time = float(f1.t[0])
                else:
                    raise NotImplementedError
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                Fv_fluct_total_avg_time       = f1.total_avg_time
                Fv_fluct_total_avg_iter_count = f1.total_avg_iter_count
                Fv_fluct_dt                   = Fv_fluct_total_avg_time/Fv_fluct_total_avg_iter_count
                
                self.attrs['Fv_fluct_total_avg_time'] = Fv_fluct_total_avg_time
                self.attrs['Fv_fluct_total_avg_iter_count'] = Fv_fluct_total_avg_iter_count
                self.attrs['Fv_fluct_dt'] = Fv_fluct_dt
                
                ## assert grid,lchar are same
                if hasattr(self,'lchar') and hasattr(f1,'lchar'):
                    np.testing.assert_allclose(self.lchar, f1.lchar, rtol=1e-12)
                if hasattr(self,'x') and hasattr(f1,'x'):
                    np.testing.assert_allclose(self.x, f1.x*lchar, rtol=1e-12)
                if hasattr(self,'y') and hasattr(f1,'y'):
                    np.testing.assert_allclose(self.y, f1.y*lchar, rtol=1e-12)
                
                # ===
                
                if (f1.scalars != f1.attr_param):
                    raise AssertionError
                if (f1.ndim3!=1):
                    raise AssertionError
                
                if (f1.accuracy == f1.IEEES):
                    dtypes = [ np.float32 for _ in f1.scalars ]
                if (f1.accuracy == f1.IEEED):
                    dtypes = [ np.float64 for _ in f1.scalars ]
                else:
                    raise ValueError
                
                ## numpy structured array
                data = np.zeros( shape=(nx,ny), dtype={'names':f1.scalars,'formats':dtypes} )
                
                ## dict for name change
                dnc = {
                "r r''r''":'r_rII_rII',
                "r u''u''":'r_uII_uII',
                "r v''v''":'r_vII_vII',
                "r w''_w''":'r_wII_wII',
                "r T''T''":'r_TII_TII',
                "r p''p''":'r_pII_pII',
                "r mu''mu''":'r_muII_muII',
                "r u''v''":'r_uII_vII',
                "r u''w''":'r_uII_wII',
                "r w''v''":'r_vII_wII',
                "r u''T''":'r_uII_TII',
                "r v''T''":'r_vII_TII',
                "r w''T''":'r_wII_TII',
                }
                
                ## populate structured array from EAS3 binary data file
                progress_bar = tqdm(total=f1.nt*f1.npar, ncols=100, desc='import eas3 Fv fluct', leave=False)
                for scalar in f1.attr_param:
                    if (scalar in dnc):
                        s_ = dnc[scalar]
                    else:
                        s_ = scalar
                    tqdm.write(even_print(f"{scalar} --> {s_}",f'({nx},{ny})',s=True))
                    for jj in range(f1.ndim2):
                        if f1.accuracy == f1.IEEES:
                            packet = struct.unpack('!'+str(f1.ndim1)+'f',f1.f.read(4*f1.ndim1))[:]
                        else:
                            packet = struct.unpack('!'+str(f1.ndim1)+'d',f1.f.read(8*f1.ndim1))[:]
                        data[scalar][:,jj] = packet
                    progress_bar.update()
                progress_bar.close()
                
                ## re-dimensionalize by characteristic freestream quantities
                for scalar in data.dtype.names:
                    if scalar in ["r r''r''",]:
                        data[scalar] *= rho_inf**3
                    elif scalar in ["r u''u''","r v''v''","r w''w''","r w''_w''","r u''v''","r u''w''","r w''v''",]:
                        data[scalar] *= rho_inf * U_inf**2
                    elif scalar in ["r u''T''","r v''T''","r w''T''",]:
                        data[scalar] *= rho_inf * U_inf * T_inf
                    elif scalar in ["r mu''mu''",]:
                        data[scalar] *= rho_inf * mu_inf**2
                    elif scalar in ["r p''p''",]:
                        data[scalar] *= rho_inf * (rho_inf * U_inf**2)**2
                    elif scalar in ["r T''T''",]:
                        data[scalar] *= rho_inf * T_inf**2
                    else:
                        raise ValueError(f"condition needed for redimensionalizing '{scalar}'")
                
                ## write structured array to ZTMD
                for scalar in data.dtype.names:
                    self.create_dataset(f'data/{dnc[scalar]}', data=data[scalar].T, chunks=None)
                
                ## derived
                r_uII_uII_rms = np.copy( np.sqrt(       data["r u''u''"]  )                             )
                r_vII_vII_rms = np.copy( np.sqrt(       data["r v''v''"]  )                             )
                r_wII_wII_rms = np.copy( np.sqrt(       data["r w''_w''"] )                             )
                r_uII_vII_rms = np.copy( np.sqrt(np.abs(data["r u''v''"]) ) * np.sign(data["r u''v''"]) )
                r_uII_wII_rms = np.copy( np.sqrt(np.abs(data["r u''w''"]) ) * np.sign(data["r u''w''"]) )
                r_vII_wII_rms = np.copy( np.sqrt(np.abs(data["r w''v''"]) ) * np.sign(data["r w''v''"]) )
                ## ... ρ·u″T″
                
                self.create_dataset( 'data/r_uII_uII_rms' , data=r_uII_uII_rms.T , chunks=None )
                self.create_dataset( 'data/r_vII_vII_rms' , data=r_vII_vII_rms.T , chunks=None )
                self.create_dataset( 'data/r_wII_wII_rms' , data=r_wII_wII_rms.T , chunks=None )
                self.create_dataset( 'data/r_uII_vII_rms' , data=r_uII_vII_rms.T , chunks=None )
                self.create_dataset( 'data/r_uII_wII_rms' , data=r_uII_wII_rms.T , chunks=None )
                self.create_dataset( 'data/r_vII_wII_rms' , data=r_vII_wII_rms.T , chunks=None )
                ## ... ρ·u″T″
                
                ## clear structured array from memory
                data = None ; del data
        
        if fn_turb_budget.exists(): ## turbulent budget
            even_print('eas3 turb budget',str(fn_turb_budget.relative_to(Path())))
            with eas3(str(fn_turb_budget),verbose=False) as f1:
                
                if (f1.t.shape[0]==1):
                    f1.total_avg_time = float(f1.t[0])
                else:
                    raise NotImplementedError
                
                ## duration over which avg was performed, iteration count and the sampling period of the avg
                turb_budget_total_avg_time       = f1.total_avg_time
                turb_budget_total_avg_iter_count = f1.total_avg_iter_count
                turb_budget_dt                   = turb_budget_total_avg_time/turb_budget_total_avg_iter_count
                
                self.attrs['turb_budget_total_avg_time'] = turb_budget_total_avg_time
                self.attrs['turb_budget_total_avg_iter_count'] = turb_budget_total_avg_iter_count
                self.attrs['turb_budget_dt'] = turb_budget_dt
                
                ## assert grid,lchar are same
                if hasattr(self,'lchar') and hasattr(f1,'lchar'):
                    np.testing.assert_allclose(self.lchar, f1.lchar, rtol=1e-12)
                if hasattr(self,'x') and hasattr(f1,'x'):
                    np.testing.assert_allclose(self.x, f1.x*lchar, rtol=1e-12)
                if hasattr(self,'y') and hasattr(f1,'y'):
                    np.testing.assert_allclose(self.y, f1.y*lchar, rtol=1e-12)
                
                # ===
                
                if (f1.scalars != f1.attr_param):
                    raise AssertionError
                if (f1.ndim3!=1):
                    raise AssertionError
                
                if (f1.accuracy == f1.IEEES):
                    dtypes = [ np.float32 for _ in f1.scalars ]
                if (f1.accuracy == f1.IEEED):
                    dtypes = [ np.float64 for _ in f1.scalars ]
                else:
                    raise ValueError
                
                ## numpy structured array
                names_ = [ s for s in f1.scalars if ( 'restart' not in s ) ]
                data = np.zeros( shape=(nx,ny), dtype={'names':names_,'formats':dtypes} )
                
                ## dict for name change
                dnc = {
                "prod.":'production',
                "dis.":'dissipation',
                "t-transp.":'turb_transport',
                "v-diff.":'visc_diffusion',
                "p-diff.":'p_diffusion',
                "p-dilat.":'p_dilatation',
                "rho-terms":'rho_terms',
                }
                
                ## populate structured array from EAS3 binary data file
                progress_bar = tqdm(total=7, ncols=100, desc='import eas3 turb budget', leave=False)
                for scalar in ["prod.","dis.","t-transp.","v-diff.","p-diff.","p-dilat.","rho-terms",]:
                    if (scalar in dnc):
                        s_ = dnc[scalar]
                    else:
                        s_ = scalar
                    tqdm.write(even_print(f"{scalar} --> {s_}",f'({nx},{ny})',s=True))
                    for jj in range(f1.ndim2):
                        if f1.accuracy == f1.IEEES:
                            packet = struct.unpack('!'+str(f1.ndim1)+'f',f1.f.read(4*f1.ndim1))[:]
                        else:
                            packet = struct.unpack('!'+str(f1.ndim1)+'d',f1.f.read(8*f1.ndim1))[:]
                        data[scalar][:,jj] = packet
                    progress_bar.update()
                progress_bar.close()
                
                ## re-dimensionalize by characteristic freestream quantities
                for scalar in data.dtype.names:
                    if scalar in ["prod.","t-transp.","p-diff.","p-dilat.","rho-terms",]:
                        data[scalar] *= U_inf**3 * rho_inf / lchar
                    elif scalar in ["dis.","v-diff.",]:
                        data[scalar] *= U_inf**2 * mu_inf / lchar**2
                    else:
                        raise ValueError(f"condition needed for redimensionalizing '{scalar}'")
                
                ## write structured array to ZTMD
                for scalar in data.dtype.names:
                    self.create_dataset(f'data/{dnc[scalar]}', data=data[scalar].T, chunks=None)
                
                ## derived
                dissipation = np.copy( data['dis.'] )
                Kolm_len = (nu**3 / np.abs(dissipation))**(1/4)
                self.create_dataset('data/Kolm_len', data=Kolm_len.T, chunks=None)
                
                ## clear structured array from memory
                data = None ; del data
        
        self.get_header(verbose=True)
        if verbose: print(72*'-')
        if verbose: print('total time : turbx.ztmd.compile_data_eas3() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def export_dict(self,**kwargs):
        '''
        pull all data from HDF5 container into memory and pack it into a dictionary
        - convenient for multi-case plotting scripts
        '''
        verbose   = kwargs.get('verbose',True)
        dsets     = kwargs.get('dsets',None)
        
        dd = {} ## the dict to return
        
        ## class-level ZTMD attrs
        attr_exclude_list = ['rank','comm','n_ranks','usingmpi','open_mode',
                             '_libver','_id','requires_wall_norm_interp',
                             'mod_avail_tqdm']
        for attr, val in self.__dict__.items():
            if (attr not in attr_exclude_list):
                
                ## throw error if not a routine type
                if isinstance(val, (int,np.int64,np.int32,float,str,dict,list,tuple,np.ndarray,bool,np.bool_,)):
                    pass
                elif (val is None):
                    pass
                else:
                    print(attr)
                    print(type(val))
                    raise TypeError
                
                dd[attr] = val
        
        ## HDF5 File Group attrs
        for attr, val in self.attrs.items():
            if (attr not in dd.keys()):
                dd[attr] = val
        
        ## Group: dims/
        for dsn in self['dims']:
            if (dsn not in dd.keys()):
                ds = self[f'dims/{dsn}']
                if (ds.ndim==0):
                    dd[dsn] = ds[()]
                elif (ds.ndim>0):
                    dd[dsn] = np.copy(ds[()])
                else:
                    raise ValueError
            else:
                #print(dsn)
                pass
        
        ## Group: data/
        for dsn in self['data']:
            if (dsn not in dd.keys()):
                if (dsets is None) or (dsn in dsets):
                    ds = self[f'data/{dsn}']
                    if (ds.ndim==2):
                        dd[dsn] = np.copy(ds[()].T)
                    else:
                        raise ValueError
            else:
                #print(dsn)
                pass
        
        ## Group: data_1Dx/
        for dsn in self['data_1Dx']:
            if (dsn not in dd.keys()):
                #if (dsets is not None) and (dsn in dsets):
                ds = self[f'data_1Dx/{dsn}']
                if (ds.ndim==1) or (ds.ndim==2):
                    dd[dsn] = np.copy(ds[()])
                else:
                    raise ValueError
            else:
                #print(dsn)
                pass
        
        if verbose:
            print(f'>>> {self.fname}')
        
        return dd
    
    # ===
    
    def calc_gradients(self, acc=6, edge_stencil='full', **kwargs):
        '''
        calculate spatial gradients of averaged quantities
        '''
        
        verbose  = kwargs.get('verbose',True)
        do_favre = kwargs.get('favre',True)
        
        if verbose: print('\n'+'ztmd.calc_gradients()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print('acc','%i'%(acc,))
        if verbose: even_print('edge_stencil','%s'%(edge_stencil,))
        if verbose: even_print('do_favre',str(do_favre))
        
        ## check
        if not hasattr(self,'rectilinear') and not hasattr(self,'curvilinear'):
            raise AssertionError('neither rectilinear nor curvilinear attr set')
        
        if hasattr(self,'rectilinear'):
            if self.rectilinear:
                if verbose: even_print('grid type','rectilinear')
        if hasattr(self,'curvilinear'):
            if self.curvilinear:
                if verbose: even_print('grid type','curvilinear')
        
        if (self.x.ndim==1) and (self.y.ndim==1):
            if hasattr(self,'rectilinear'):
                if not self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if self.curvilinear:
                    raise AssertionError
        elif (self.x.ndim==2) and (self.y.ndim==2):
            if hasattr(self,'rectilinear'):
                if self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if not self.curvilinear:
                    raise AssertionError
        else:
            raise ValueError
        
        # ===
        
        if self.curvilinear: ## get metric tensor 2D
            
            M = get_metric_tensor_2d(self.x, self.y, acc=acc, edge_stencil=edge_stencil, verbose=False)
            
            ddx_q1 = np.copy( M[:,:,0,0] ) ## ξ_x
            ddx_q2 = np.copy( M[:,:,1,0] ) ## η_x
            ddy_q1 = np.copy( M[:,:,0,1] ) ## ξ_y
            ddy_q2 = np.copy( M[:,:,1,1] ) ## η_y
            
            if verbose: even_print('ξ_x','%s'%str(ddx_q1.shape))
            if verbose: even_print('η_x','%s'%str(ddx_q2.shape))
            if verbose: even_print('ξ_y','%s'%str(ddy_q1.shape))
            if verbose: even_print('η_y','%s'%str(ddy_q2.shape))
            
            M = None; del M
            
            ## the 'computational' grid (unit Cartesian)
            #x_comp = np.arange(nx, dtype=np.float64)
            #y_comp = np.arange(ny, dtype=np.float64)
            x_comp = 1.
            y_comp = 1.
        
        print(72*'-')
        
        # === get gradients of [u,v,p,T,ρ]
        
        if ('data/u' in self):
            
            u = np.copy( self['data/u'][()].T )
            
            if self.rectilinear:
                ddx_u = gradient(u, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_u = gradient(u, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_u_comp = gradient(u, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_u_comp = gradient(u, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_u      = ddx_u_comp*ddx_q1 + ddy_u_comp*ddx_q2
                ddy_u      = ddx_u_comp*ddy_q1 + ddy_u_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_u' in self): del self['data/ddx_u']
            self.create_dataset('data/ddx_u', data=ddx_u.T, chunks=None)
            
            if ('data/ddy_u' in self): del self['data/ddy_u']
            self.create_dataset('data/ddy_u', data=ddy_u.T, chunks=None)
            
            if verbose: even_print('ddx[u]','%s'%str(ddx_u.shape))
            if verbose: even_print('ddy[u]','%s'%str(ddy_u.shape))
        
        if ('data/v' in self):
            
            v = np.copy( self['data/v'][()].T )
            
            if self.rectilinear:
                ddx_v = gradient(v, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_v = gradient(v, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_v_comp = gradient(v, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_v_comp = gradient(v, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_v      = ddx_v_comp*ddx_q1 + ddy_v_comp*ddx_q2
                ddy_v      = ddx_v_comp*ddy_q1 + ddy_v_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_v' in self): del self['data/ddx_v']
            self.create_dataset('data/ddx_v', data=ddx_v.T, chunks=None)
            
            if ('data/ddy_v' in self): del self['data/ddy_v']
            self.create_dataset('data/ddy_v', data=ddy_v.T, chunks=None)
            
            if verbose: even_print('ddx[v]','%s'%str(ddx_v.shape))
            if verbose: even_print('ddy[v]','%s'%str(ddy_v.shape))
        
        if ('data/p' in self):
            
            p = np.copy( self['data/p'][()].T )
            
            if self.rectilinear:
                ddx_p = gradient(p, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_p = gradient(p, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_p_comp = gradient(p, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_p_comp = gradient(p, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_p      = ddx_p_comp*ddx_q1 + ddy_p_comp*ddx_q2
                ddy_p      = ddx_p_comp*ddy_q1 + ddy_p_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_p' in self): del self['data/ddx_p']
            dset = self.create_dataset('data/ddx_p', data=ddx_p.T, chunks=None)
            
            if ('data/ddy_p' in self): del self['data/ddy_p']
            dset = self.create_dataset('data/ddy_p', data=ddy_p.T, chunks=None)
            
            if verbose: even_print('ddx[p]','%s'%str(ddx_p.shape))
            if verbose: even_print('ddy[p]','%s'%str(ddy_p.shape))
        
        if ('data/T' in self):
            
            T = np.copy( self['data/T'][()].T )
            
            if self.rectilinear:
                ddx_T = gradient(T, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_T = gradient(T, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_T_comp = gradient(T, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_T_comp = gradient(T, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_T      = ddx_T_comp*ddx_q1 + ddy_T_comp*ddx_q2
                ddy_T      = ddx_T_comp*ddy_q1 + ddy_T_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_T' in self): del self['data/ddx_T']
            dset = self.create_dataset('data/ddx_T', data=ddx_T.T, chunks=None)
            
            if ('data/ddy_T' in self): del self['data/ddy_T']
            dset = self.create_dataset('data/ddy_T', data=ddy_T.T, chunks=None)
            
            if verbose: even_print('ddx[T]','%s'%str(ddx_T.shape))
            if verbose: even_print('ddy[T]','%s'%str(ddy_T.shape))
        
        if ('data/rho' in self):
            
            r = np.copy( self['data/rho'][()].T )
            
            if self.rectilinear:
                ddx_r = gradient(r, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_r = gradient(r, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_r_comp = gradient(r, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_r_comp = gradient(r, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_r      = ddx_r_comp*ddx_q1 + ddy_r_comp*ddx_q2
                ddy_r      = ddx_r_comp*ddy_q1 + ddy_r_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_r' in self): del self['data/ddx_r']
            dset = self.create_dataset('data/ddx_r', data=ddx_r.T, chunks=None)
            
            if ('data/ddy_r' in self): del self['data/ddy_r']
            dset = self.create_dataset('data/ddy_r', data=ddy_r.T, chunks=None)
            
            if verbose: even_print('ddx[ρ]','%s'%str(ddx_r.shape))
            if verbose: even_print('ddy[ρ]','%s'%str(ddy_r.shape))
        
        # === vorticity
        
        ## z-vorticity :: ω_z
        vort_z = ddx_v - ddy_u
        
        if ('data/vort_z' in self): del self['data/vort_z']
        self.create_dataset('data/vort_z', data=vort_z.T, chunks=None)
        if verbose: even_print('ω_z','%s'%str(vort_z.shape))
        
        ## divergence (in xy-plane)
        div_xy = ddx_u + ddy_v
        
        if ('data/div_xy' in self): del self['data/div_xy']
        self.create_dataset('data/div_xy', data=div_xy.T, chunks=None)
        if verbose: even_print('div_xy','%s'%str(div_xy.shape))
        
        # === 
        
        if ('data/utang' in self):
            
            utang = np.copy( self['data/utang'][()].T )
            
            if self.rectilinear:
                ddx_utang = gradient(utang, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_utang = gradient(utang, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_utang_comp = gradient(utang, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_utang_comp = gradient(utang, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_utang      = ddx_utang_comp*ddx_q1 + ddy_utang_comp*ddx_q2
                ddy_utang      = ddx_utang_comp*ddy_q1 + ddy_utang_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_utang' in self): del self['data/ddx_utang']
            dset = self.create_dataset('data/ddx_utang', data=ddx_utang.T, chunks=None)
            if verbose: even_print('ddx[utang]','%s'%str(ddx_utang.shape))
            
            if ('data/ddy_utang' in self): del self['data/ddy_utang']
            dset = self.create_dataset('data/ddy_utang', data=ddy_utang.T, chunks=None)
            if verbose: even_print('ddy[utang]','%s'%str(ddy_utang.shape))
        
        if ('data/unorm' in self):
            
            unorm = np.copy( self['data/unorm'][()].T )
            
            if self.rectilinear:
                ddx_unorm = gradient(unorm, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_unorm = gradient(unorm, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            elif self.curvilinear:
                ddx_unorm_comp = gradient(unorm, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                ddy_unorm_comp = gradient(unorm, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                ddx_unorm      = ddx_unorm_comp*ddx_q1 + ddy_unorm_comp*ddx_q2
                ddy_unorm      = ddx_unorm_comp*ddy_q1 + ddy_unorm_comp*ddy_q2
            else:
                raise ValueError
            
            if ('data/ddx_unorm' in self): del self['data/ddx_unorm']
            dset = self.create_dataset('data/ddx_unorm', data=ddx_unorm.T, chunks=None)
            if verbose: even_print('ddx[unorm]','%s'%str(ddx_unorm.shape))
            
            if ('data/ddy_unorm' in self): del self['data/ddy_unorm']
            dset = self.create_dataset('data/ddy_unorm', data=ddy_unorm.T, chunks=None)
            if verbose: even_print('ddy[unorm]','%s'%str(ddy_unorm.shape))
        
        # === Favre
        
        if do_favre:
            
            print(72*'-')
            
            if ('data/u_Fv' in self):
                
                u_Fv = np.copy( self['data/u_Fv'][()].T )
                
                if self.rectilinear:
                    ddx_u_Fv = gradient(u_Fv, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddy_u_Fv = gradient(u_Fv, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                elif self.curvilinear:
                    ddx_u_Fv_comp = gradient(u_Fv, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddy_u_Fv_comp = gradient(u_Fv, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddx_u_Fv      = ddx_u_Fv_comp*ddx_q1 + ddy_u_Fv_comp*ddx_q2
                    ddy_u_Fv      = ddx_u_Fv_comp*ddy_q1 + ddy_u_Fv_comp*ddy_q2
                else:
                    raise ValueError
                
                if ('data/ddx_u_Fv' in self): del self['data/ddx_u_Fv']
                self.create_dataset('data/ddx_u_Fv', data=ddx_u_Fv.T, chunks=None)
                
                if ('data/ddy_u_Fv' in self): del self['data/ddy_u_Fv']
                self.create_dataset('data/ddy_u_Fv', data=ddy_u_Fv.T, chunks=None)
                
                if verbose: even_print('ddx[u_Fv]','%s'%str(ddx_u_Fv.shape))
                if verbose: even_print('ddy[u_Fv]','%s'%str(ddy_u_Fv.shape))
            
            if ('data/v_Fv' in self):
                
                v_Fv = np.copy( self['data/v_Fv'][()].T )
                
                if self.rectilinear:
                    ddx_v_Fv = gradient(v_Fv, self.x, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddy_v_Fv = gradient(v_Fv, self.y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                elif self.curvilinear:
                    ddx_v_Fv_comp = gradient(v_Fv, x_comp, axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddy_v_Fv_comp = gradient(v_Fv, y_comp, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddx_v_Fv      = ddx_v_Fv_comp*ddx_q1 + ddy_v_Fv_comp*ddx_q2
                    ddy_v_Fv      = ddx_v_Fv_comp*ddy_q1 + ddy_v_Fv_comp*ddy_q2
                else:
                    raise ValueError
                
                if ('data/ddx_v_Fv' in self): del self['data/ddx_v_Fv']
                self.create_dataset('data/ddx_v_Fv', data=ddx_v_Fv.T, chunks=None)
                
                if ('data/ddy_v_Fv' in self): del self['data/ddy_v_Fv']
                self.create_dataset('data/ddy_v_Fv', data=ddy_v_Fv.T, chunks=None)
                
                if verbose: even_print('ddx[v_Fv]','%s'%str(ddx_v_Fv.shape))
                if verbose: even_print('ddy[v_Fv]','%s'%str(ddy_v_Fv.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_gradients() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_psvel(self,**kwargs):
        '''
        calculate pseudovelocity, wall-normal cumulative integration of (-) z-vorticity
        '''
        
        verbose  = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_psvel()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not ('data/vort_z' in self):
            raise ValueError('data/vort_z not in ztmd')
        
        vort_z = np.copy( self['data/vort_z'][()].T )
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T ) ## 2D
            y = np.copy( self['dims/y'][()].T ) ## 2D
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            utang = np.copy( self['data/utang'][()].T )
            unorm = np.copy( self['data/unorm'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        ## pseudo-velocity is a cumulative integration of (-) z-vorticity
        psvel = np.zeros(shape=(nx,ny), dtype=np.float64)
        for i in range(nx):
            psvel_     = sp.integrate.cumulative_trapezoid(-1*vort_z[i,:], y_, initial=0.)
            psvel[i,:] = psvel_
        
        if ('data/psvel' in self):
            del self['data/psvel']
        self.create_dataset('data/psvel', data=psvel.T, chunks=None)
        if verbose: even_print('data/psvel','%s'%str(psvel.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_psvel() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_wall_quantities(self, acc=6, edge_stencil='full', **kwargs):
        '''
        get 1D wall quantities
        -----
        - [ ρ_wall, ν_wall, μ_wall, T_wall ]
        - τ_wall = μ_wall·ddn[utang] :: [kg/(m·s)]·[m/s]/[m] = [kg/(m·s²)] = [N/m²] = [Pa]
        - u_τ = (τ_wall/ρ_wall)^(1/2)
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_wall_quantities()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print('acc','%i'%(acc,))
        if verbose: even_print('edge_stencil','%s'%(edge_stencil,))
        
        ## check
        if (self.x.ndim==1) and (self.y.ndim==1):
            if hasattr(self,'rectilinear'):
                if not self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if self.curvilinear:
                    raise AssertionError
        elif (self.x.ndim==2) and (self.y.ndim==2):
            if hasattr(self,'rectilinear'):
                if self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if not self.curvilinear:
                    raise AssertionError
        else:
            raise ValueError
        
        # ===
        
        if self.curvilinear:
            
            if self.requires_wall_norm_interp:
                
                gndata = 'data_2Dw' ## group name for 2D data (interpolated)
                gndims = 'dims_2Dw' ## group name for 2D dims (interpolated)
                
                ## wall-normal interpolation coordinates (2D)
                x_wn = np.copy( self[f'{gndata}/x'][()].T )
                y_wn = np.copy( self[f'{gndata}/y'][()].T )
                s_wn = np.copy( self[f'{gndata}/wall_distance'][()].T )
            
            else:
                
                gndata = 'data' ## group name for 2D data
                gndims = 'dims' ## group name for 2D dims
                
                x_wn = np.copy( self[f'dims/x'][()].T )
                y_wn = np.copy( self[f'dims/y'][()].T )
                s_wn = np.copy( self[f'dims/snorm'][()] )
                #s_wn = np.broadcast_to(s_wn, (self.nx,self.ny))
            
            if (f'{gndata}/utang' not in self):
                raise AssertionError(f'{gndata}/utang not present')
            
            ## wall-normal interpolated scalars (2D)
            utang_wn  = np.copy( self[f'{gndata}/utang'][()].T  )
            T_wn      = np.copy( self[f'{gndata}/T'][()].T      )
            vort_z_wn = np.copy( self[f'{gndata}/vort_z'][()].T )
        
        # === get ρ_wall, ν_wall, μ_wall, T_wall
        
        rho = np.copy( self['data/rho'][()].T )
        rho_wall = np.copy( rho[:,0] )
        if ('data_1Dx/rho_wall' in self): del self['data_1Dx/rho_wall']
        dset = self.create_dataset('data_1Dx/rho_wall', data=rho_wall, chunks=None)
        #dset.attrs['dimensional'] = False
        #dset.attrs['unit'] = '[kg/m³]'
        if verbose: even_print('data_1Dx/rho_wall','%s'%str(rho_wall.shape))
        
        nu = np.copy( self['data/nu'][()].T )
        nu_wall = np.copy( nu[:,0] )
        if ('data_1Dx/nu_wall' in self): del self['data_1Dx/nu_wall']
        dset = self.create_dataset('data_1Dx/nu_wall', data=nu_wall, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[m²/s]'
        if verbose: even_print('data_1Dx/nu_wall','%s'%str(nu_wall.shape))
        
        mu = np.copy( self['data/mu'][()].T )
        mu_wall = np.copy( mu[:,0] )
        if ('data_1Dx/mu_wall' in self): del self['data_1Dx/mu_wall']
        dset = self.create_dataset('data_1Dx/mu_wall', data=mu_wall, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[kg/(m·s)]'
        if verbose: even_print('data_1Dx/mu_wall','%s'%str(mu_wall.shape))
        
        T = np.copy( self['data/T'][()].T )
        T_wall = np.copy( T[:,0] )
        if ('data_1Dx/T_wall' in self): del self['data_1Dx/T_wall']
        dset = self.create_dataset('data_1Dx/T_wall', data=T_wall, chunks=None)
        #dset.attrs['dimensional'] = False
        #dset.attrs['unit'] = '[K]'
        if verbose: even_print('data_1Dx/T_wall','%s'%str(T_wall.shape))
        
        # === get wall ddn[]
        
        if self.rectilinear:
            
            ddy_u = np.copy( self['data/ddy_u'][()].T )
        
        elif self.curvilinear:
            
            if True:
                
                if (s_wn.ndim==2): ## wall-normal distance (s_norm) is a 2D field
                    
                    ddn_utang  = np.zeros((self.nx,self.ny), dtype=np.float64) ## dimensional [m/s]/[m] = [1/s]
                    ddn_vort_z = np.zeros((self.nx,self.ny), dtype=np.float64)
                    
                    progress_bar = tqdm(total=self.nx, ncols=100, desc='get ddn[]', leave=False, file=sys.stdout)
                    for i in range(self.nx):
                        ddn_utang[i,:]  = gradient(utang_wn[i,:]  , s_wn[i,:], axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                        ddn_vort_z[i,:] = gradient(vort_z_wn[i,:] , s_wn[i,:], axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
                        progress_bar.update()
                    progress_bar.close()
                
                elif (s_wn.ndim==1): ## wall-normal distance (s_norm) is a 1D vector
                    
                    ddn_utang  = gradient(utang_wn  , s_wn, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                    ddn_vort_z = gradient(vort_z_wn , s_wn, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
                
                if (f'{gndata}/ddn_utang' in self): del self[f'{gndata}/ddn_utang']
                dset = self.create_dataset(f'{gndata}/ddn_utang', data=ddn_utang.T, chunks=None)
                #dset.attrs['dimensional'] = True; dset.attrs['unit'] = '1/s'
                
                if (f'{gndata}/ddn_vort_z' in self): del self[f'{gndata}/ddn_vort_z']
                dset = self.create_dataset(f'{gndata}/ddn_vort_z', data=ddn_vort_z.T, chunks=None)
                
                if ('data_1Dx/ddn_utang_wall' in self): del self['data_1Dx/ddn_utang_wall']
                dset = self.create_dataset('data_1Dx/ddn_utang_wall', data=ddn_utang[:,0], chunks=None)
                #dset.attrs['dimensional'] = True; dset.attrs['unit'] = '1/s'
            
            else:
                
                ddn_utang  = np.copy( self[f'{gndata}/ddn_utang'][()].T  )
                ddn_vort_z = np.copy( self[f'{gndata}/ddn_vort_z'][()].T )
        
        else:
            raise ValueError
        
        # === calculate τ_wall & u_τ
        
        ## wall shear stress τ_wall
        if self.rectilinear:
            tau_wall = np.copy( mu_wall * ddy_u[:,0] )
        elif self.curvilinear:
            tau_wall = np.copy( mu_wall * ddn_utang[:,0] )
        else:
            raise ValueError
        
        if ('data_1Dx/tau_wall' in self): del self['data_1Dx/tau_wall']
        dset = self.create_dataset('data_1Dx/tau_wall', data=tau_wall, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[N/m²]' ## = [kg/(m·s²)] = [N/m²] = [Pa]
        if verbose: even_print('data_1Dx/tau_wall','%s'%str(tau_wall.shape))
        
        ## friction velocity u_τ [m/s]
        u_tau = np.copy( np.sqrt( tau_wall / rho_wall ) )
        
        if ('data_1Dx/u_tau' in self): del self['data_1Dx/u_tau']
        dset = self.create_dataset('data_1Dx/u_tau', data=u_tau, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[m/s]'
        if verbose: even_print('data_1Dx/u_tau','%s'%str(u_tau.shape))
        
        # === inner scales: length, velocity & time
        
        sc_u_in = np.copy( u_tau              )
        sc_l_in = np.copy( nu_wall / u_tau    )
        sc_t_in = np.copy( nu_wall / u_tau**2 )
        np.testing.assert_allclose(sc_t_in, sc_l_in/sc_u_in, rtol=1e-6, atol=1e-12)
        
        if ('data_1Dx/sc_u_in' in self): del self['data_1Dx/sc_u_in']
        dset = self.create_dataset('data_1Dx/sc_u_in', data=sc_u_in, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[m/s]'
        if verbose: even_print('data_1Dx/sc_u_in','%s'%str(sc_u_in.shape))
        
        if ('data_1Dx/sc_l_in' in self): del self['data_1Dx/sc_l_in']
        dset = self.create_dataset('data_1Dx/sc_l_in', data=sc_l_in, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[m]'
        if verbose: even_print('data_1Dx/sc_l_in','%s'%str(sc_l_in.shape))
        
        if ('data_1Dx/sc_t_in' in self): del self['data_1Dx/sc_t_in']
        dset = self.create_dataset('data_1Dx/sc_t_in', data=sc_t_in, chunks=None)
        #dset.attrs['dimensional'] = True
        #dset.attrs['unit'] = '[s]'
        if verbose: even_print('data_1Dx/sc_t_in','%s'%str(sc_t_in.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_wall_quantities() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === curvilinear / curved TBL specific
    
    def calc_s_wall(self,**kwargs):
        '''
        calculate wall/top path length 's' (numerically integrated, not continuous!)
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_s_wall()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if self.rectilinear:
            
            s_wall = np.copy( self.x - self.x.min() )
            s_top  = np.copy( self.x - self.x.min() )
        
        else: ## curvilinear
            
            ds_wall = np.sqrt( np.diff(self.x[:,0])**2 + np.diff(self.y[:,0])**2 )
            s_wall  = np.cumsum(np.concatenate([[0.],ds_wall]))
            
            ds_top = np.sqrt( np.diff(self.x[:,-1])**2 + np.diff(self.y[:,-1])**2 )
            s_top  = np.cumsum(np.concatenate([[0.],ds_top]))
        
        if verbose: even_print( 's_wall/lchar max' , '%0.8f'%(s_wall.max()/self.lchar) )
        if verbose: even_print( 's_top/lchar max'  , '%0.8f'%(s_top.max()/self.lchar)  )
        
        if ('dims/s_wall' in self):
            del self['dims/s_wall']
        dset = self.create_dataset('dims/s_wall', data=s_wall, chunks=None)
        if verbose: even_print( 'dims/s_wall', str(dset.shape) )
        
        if ('dims/s_top' in self):
            del self['dims/s_top']
        dset = self.create_dataset('dims/s_top', data=s_top, chunks=None)
        if verbose: even_print( 'dims/s_top', str(dset.shape) )
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_s_wall() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_geom_data(self, fn_dat=None, **kwargs):
        '''
        add geom data (from pickled .dat, usually from tgg)
        - dims/stang : (nx,)
        - dims/snorm : (ny,)
        - dims/crv_R : (nx,)
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.add_geom_data()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_dat):
            raise FileNotFoundError('file does not exist: %s'%str(fn_dat))
        
        ## open data file from tgg
        with open(fn_dat,'rb') as f:
            dd = pickle.load(f)
        
        if ('xy2d' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain xy2d')
        
        ## check if consistent with [xy] grid in ref file
        xy2d = np.copy( dd['xy2d'] )
        
        if (self.x.ndim==2) and (self.y.ndim==2):
            np.testing.assert_allclose(xy2d[:,:,0], self.x/self.lchar, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,:,1], self.y/self.lchar, rtol=1e-14, atol=1e-14)
        elif (self.x.ndim==1) and (self.y.ndim==1):
            np.testing.assert_allclose(xy2d[:,0,0], self.x/self.lchar, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[0,:,1], self.y/self.lchar, rtol=1e-14, atol=1e-14)
        else:
            raise ValueError
        
        if verbose: even_print('check passed', 'x grid')
        if verbose: even_print('check passed', 'y grid')
        
        ## key names for 'dims' group
        kn_dims = ['stang', 'snorm', 'crv_R'] 
        for k in kn_dims:
            if (k in dd.keys()):
                dsn = f'dims/{k}'
                if (dd[k] is None):
                    continue
                data = np.copy(dd[k])
                data *= self.lchar ## re-dimensionalize (ztmd is dimensional)
                if (dsn in self): del self[dsn]
                ds = self.create_dataset(dsn, data=data, chunks=None)
                if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        kn_dims = ['z', 's', 's_top', 's_wall', 'R_min', 'path_len', 'W', 'H'] 
        for k in kn_dims:
            if (k in dd.keys()):
                dsn = f'dims/{k}'
                if (dd[k] is None):
                    continue
                if isinstance(dd[k], np.ndarray):
                    data = np.copy(dd[k])
                    data *= self.lchar ## re-dimensionalize (ztmd is dimensional)
                if isinstance(dd[k], (int,float)):
                    data = float(dd[k])
                    data *= self.lchar ## re-dimensionalize (ztmd is dimensional)
                if (dsn in self): del self[dsn]
                ds = self.create_dataset(dsn, data=data, chunks=None)
                if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        kn_dims = ['curve_arc_angle']
        for k in kn_dims:
            if (k in dd.keys()):
                dsn = f'dims/{k}'
                if (dd[k] is None):
                    continue
                if (dsn in self): del self[dsn]
                ds = self.create_dataset(dsn, data=dd[k], chunks=None)
                if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        ## check
        if self.rectilinear:
            
            stang_ = self[f'dims/stang'][()]
            x_ = self[f'dims/x'][()]
            np.testing.assert_allclose(stang_, x_-x_.min(), rtol=1e-14, atol=1e-14)
            
            snorm_ = self[f'dims/snorm'][()]
            y_ = self[f'dims/y'][()]
            np.testing.assert_allclose(snorm_, y_-y_.min(), rtol=1e-14, atol=1e-14)
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.add_geom_data() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_csys_vecs_xy(self, fn_dat=None, **kwargs):
        '''
        add csys data (from pickled .dat, usually from tgg)
        - csys/vtang : (nx,ny,2)
        - csys/vnorm : (nx,ny,2)
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.add_csys_vecs_xy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not os.path.isfile(fn_dat):
            raise FileNotFoundError('file does not exist: %s'%str(fn_dat))
        
        ## open data file from tgg
        with open(fn_dat,'rb') as f:
            dd = pickle.load(f)
        
        if ('xy2d' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain xy2d')
        if ('vtang' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain vtang')
        if ('vnorm' not in dd.keys()):
            raise ValueError(f'file {fn_dat} does not contain vnorm')
        
        xy2d  = dd['xy2d']
        vtang = dd['vtang']
        vnorm = dd['vnorm']
        
        #wall_distance = dd['wall_distance']
        #s_wall_2d     = dd['s_wall_2d'] ## curve path length of point on wall (nx,ny)
        #p_wall_2d     = dd['p_wall_2d'] ## projection point on wall (nx,ny,2)
        
        if (self.x.ndim==2) and (self.y.ndim==2):
            np.testing.assert_allclose(xy2d[:,:,0], self.x/self.lchar, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[:,:,1], self.y/self.lchar, rtol=1e-14, atol=1e-14)
        elif (self.x.ndim==1) and (self.y.ndim==1):
            np.testing.assert_allclose(xy2d[:,0,0], self.x/self.lchar, rtol=1e-14, atol=1e-14)
            np.testing.assert_allclose(xy2d[0,:,1], self.y/self.lchar, rtol=1e-14, atol=1e-14)
        else:
            raise ValueError
        
        if verbose: even_print('check passed', 'x grid')
        if verbose: even_print('check passed', 'y grid')
        
        dd = None; del dd
        xy2d = None; del xy2d
        
        ## re-dimensionalize
        #wall_distance *= self.lchar
        
        #if not (wall_distance.shape == (self.nx,self.ny)):
        #    raise ValueError('wall_distance.shape != (self.nx,self.ny)')
        if not (vtang.shape == (self.nx,self.ny,2)):
            raise ValueError('vtang.shape != (self.nx,self.ny,2)')
        if not (vnorm.shape == (self.nx,self.ny,2)):
            raise ValueError('vnorm.shape != (self.nx,self.ny,2)')
        
        ## write wall distance scalar (nx,ny)
        #if ('data/wall_distance' in self): del self['data/wall_distance']
        #self.create_dataset('data/wall_distance', data=wall_distance.T, chunks=None)
        
        ## write wall normal / tangent basis vectors
        if ('csys/vtang' in self): del self['csys/vtang']
        dset = self.create_dataset('csys/vtang', data=vtang, chunks=None)
        if verbose: even_print('csys/vtang',str(dset.shape))
        
        if ('csys/vnorm' in self): del self['csys/vnorm']
        dset = self.create_dataset('csys/vnorm', data=vnorm, chunks=None)
        if verbose: even_print('csys/vnorm',str(dset.shape))
        
        ## write continuous wall point coordinate & wall path length
        #if ('csys/s_wall_2d' in self): del self['csys/s_wall_2d']
        #self.create_dataset('csys/s_wall_2d', data=s_wall_2d, chunks=None)
        #if ('csys/p_wall_2d' in self): del self['csys/p_wall_2d']
        #self.create_dataset('csys/p_wall_2d', data=p_wall_2d, chunks=None)
        
        ## check
        if self.rectilinear:
            
            vtang_        = np.zeros((self.nx,self.ny,2),dtype=np.float64)
            vtang_[:,:,:] = np.array([1,0],dtype=np.float64)
            np.testing.assert_allclose(vtang_, vtang, rtol=1e-14, atol=1e-14)
            
            vnorm_        = np.zeros((self.nx,self.ny,2),dtype=np.float64)
            vnorm_[:,:,:] = np.array([0,1],dtype=np.float64)
            np.testing.assert_allclose(vnorm_, vnorm, rtol=1e-14, atol=1e-14)
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.add_csys_vecs_xy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vel_tangnorm(self, **kwargs):
        '''
        add tangent & normal velocity [utang,unorm] to file
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_vel_tangnorm()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not ('data/u' in self):
            raise ValueError('data/u not in hdf5')
        if not ('data/v' in self):
            raise ValueError('data/v not in hdf5')
        if not ('csys/vtang' in self):
            raise ValueError('csys/vtang not in hdf5')
        if not ('csys/vnorm' in self):
            raise ValueError('csys/vnorm not in hdf5')
        if not (self.open_mode=='a') or (self.open_mode=='w'):
            raise ValueError('not able to write to hdf5 file')
        
        ## read unit vectors (wall tangent, wall norm) from HDF5
        vtang = np.copy( self['csys/vtang'][()] )
        vnorm = np.copy( self['csys/vnorm'][()] )
        
        # === Reynolds
        
        if ('data/u' in self) and ('data/v' in self):
            
            ## read 2D velocities
            u  = np.copy( self['data/u'][()].T )
            v  = np.copy( self['data/v'][()].T )
            uv = np.stack((u,v), axis=-1)
            
            umag1 = np.copy( np.sqrt( u**2 + v**2 ) )
            
            ## inner product of velocity vector and basis vector (csys transform)
            utang = np.einsum('xyi,xyi->xy', vtang, uv)
            unorm = np.einsum('xyi,xyi->xy', vnorm, uv)
            
            umag2 = np.copy( np.sqrt( utang**2 + unorm**2 ) )
            
            if (umag1.dtype==np.dtype(np.float32)):
                np.testing.assert_allclose(umag1, umag2, rtol=1e-6) ## single precision
            elif (umag2.dtype==np.dtype(np.float64)):
                np.testing.assert_allclose(umag1, umag2, rtol=1e-12) ## double precision
            else:
                raise ValueError
            
            # if self.get('data/u').attrs['dimensional']:
            #     raise AssertionError('u is dimensional')
            # if self.get('data/v').attrs['dimensional']:
            #     raise AssertionError('v is dimensional')
            
            if ('data/utang' in self): del self['data/utang']
            dset = self.create_dataset('data/utang', data=utang.T, chunks=None)
            #dset.attrs['dimensional'] = False
            if verbose: even_print('utang','%s'%str(utang.shape))
            
            if ('data/unorm' in self): del self['data/unorm']
            dset = self.create_dataset('data/unorm', data=unorm.T, chunks=None)
            #dset.attrs['dimensional'] = False
            if verbose: even_print('unorm','%s'%str(unorm.shape))
            
            ## assert that in rectilinear case that u==utang & v==unorm
            if self.rectilinear:
                np.testing.assert_allclose(u, utang, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(v, unorm, rtol=1e-14, atol=1e-14)
        
        # === Favre
        
        if ('data/u_Fv' in self) and ('data/v_Fv' in self):
            
            ## read 2D velocities
            u_Fv     = np.copy( self['data/u_Fv'][()].T )
            v_Fv     = np.copy( self['data/v_Fv'][()].T )
            uv_Fv    = np.stack((u_Fv,v_Fv), axis=-1)
            umag1_Fv = np.copy( np.sqrt( u_Fv**2 + v_Fv**2 ) )
            
            ## inner product of velocity vector and basis vector (csys transform)
            utang_Fv = np.einsum('xyi,xyi->xy', vtang, uv_Fv)
            unorm_Fv = np.einsum('xyi,xyi->xy', vnorm, uv_Fv)
            
            umag2_Fv = np.copy( np.sqrt( utang_Fv**2 + unorm_Fv**2 ) )
            
            if (umag1_Fv.dtype==np.dtype(np.float32)):
                np.testing.assert_allclose(umag1_Fv, umag2_Fv, rtol=1e-6) ## single precision
            elif (umag2_Fv.dtype==np.dtype(np.float64)):
                np.testing.assert_allclose(umag1_Fv, umag2_Fv, rtol=1e-12) ## double precision
            else:
                raise ValueError
            
            if ('data/utang_Fv' in self): del self['data/utang_Fv']
            dset = self.create_dataset('data/utang_Fv', data=utang_Fv.T, chunks=None)
            if verbose: even_print('utang_Fv','%s'%str(utang_Fv.shape))
            
            if ('data/unorm_Fv' in self): del self['data/unorm_Fv']
            dset = self.create_dataset('data/unorm_Fv', data=unorm_Fv.T, chunks=None)
            #dset.attrs['dimensional'] = False
            if verbose: even_print('unorm_Fv','%s'%str(unorm_Fv.shape))
            
            ## assert that in rectilinear case that u==utang & v==unorm
            if self.rectilinear:
                np.testing.assert_allclose(u_Fv, utang_Fv, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(v_Fv, unorm_Fv, rtol=1e-14, atol=1e-14)
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_vel_tangnorm() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_vel_tangnorm_mean_removed(self, **kwargs):
        '''
        calculate utangI_utangI, unormI_unormI
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_vel_tangnorm_mean_removed()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        #if not ('data/wall_distance' in self):
        #    raise ValueError('data/wall_distance not in hdf5')
        if not ('dims/snorm' in self):
            raise ValueError('dims/snorm not in hdf5')
        if not ('data/uI_uI' in self):
            raise ValueError('data/uI_uI not in hdf5')
        if not ('data/vI_vI' in self):
            raise ValueError('data/vI_vI not in hdf5')
        if not ('csys/vtang' in self):
            raise ValueError('csys/vtang not in hdf5')
        if not ('csys/vnorm' in self):
            raise ValueError('csys/vnorm not in hdf5')
        if not (self.open_mode=='a') or (self.open_mode=='w'):
            raise ValueError('not able to write to hdf5 file')
        
        ## read unit vectors (wall tangent, wall norm) from HDF5
        vtang = np.copy( self['csys/vtang'][()] )
        vnorm = np.copy( self['csys/vnorm'][()] )
        trafo = np.stack((vtang,vnorm),axis=-1)
        
        ## transpose [2,2] matrix at every [x,y]
        trafoT = np.transpose(trafo,(0,1,3,2))
        
        # === Reynolds
        
        if ('data/uI_uI' in self) and ('data/uI_vI' in self) and ('data/vI_vI' in self):
            
            uI_uI = np.copy( self['data/uI_uI'][()].T )
            vI_vI = np.copy( self['data/vI_vI'][()].T )
            uI_vI = np.copy( self['data/uI_vI'][()].T )
            vI_uI = np.copy( uI_vI )
            
            ## construct 2D tensor --> (nx,ny,2,2)
            uIuI_ij = np.stack( (np.stack( [uI_uI, uI_vI], axis=-1 ),
                                 np.stack( [vI_uI, vI_vI], axis=-1 )), axis=-1 )
            
            #uIuI_ij = np.transpose(uIuI_ij, (0,1,3,2))
            
            uI_uI_rms = np.copy( self['data/uI_uI_rms'][()].T )
            vI_vI_rms = np.copy( self['data/vI_vI_rms'][()].T )
            uI_vI_rms = np.copy( self['data/uI_vI_rms'][()].T )
            vI_uI_rms = np.copy( uI_vI_rms )
            
            ## construct 2D tensor --> (nx,ny,2,2)
            uIuI_rms_ij = np.stack( (np.stack( [uI_uI_rms, uI_vI_rms], axis=-1 ),
                                     np.stack( [vI_uI_rms, vI_vI_rms], axis=-1 )), axis=-1 )
            
            #uIuI_rms_ij = np.transpose(uIuI_rms_ij, (0,1,3,2))
            
            ## check
            np.testing.assert_allclose(uI_uI, uI_uI_rms**2, rtol=1e-6, atol=1e-6)
            np.testing.assert_allclose(vI_vI, vI_vI_rms**2, rtol=1e-6, atol=1e-6)
            
            ## these three ops are the same
            #uIuI_ij_tn_A = np.einsum('xyij,xyjk,xykl->xyil', trafoT, uIuI_ij, trafo)
            #uIuI_ij_tn_B = np.einsum('xyji,xyjk,xykl->xyil', trafo,  uIuI_ij, trafo)
            #uIuI_ij_tn_C = np.matmul(np.matmul(trafoT, uIuI_ij), trafo)
            #np.testing.assert_allclose(uIuI_ij_tn_A, uIuI_ij_tn_B, atol=1e-14, rtol=1e-14)
            #np.testing.assert_allclose(uIuI_ij_tn_B, uIuI_ij_tn_C, atol=1e-14, rtol=1e-14)
            
            uIuI_ij_tn     = np.einsum('xyji,xyjk,xykl->xyil', trafo, uIuI_ij,     trafo)
            uIuI_rms_ij_tn = np.einsum('xyji,xyjk,xykl->xyil', trafo, uIuI_rms_ij, trafo)
            
            utI_utI = np.copy(uIuI_ij_tn[:,:,0,0])
            unI_unI = np.copy(uIuI_ij_tn[:,:,1,1])
            utI_unI = np.copy(uIuI_ij_tn[:,:,0,1])
            unI_utI = np.copy(uIuI_ij_tn[:,:,1,0])
            
            np.testing.assert_allclose( utI_unI, unI_utI, rtol=1e-14, atol=1e-14 )
            
            utI_utI_rms = np.copy(uIuI_rms_ij_tn[:,:,0,0])
            unI_unI_rms = np.copy(uIuI_rms_ij_tn[:,:,1,1])
            utI_unI_rms = np.copy(uIuI_rms_ij_tn[:,:,0,1])
            unI_utI_rms = np.copy(uIuI_rms_ij_tn[:,:,1,0])
            
            np.testing.assert_allclose( utI_unI_rms, unI_utI_rms, rtol=1e-14, atol=1e-14 )
            
            ## fails
            #np.testing.assert_allclose( utI_utI, utI_utI_rms**2, rtol=1e-6, atol=1e-6 )
            #np.testing.assert_allclose( unI_unI, unI_unI_rms**2, rtol=1e-6, atol=1e-6 )
            
            ## only write full covariances, not RMSes
            
            if ('data/utI_utI' in self): del self['data/utI_utI']
            dset = self.create_dataset('data/utI_utI', data=utI_utI.T, chunks=None)
            if verbose: even_print('utI_utI','%s'%str(utI_utI.shape))
            
            if ('data/unI_unI' in self): del self['data/unI_unI']
            dset = self.create_dataset('data/unI_unI', data=unI_unI.T, chunks=None)
            if verbose: even_print('unI_unI','%s'%str(unI_unI.shape))
            
            if ('data/utI_unI' in self): del self['data/utI_unI']
            dset = self.create_dataset('data/utI_unI', data=utI_unI.T, chunks=None)
            if verbose: even_print('utI_unI','%s'%str(utI_unI.shape))
            
            ## check
            if self.rectilinear:
                np.testing.assert_allclose(utI_utI, uI_uI, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(unI_unI, vI_vI, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(utI_unI, uI_vI, rtol=1e-14, atol=1e-14)
        
        # === Favre
        
        if ('data/r_uII_uII' in self) and ('data/r_uII_vII' in self) and ('data/r_vII_vII' in self):
            
            r_uII_uII = np.copy( self['data/r_uII_uII'][()].T )
            r_vII_vII = np.copy( self['data/r_vII_vII'][()].T )
            r_uII_vII = np.copy( self['data/r_uII_vII'][()].T )
            r_vII_uII = np.copy( r_uII_vII )
            
            ## construct 2D tensor --> (nx,ny,2,2)
            r_uIIuII_ij = np.stack( (np.stack( [r_uII_uII, r_uII_vII], axis=-1 ),
                                     np.stack( [r_vII_uII, r_vII_vII], axis=-1 )), axis=-1 )
            
            r_uII_uII_rms = np.copy( self['data/r_uII_uII_rms'][()].T )
            r_vII_vII_rms = np.copy( self['data/r_vII_vII_rms'][()].T )
            r_uII_vII_rms = np.copy( self['data/r_uII_vII_rms'][()].T )
            r_vII_uII_rms = np.copy( r_uII_vII_rms )
            
            ## construct 2D tensor --> (nx,ny,2,2)
            r_uIIuII_rms_ij = np.stack( (np.stack( [r_uII_uII_rms, r_uII_vII_rms], axis=-1 ),
                                         np.stack( [r_vII_uII_rms, r_vII_vII_rms], axis=-1 )), axis=-1 )
            
            ## check
            np.testing.assert_allclose(r_uII_uII, r_uII_uII_rms**2, rtol=1e-6, atol=1e-6)
            np.testing.assert_allclose(r_vII_vII, r_vII_vII_rms**2, rtol=1e-6, atol=1e-6)
            
            r_uIIuII_ij_tn     = np.einsum('xyji,xyjk,xykl->xyil', trafo, r_uIIuII_ij,     trafo)
            r_uIIuII_rms_ij_tn = np.einsum('xyji,xyjk,xykl->xyil', trafo, r_uIIuII_rms_ij, trafo)
            
            r_utII_utII = np.copy(r_uIIuII_ij_tn[:,:,0,0])
            r_unII_unII = np.copy(r_uIIuII_ij_tn[:,:,1,1])
            r_utII_unII = np.copy(r_uIIuII_ij_tn[:,:,0,1])
            r_unII_utII = np.copy(r_uIIuII_ij_tn[:,:,1,0])
            
            np.testing.assert_allclose( r_utII_unII, r_unII_utII, rtol=1e-14, atol=1e-14 )
            
            r_utII_utII_rms = np.copy(uIuI_rms_ij_tn[:,:,0,0])
            r_unII_unII_rms = np.copy(uIuI_rms_ij_tn[:,:,1,1])
            r_utII_unII_rms = np.copy(uIuI_rms_ij_tn[:,:,0,1])
            r_unII_utII_rms = np.copy(uIuI_rms_ij_tn[:,:,1,0])
            
            np.testing.assert_allclose( r_utII_unII_rms, r_unII_utII_rms, rtol=1e-14, atol=1e-14 )
            
            ## fails!
            #np.testing.assert_allclose( r_utII_utII, r_utII_utII_rms**2, rtol=1e-6, atol=1e-6 )
            #np.testing.assert_allclose( r_unII_unII, r_unII_unII_rms**2, rtol=1e-6, atol=1e-6 )
            
            ## only write full covariances, not RMSes
            
            if ('data/r_utII_utII' in self): del self['data/r_utII_utII']
            dset = self.create_dataset('data/r_utII_utII', data=r_utII_utII.T, chunks=None)
            if verbose: even_print('r_utII_utII','%s'%str(r_utII_utII.shape))
            
            if ('data/r_unII_unII' in self): del self['data/r_unII_unII']
            dset = self.create_dataset('data/r_unII_unII', data=r_unII_unII.T, chunks=None)
            if verbose: even_print('r_unII_unII','%s'%str(r_unII_unII.shape))
            
            if ('data/r_utII_unII' in self): del self['data/r_utII_unII']
            dset = self.create_dataset('data/r_utII_unII', data=r_utII_unII.T, chunks=None)
            if verbose: even_print('r_utII_unII','%s'%str(r_utII_unII.shape))
            
            ## check
            if self.rectilinear:
                np.testing.assert_allclose(r_utII_utII, r_uII_uII, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(r_unII_unII, r_vII_vII, rtol=1e-14, atol=1e-14)
                np.testing.assert_allclose(r_utII_unII, r_uII_vII, rtol=1e-14, atol=1e-14)
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_vel_tangnorm_mean_removed() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing: mass & momentum balance
    
    @staticmethod
    def mass_balance_kernel(x,y, u,v,rho, transformation_matrix=None, acc=8, edge_stencil='half'):
        '''
        get Cartesian mass balance terms in 2D plane
        '''
        
        ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
        uv2d = np.copy(np.stack((u,v), axis=-1))
        
        ## rotate & translate [x,y]
        if (transformation_matrix is not None):
            
            ## rotate & translate [x,y]
            xy2d_tmp = np.copy( np.stack( ( x, y, np.ones_like(x) ), axis=-1) )
            xy2d_tmp = np.copy( np.einsum( 'ij,xyj->xyi', transformation_matrix , xy2d_tmp ) )
            xy2d     = np.copy( xy2d_tmp[:,:,:2] )
            xy2d_tmp = None; del xy2d_tmp
            
            ## overwrite
            x = np.squeeze(np.copy(xy2d[:,:,0]))
            y = np.squeeze(np.copy(xy2d[:,:,1]))
            
            ## make a copy of the [2x2] matrix
            rotation_matrix_2x2 = np.copy(transformation_matrix[:2,:2])
            
            ## rotate [u,v]
            uv2d = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d ) )
            
            ## overwrite
            u = np.squeeze(np.copy(uv2d[:,:,0]))
            v = np.squeeze(np.copy(uv2d[:,:,1]))
        
        else:
            
            ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
            xy2d = np.copy(np.stack((x,y), axis=-1))
        
        ## get metric tensor
        M = get_metric_tensor_2d(x, y, acc=acc, edge_stencil=edge_stencil, verbose=False)
        ddx1_q1 = np.copy( M[:,:,0,0] ) ## ∂q1/∂x1 --> ∂q1/∂x or ∂q1/∂r
        ddx1_q2 = np.copy( M[:,:,1,0] ) ## ∂q2/∂x1 --> ∂q2/∂x or ∂q2/∂r
        ddx2_q1 = np.copy( M[:,:,0,1] ) ## ∂q1/∂x2 --> ∂q1/∂y or ∂q1/∂θ
        ddx2_q2 = np.copy( M[:,:,1,1] ) ## ∂q2/∂x2 --> ∂q2/∂y or ∂q2/∂θ
        M = None ; del M
        
        # ===
        
        A = np.copy( rho * u )
        ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
        ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
        ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
        
        ddx_r_u = np.copy(ddx_A)
        ddy_r_u = np.copy(ddy_A)
        
        ddx_A = None; del ddx_A
        ddy_A = None; del ddy_A
        
        
        A = np.copy( rho * v )
        ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
        ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
        ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
        ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
        
        ddx_r_v = np.copy(ddx_A)
        ddy_r_v = np.copy(ddy_A)
        
        ddx_A = None; del ddx_A
        ddy_A = None; del ddy_A
        
        # ===
        
        data = {'t1':ddx_r_u,
                't2':ddy_r_v,
               }
        
        return data
    
    def calc_mass_balance_terms(self, **kwargs):
        '''
        calculate terms of time-averaged mass conservation equation
        '''
        
        verbose      = kwargs.get('verbose',True)
        local_csys   = kwargs.get('local_csys',False)
        gn           = kwargs.get('gn','data_mass') ## group name
        acc          = kwargs.get('acc',8)
        edge_stencil = kwargs.get('edge_stencil','half')
        xis          = kwargs.get('xis',None)
        
        csys_type = 'cartesian' ## dummy for now... mass balance in cylindrical not yet implemented
        
        if verbose: print('\n'+'ztmd.calc_mass_balance_terms()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print('local_csys',str(local_csys))
        if verbose: even_print('csys_type',csys_type) ## dummy for now
        if verbose: even_print('group name',f'{gn}/')
        
        if local_csys:
            if self.rectilinear:
                print('>>> WARNING: this grid is rectilinear. Using local csys is not needed.')
            if not ('csys/vtang' in self):
                raise ValueError('csys/vtang not in hdf5')
            if not ('csys/vnorm' in self):
                raise ValueError('csys/vnorm not in hdf5')
            
            ## read unit vectors (wall tangent, wall norm) from HDF5
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            if verbose: even_print('vtang',str(vtang.shape))
            if verbose: even_print('vnorm',str(vnorm.shape))
        
        ## if group already exists, delete it entirely
        if (gn in self):
            del self[gn]
            if verbose: even_print('deleted group',f'/{gn}/')
        
        ## create group
        grp = self.create_group(gn)
        if verbose: even_print('created group',f'{grp.name}/')
        
        ## set group attributes
        grp.attrs['local_csys'] = local_csys
        if verbose: even_print(f'{gn}.local_csys',str(grp.attrs['local_csys']))
        
        if verbose: print(72*'-')
        
        terms = {'t1':r'$\partial_x [\bar{\rho} \tilde{u}]$',
                 't2':r'$\partial_y [\bar{\rho} \tilde{v}]$',
                 }
        
        ## initialize dsets
        for dsn, tag in terms.items():
            shape = (self.ny,self.nx)
            if (f'{gn}/{dsn}' in self): del self[f'{gn}/{dsn}']
            dset = self.create_dataset(f'{gn}/{dsn}', 
                                       shape=shape, 
                                       dtype=np.float32,
                                       chunks=None)
            if verbose: even_print(f'{gn}/{dsn}',str(shape))
            dset.attrs['latex'] = tag
            if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
        
        if verbose: print(72*'-')
        
        ## copy data
        u   = np.copy( self['data/u'][()].T   )
        v   = np.copy( self['data/v'][()].T   )
        #w   = np.copy( self['data/w'][()].T   )
        rho = np.copy( self['data/rho'][()].T )
        #p   = np.copy( self['data/p'][()].T   )
        #T   = np.copy( self['data/T'][()].T   )
        #mu  = np.copy( self['data/mu'][()].T  )
        if verbose: even_print('u',str(u.shape))
        if verbose: even_print('v',str(v.shape))
        if verbose: even_print('ρ',str(rho.shape))
        
        u_Fv = np.copy( self['data/u_Fv'][()].T )
        v_Fv = np.copy( self['data/v_Fv'][()].T )
        if verbose: even_print('u_Fv',str(u_Fv.shape))
        if verbose: even_print('v_Fv',str(v_Fv.shape))
        
        x = np.copy(self.x)
        y = np.copy(self.y)
        
        if (csys_type=='cylindrical'):
            crv_R = np.copy( self['dims/crv_R'][()] )
        
        if (x.ndim==1) and (y.ndim==1):
            x,y = np.meshgrid(x,y, indexing='ij')
        if verbose: even_print('x',str(x.shape))
        if verbose: even_print('y',str(y.shape))
        
        if local_csys:
            
            # ## calculate grid points for specific postions in char. lengths
            # s_vert_locs = [-100,-50,-25,0,+25,+50,+100]
            # svert  = (self.stang-250*self.lchar)/self.lchar
            # xis    = [ np.abs(s-svert).argmin() for s in s_vert_locs ]
            
            if (xis is None):
                xis = np.arange(self.nx)
                n_prog = self.nx
            else:
                n_prog = len(xis)
            
            if verbose: progress_bar = tqdm(total=n_prog, ncols=100, desc='mass balance', leave=False, file=sys.stdout)
            
            ## iterate through axis=0 indices
            for xi in xis:
                
                rotation_angle_z = np.arcsin(vnorm[xi,0,0])
                #rotation_angle_z = np.arcsin(-vtang[xi,0,1])
                #if verbose: tqdm.write( even_print('rotation_angle_z', f'{np.degrees(rotation_angle_z):0.14f} [deg]', s=True) )
                
                rotation_matrix_1 = np.array( [[ np.cos(rotation_angle_z) , -np.sin(rotation_angle_z) , 0 ],
                                               [ np.sin(rotation_angle_z) ,  np.cos(rotation_angle_z) , 0 ],
                                               [ 0                        ,  0                        , 1 ]], dtype=np.float64 )
                
                rotation_matrix_2 = np.array( [[ vtang[xi,0,0] ,  vtang[xi,0,1] , 0 ],
                                               [ vnorm[xi,0,0] ,  vnorm[xi,0,1] , 0 ],
                                               [ 0             ,  0             , 1 ]], dtype=np.float64 )
                
                ## just showing that these are the same
                np.testing.assert_allclose( rotation_matrix_1 , rotation_matrix_2 , rtol=1e-14 , atol=1e-14 )
                rotation_matrix = np.copy(rotation_matrix_1)
                
                ## translate to [cx,cy]
                cx = x[xi,0]
                cy = y[xi,0]
                translation_matrix_A = np.array( [[ 1 , 0 , -cx ],
                                                  [ 0 , 1 , -cy ],
                                                  [ 0 , 0 , 1   ]], dtype=np.float64 )
                
                ## translate in [y] by -R (the local curvature radius)
                if (csys_type=='cylindrical'):
                    cx = 0.
                    cy = crv_R[xi]
                    translation_matrix_B = np.array( [[ 1 , 0 , 0.  ],
                                                      [ 0 , 1 , -cy ],
                                                      [ 0 , 0 , 1   ]], dtype=np.float64 )
                else:
                    translation_matrix_B = np.identity(3, dtype=np.float64)
                
                ## form the full [3x3] matrix by multiplying together --> order matters!
                transformation_matrix = np.einsum( 'ij,jk,kl->il', translation_matrix_B , rotation_matrix, translation_matrix_A )
                
                # ===
                
                ## send to kernel
                data = ztmd.mass_balance_kernel(x=x,y=y,
                                                u=u_Fv, v=v_Fv, rho=rho,
                                                transformation_matrix=transformation_matrix,
                                                acc=acc,edge_stencil=edge_stencil,
                                                )
                
                ## write to HDF5 (only data from single axis=0 index)
                for dsn, arr in data.items():
                    dset = self[f'{gn}/{dsn}']
                    dset[:,xi] = arr[xi,:] ## arrays in HDF5 are transposed
                
                if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        else:
            
            ## send to kernel
            data = ztmd.mass_balance_kernel(x=x,y=y,
                                            u=u_Fv, v=v_Fv, rho=rho,
                                            transformation_matrix=None,
                                            acc=acc,edge_stencil=edge_stencil,
                                            )
            
            ## write to HDF5 (only data from single axis=0 index)
            for dsn, arr in data.items():
                dset = self[f'{gn}/{dsn}']
                dset[:,:] = arr.T ## arrays in HDF5 are transposed
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_mass_balance_terms() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    @staticmethod
    def momentum_balance_kernel_cartesian(x,y, u,v, u_Fv,v_Fv, p,rho,mu, r_uII_uII,r_uII_vII,r_vII_vII, transformation_matrix=None, acc=8, edge_stencil='half'):
        '''
        get Cartesian [x] momentum balance terms in 2D plane
        '''
        
        nx,ny = x.shape
        
        ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
        uv2d    = np.copy(np.stack((u,v),       axis=-1))
        uv2d_Fv = np.copy(np.stack((u_Fv,v_Fv), axis=-1))
        
        ## rotate & translate [x,y]
        if (transformation_matrix is not None):
            
            ## rotate & translate [x,y]
            xy2d_tmp = np.copy( np.stack( ( x, y, np.ones_like(x) ), axis=-1) )
            xy2d_tmp = np.copy( np.einsum( 'ij,xyj->xyi', transformation_matrix , xy2d_tmp ) )
            xy2d     = np.copy( xy2d_tmp[:,:,:2] )
            xy2d_tmp = None; del xy2d_tmp
            
            ## overwrite
            x = np.squeeze(np.copy(xy2d[:,:,0]))
            y = np.squeeze(np.copy(xy2d[:,:,1]))
            
            ## make a copy of the [2x2] matrix
            rotation_matrix_2x2 = np.copy(transformation_matrix[:2,:2])
            
            ## rotate [u,v]
            uv2d = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d ) )
            
            ## overwrite
            u = np.squeeze(np.copy(uv2d[:,:,0]))
            v = np.squeeze(np.copy(uv2d[:,:,1]))
            
            ## rotate [u_Fv,v_Fv]
            uv2d_Fv = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d_Fv ) )
            
            ## overwrite
            u_Fv = np.squeeze(np.copy(uv2d_Fv[:,:,0]))
            v_Fv = np.squeeze(np.copy(uv2d_Fv[:,:,1]))
            
            if True: ## Re-Stress tensor: global Cartesian rotate
                
                r_uiIIujII = np.zeros((nx,ny,2,2), dtype=np.float64)
                r_uiIIujII[:,:,0,0] = np.copy( r_uII_uII )
                r_uiIIujII[:,:,0,1] = np.copy( r_uII_vII )
                r_uiIIujII[:,:,1,0] = np.copy( r_uII_vII )
                r_uiIIujII[:,:,1,1] = np.copy( r_vII_vII )
                
                ## assert symmetric
                np.testing.assert_allclose(r_uiIIujII, np.transpose(r_uiIIujII, (0,1,3,2)), atol=1e-14, rtol=1e-14)
                
                ## do tensor rotation : [A'] = [R][A][R]^T
                r_uiIIujII = np.copy( np.einsum( 'ij,xyjk,kl->xyil' , rotation_matrix_2x2 , r_uiIIujII , rotation_matrix_2x2.T ) )
                
                r_uII_uII = np.copy(r_uiIIujII[:,:,0,0])
                r_uII_vII = np.copy(r_uiIIujII[:,:,0,1])
                r_vII_uII = np.copy(r_uiIIujII[:,:,1,0])
                r_vII_vII = np.copy(r_uiIIujII[:,:,1,1])
                
                r_uiIIujII = None ; del r_uiIIujII
                
                np.testing.assert_allclose(r_uII_vII, r_vII_uII, atol=1e-7, rtol=1e-7)
        
        else:
            
            ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
            xy2d = np.copy(np.stack((x,y), axis=-1))
        
        # ===
        
        ## get metric tensor M = δ[q1,q2]/δ[x1,x2] : δ[q1,q2]/δ[x,y] or δ[q1,q2]/δ[r,θ]
        M = get_metric_tensor_2d(x, y, acc=acc, edge_stencil=edge_stencil, verbose=False)
        ddx1_q1 = np.copy( M[:,:,0,0] ) ## ∂q1/∂x1 --> ∂q1/∂x or ∂q1/∂r
        ddx1_q2 = np.copy( M[:,:,1,0] ) ## ∂q2/∂x1 --> ∂q2/∂x or ∂q2/∂r
        ddx2_q1 = np.copy( M[:,:,0,1] ) ## ∂q1/∂x2 --> ∂q1/∂y or ∂q1/∂θ
        ddx2_q2 = np.copy( M[:,:,1,1] ) ## ∂q2/∂x2 --> ∂q2/∂y or ∂q2/∂θ
        M = None ; del M
        
        # ===
        
        if True: ## calculate gradients : Cartesian [x,y]
            
            ## [x1,x2] = [x,y]
            ## ∂(A)/∂x1 = ∂(A)/∂(q1)·∂(q1)/∂(x1) + ∂(A)/∂(q2)·∂(q2)/∂(x1)
            ## ∂(A)/∂x2 = ∂(A)/∂(q1)·∂(q1)/∂(x2) + ∂(A)/∂(q2)·∂(q2)/∂(x2)
            
            A = np.copy( u )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_u = np.copy(ddx_A)
            ddy_u = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( v )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_v = np.copy(ddx_A)
            ddy_v = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( rho )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_rho = np.copy(ddx_A)
            ddy_rho = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( p )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_p = np.copy(ddx_A)
            ddy_p = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( u_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_u_Fv = np.copy(ddx_A)
            ddy_u_Fv = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            
            A = np.copy( v_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_v_Fv = np.copy(ddx_A)
            ddy_v_Fv = np.copy(ddy_A)
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
        
        if True: ## construct momentum balance : Cartesian [x,y]
            
            tau_xx = np.copy( +(4/3)*mu*ddx_u - (2/3)*mu*ddy_v )
            tau_yy = np.copy( -(2/3)*mu*ddx_u + (4/3)*mu*ddy_v )
            
            tau_xy = np.copy( mu*(ddx_v + ddy_u) )
            #tau_yx = np.copy( tau_xy )
            
            # === ∂x[ρ·u·u]
            
            A = np.copy( rho * u_Fv * u_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_r_uFv_uFv = np.copy(ddx_A)
            t1x = np.copy( ddx_r_uFv_uFv )
            t1x_lbl = r'$\partial_{x}[\bar{\rho} \tilde{u} \tilde{u}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === ∂x[ρ·u·v] & ∂y[ρ·u·v]
            
            A = np.copy( rho * u_Fv * v_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_r_uFv_vFv = np.copy(ddx_A)
            ddy_r_uFv_vFv = np.copy(ddy_A)
            
            t1y = np.copy( ddx_r_uFv_vFv )
            t1y_lbl = r'$\partial_{x}[\bar{\rho} \tilde{u} \tilde{v}]$'
            
            t2x = np.copy( ddy_r_uFv_vFv )
            t2x_lbl = r'$\partial_{y}[\bar{\rho} \tilde{u} \tilde{v}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === ∂y[ρ·v·v]
            
            A = np.copy( rho * v_Fv * v_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddy_r_vFv_vFv = np.copy(ddy_A)
            t2y = np.copy( ddy_r_vFv_vFv )
            t2y_lbl = r'$\partial_{y}[\bar{\rho} \tilde{v} \tilde{v}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === ∂x[ρ·u″·u″]
            
            A = np.copy(r_uII_uII)
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_r_uII_uII = np.copy(ddx_A)
            t5x = np.copy( ddx_r_uII_uII )
            t5x_lbl = r'$\partial_{x}[ \overline{ \rho u^{\prime\prime} u^{\prime\prime}} ]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === ∂x[ρ·u″·v″] & ∂y[ρ·u″·v″]
            
            A = np.copy(r_uII_vII)
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_r_uII_vII = np.copy(ddx_A)
            ddy_r_uII_vII = np.copy(ddy_A)
            
            t5y = np.copy( ddx_r_uII_vII )
            t5y_lbl = r'$\partial_{x}[ \overline{ \rho u^{\prime\prime} v^{\prime\prime}} ]$'
            t7x = np.copy( ddy_r_uII_vII )
            t7x_lbl = r'$\partial_{y}[ \overline{ \rho u^{\prime\prime} v^{\prime\prime}} ]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === ∂y[ρ·v″·v″]
            
            A = np.copy(r_vII_vII)
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddy_r_vII_vII = np.copy(ddy_A)
            t7y = np.copy( ddy_r_vII_vII )
            t7y_lbl = r'$\partial_{y}[ \overline{\rho v^{\prime\prime} v^{\prime\prime}} ]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === ∂x[p] & ∂y[p]
            
            t3x = np.copy( ddx_p )
            t3x_lbl = r'$\partial_{x}[p]$'
            
            t3y = np.copy( ddy_p )
            t3y_lbl = r'$\partial_{y}[p]$'
            
            # === ∂x[τxx]
            
            ddq1_A = gradient(tau_xx, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(tau_xx, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_tau_xx = np.copy(ddx_A)
            t4x = np.copy( -1 * ddx_tau_xx )
            t4x_lbl = r'$-\partial_{x}[\overline{\tau_{xx}}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === ∂x[τxy] & ∂y[τxy]
            
            ddq1_A = gradient(tau_xy, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(tau_xy, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddx_tau_xy = np.copy(ddx_A)
            ddy_tau_xy = np.copy(ddy_A)
            
            t4y = np.copy( -1 * ddx_tau_xy )
            t4y_lbl = r'$-\partial_{x}[\overline{\tau_{xy}}]$'
            
            t6x = np.copy( -1 * ddy_tau_xy )
            t6x_lbl = r'$-\partial_{y}[\overline{\tau_{xy}}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
            
            # === ∂y[τyy]
            
            ddq1_A = gradient(tau_yy, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(tau_yy, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx_A  = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddy_A  = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddy_tau_yy = np.copy(ddy_A)
            t6y = np.copy( -1 * ddy_tau_yy )
            t6y_lbl = r'$-\partial_{y}[\overline{\tau_{yy}}]$'
            
            ddx_A = None; del ddx_A
            ddy_A = None; del ddy_A
        
        # ===
        
        data_x = {
            't1':np.copy(t1x),
            't2':np.copy(t2x),
            't3':np.copy(t3x),
            't4':np.copy(t4x),
            't5':np.copy(t5x),
            't6':np.copy(t6x),
            't7':np.copy(t7x),
            }
        
        data_y = {
            't1':np.copy(t1y),
            't2':np.copy(t2y),
            't3':np.copy(t3y),
            't4':np.copy(t4y),
            't5':np.copy(t5y),
            't6':np.copy(t6y),
            't7':np.copy(t7y),
            }
        
        return data_x, data_y
    
    @staticmethod
    def momentum_balance_kernel_cylindrical(x,y, u,v, u_Fv,v_Fv, p,rho,mu, r_uII_uII,r_uII_vII,r_vII_vII, transformation_matrix=None, acc=8, edge_stencil='half'):
        '''
        get cylindrical [r,θ] momentum balance terms in 2D plane
        '''
        
        nx,ny = x.shape
        
        ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
        uv2d    = np.copy(np.stack((u,v),       axis=-1))
        uv2d_Fv = np.copy(np.stack((u_Fv,v_Fv), axis=-1))
        
        ## rotate & translate [x,y]
        if (transformation_matrix is not None):
            
            ## rotate & translate [x,y]
            xy2d_tmp = np.copy( np.stack( ( x, y, np.ones_like(x) ), axis=-1) )
            xy2d_tmp = np.copy( np.einsum( 'ij,xyj->xyi', transformation_matrix , xy2d_tmp ) )
            xy2d     = np.copy( xy2d_tmp[:,:,:2] )
            xy2d_tmp = None; del xy2d_tmp
            
            ## overwrite
            x = np.squeeze(np.copy(xy2d[:,:,0]))
            y = np.squeeze(np.copy(xy2d[:,:,1]))
            
            ## make a copy of the [2x2] matrix
            rotation_matrix_2x2 = np.copy(transformation_matrix[:2,:2])
            
            ## rotate [u,v]
            uv2d = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d ) )
            
            ## overwrite
            u = np.squeeze(np.copy(uv2d[:,:,0]))
            v = np.squeeze(np.copy(uv2d[:,:,1]))
            
            ## rotate [u_Fv,v_Fv]
            uv2d_Fv = np.copy( np.einsum( 'ij,xyj->xyi' , rotation_matrix_2x2 , uv2d_Fv ) )
            
            ## overwrite
            u_Fv = np.squeeze(np.copy(uv2d_Fv[:,:,0]))
            v_Fv = np.squeeze(np.copy(uv2d_Fv[:,:,1]))
            
            if True: ## Re-Stress tensor: global Cartesian rotate
                
                r_uiIIujII = np.zeros((nx,ny,2,2), dtype=np.float64)
                r_uiIIujII[:,:,0,0] = np.copy( r_uII_uII )
                r_uiIIujII[:,:,0,1] = np.copy( r_uII_vII )
                r_uiIIujII[:,:,1,0] = np.copy( r_uII_vII )
                r_uiIIujII[:,:,1,1] = np.copy( r_vII_vII )
                
                ## assert symmetric
                np.testing.assert_allclose(r_uiIIujII, np.transpose(r_uiIIujII, (0,1,3,2)), atol=1e-14, rtol=1e-14)
                
                ## do tensor rotation : [A'] = [R][A][R]^T
                r_uiIIujII = np.copy( np.einsum( 'ij,xyjk,kl->xyil' , rotation_matrix_2x2 , r_uiIIujII , rotation_matrix_2x2.T ) )
                
                r_uII_uII = np.copy(r_uiIIujII[:,:,0,0])
                r_uII_vII = np.copy(r_uiIIujII[:,:,0,1])
                r_vII_uII = np.copy(r_uiIIujII[:,:,1,0])
                r_vII_vII = np.copy(r_uiIIujII[:,:,1,1])
                
                r_uiIIujII = None ; del r_uiIIujII
                
                np.testing.assert_allclose(r_uII_vII, r_vII_uII, atol=1e-7, rtol=1e-7)
        
        else:
            
            ## stack 2D arrays, each with shape (nx,ny) along third axis --> shape=(nx,ny,2)
            xy2d = np.copy(np.stack((x,y), axis=-1))
        
        # ===
        
        ## calculate coordinate arrays [r,θ] from [x,y]
        r     = np.copy( np.sqrt(x**2 + y**2) )
        theta = np.copy( np.arctan2(y,x) )
        
        ## calculate [ur,uθ] from [u,v]
        vel_cart2cyl_trans_mat = np.zeros((nx,ny,2,2), dtype=np.float64)
        for i in range(nx):
            for j in range(ny):
                vel_cart2cyl_trans_mat[i,j,:,:] = np.array( [[  np.cos(theta[i,j]) , np.sin(theta[i,j]) ],
                                                             [ -np.sin(theta[i,j]) , np.cos(theta[i,j]) ]], dtype=np.float64 )
        
        ## ur, uθ
        urut2d = np.einsum( 'xyji,xyi->xyj', vel_cart2cyl_trans_mat, uv2d )
        ur = np.squeeze(np.copy(urut2d[:,:,0]))
        ut = np.squeeze(np.copy(urut2d[:,:,1]))
        
        ## ur_Fv, uθ_Fv
        urut2d_Fv = np.einsum( 'xyji,xyi->xyj', vel_cart2cyl_trans_mat, uv2d_Fv )
        ur_Fv = np.squeeze(np.copy(urut2d_Fv[:,:,0]))
        ut_Fv = np.squeeze(np.copy(urut2d_Fv[:,:,1]))
        
        ## Re-Stress tensor : convert from Cartesian (rotated) to cylindrical components
        r_uiIIujII = np.zeros((nx,ny,2,2), dtype=np.float64)
        r_uiIIujII[:,:,0,0] = np.copy( r_uII_uII )
        r_uiIIujII[:,:,0,1] = np.copy( r_uII_vII )
        r_uiIIujII[:,:,1,0] = np.copy( r_uII_vII )
        r_uiIIujII[:,:,1,1] = np.copy( r_vII_vII )
        
        r_uII_uII = None; del r_uII_uII
        r_uII_vII = None; del r_uII_vII
        r_vII_uII = None; del r_vII_uII
        r_vII_vII = None; del r_vII_vII
        
        ## assert symmetric
        np.testing.assert_allclose(r_uiIIujII, np.transpose(r_uiIIujII, (0,1,3,2)), atol=1e-14, rtol=1e-14)
        
        ## csys conversion : Cartesian --> cylindrical
        r_uiIIujII = np.copy( np.einsum( 'xyij,xyjk,xykl->xyil' , vel_cart2cyl_trans_mat , r_uiIIujII , np.transpose(vel_cart2cyl_trans_mat,(0,1,3,2)) ) )
        
        r_urII_urII = np.copy(r_uiIIujII[:,:,0,0])
        r_urII_utII = np.copy(r_uiIIujII[:,:,0,1])
        r_utII_urII = np.copy(r_uiIIujII[:,:,1,0])
        r_utII_utII = np.copy(r_uiIIujII[:,:,1,1])
        
        r_uiIIujII = None ; del r_uiIIujII
        
        ## assert symmetric again
        np.testing.assert_allclose(r_urII_utII, r_utII_urII, atol=1e-7, rtol=1e-7)
        
        # ===
        
        ## get metric tensor M = δ[q1,q2]/δ[x1,x2] : δ[q1,q2]/δ[x,y] or δ[q1,q2]/δ[r,θ]
        M = get_metric_tensor_2d(r, theta, acc=acc, edge_stencil=edge_stencil, verbose=False)
        ddx1_q1 = np.copy( M[:,:,0,0] ) ## ∂q1/∂x1 --> ∂q1/∂x or ∂q1/∂r
        ddx1_q2 = np.copy( M[:,:,1,0] ) ## ∂q2/∂x1 --> ∂q2/∂x or ∂q2/∂r
        ddx2_q1 = np.copy( M[:,:,0,1] ) ## ∂q1/∂x2 --> ∂q1/∂y or ∂q1/∂θ
        ddx2_q2 = np.copy( M[:,:,1,1] ) ## ∂q2/∂x2 --> ∂q2/∂y or ∂q2/∂θ
        M = None ; del M
        
        # ===
        
        if True: ## calculate gradients : cylindrical [r,θ]
            
            ## [x1,x2] = [r,θ]
            ## ∂(A)/∂x1 = ∂(A)/∂(q1)·∂(q1)/∂(x1) + ∂(A)/∂(q2)·∂(q2)/∂(x1)
            ## ∂(A)/∂x2 = ∂(A)/∂(q1)·∂(q1)/∂(x2) + ∂(A)/∂(q2)·∂(q2)/∂(x2)
            
            A = np.copy( ur )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_ur = np.copy(ddx1_A)
            ddt_ur = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( ut )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_ut = np.copy(ddx1_A)
            ddt_ut = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( rho )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_rho = np.copy(ddx1_A)
            ddt_rho = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( p )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_p = np.copy(ddx1_A)
            ddt_p = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( ur_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_ur_Fv = np.copy(ddx1_A)
            ddt_ur_Fv = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            
            A = np.copy( ut_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_ut_Fv = np.copy(ddx1_A)
            ddt_ut_Fv = np.copy(ddx2_A)
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
        
        if True: ## construct momentum balance : cylindrical [r,θ]
            
            ## τ_rr, τ_θθ, τ_rθ
            tau_rr = np.copy( +(4/3)*mu*ddr_ur - (2/3)*(1/r)*mu*ddt_ut - (2/3)*mu*(ur/r) )
            tau_tt = np.copy( -(2/3)*mu*ddr_ur + (4/3)*(1/r)*mu*ddt_ut + (4/3)*mu*(ur/r) )
            tau_rt = np.copy( mu*( (1/r)*ddt_ur + ddr_ut - (ut/r) ) )
            
            # === ∂r[ρ·ur·ur]
            
            A = np.copy( rho * ur_Fv * ur_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_rho_urFv_urFv = np.copy(ddx1_A)
            t1x1 = np.copy( ddr_rho_urFv_urFv )
            t1x1_lbl = r'$\partial_{r}[ \bar{\rho} \tilde{u}_r \tilde{u}_r ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === (1/r)·∂θ[ρ·ur·uθ]
            
            A = np.copy( rho * ur_Fv * ut_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_rho_urFv_utFv = np.copy(ddx2_A)
            t2x1 = np.copy( (1/r)*ddt_rho_urFv_utFv )
            t2x1_lbl = r'$\frac{1}{r} \partial_{\theta}[ \bar{\rho} \tilde{u}_r \tilde{u}_\theta ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === ∂r[ρ·ur·uθ]
            
            A = np.copy( rho * ur_Fv * ut_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_rho_urFv_utFv = np.copy(ddx1_A)
            t1x2 = np.copy( ddr_rho_urFv_utFv )
            t1x2_lbl = r'$\partial_{r}[ \bar{\rho} \tilde{u}_r \tilde{u}_\theta ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === ∂θ[ρ·uθ·uθ]
            
            A = np.copy( rho * ut_Fv * ut_Fv )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_rho_utFv_utFv = np.copy(ddx2_A)
            t2x2 = np.copy( (1/r)*ddt_rho_utFv_utFv )
            t2x2_lbl = r'$\frac{1}{r} \partial_{\theta}[ \bar{\rho} \tilde{u}_\theta \tilde{u}_\theta ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === ∂r[p] & ∂θ[p]
            
            t3x1 = np.copy( ddr_p )
            t3x1_lbl = r'$\partial_{r}[\bar{p}]$'
            
            t3x2 = np.copy( (1/r)*ddt_p )
            t3x2_lbl = r'$\frac{1}{r} \partial_{\theta}[\bar{p}]$'
            
            # === ∂r[τ_rr]
            
            A = np.copy( tau_rr )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_tau_rr = np.copy( ddx1_A )
            t4x1 = np.copy( -ddr_tau_rr )
            t4x1_lbl = r'$-\partial_{r}[ \overline{\tau}_{r r} ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === -(1/r)·∂θ[τ_θθ]
            
            A = np.copy( tau_tt )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_tau_tt = np.copy( ddx2_A )
            t6x2 = np.copy( -(1/r)*ddt_tau_tt )
            t6x2_lbl = r'$-\frac{1}{r} \partial_{\theta}[ \overline{\tau}_{\theta \theta} ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === ∂r[ρ·ur″·ur″]
            
            A = np.copy( r_urII_urII )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_r_urII_urII = np.copy( ddx1_A )
            t5x1 = np.copy( ddr_r_urII_urII )
            t5x1_lbl = r'$\partial_{r}[ \overline{ \rho u^{\prime \prime}_r  u^{\prime \prime}_r } ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === -(1/r)·∂θ[τ_rθ]  &  -∂r[τ_rθ]
            
            A = np.copy( tau_rt )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddr_tau_rt = np.copy( ddx1_A )
            t4x2 = np.copy( -ddr_tau_rt )
            t4x2_lbl = r'$-\partial_{r}[ \overline{\tau}_{r \theta} ]$'
            
            ddt_tau_rt = np.copy( ddx2_A )
            t6x1 = np.copy( -(1/r)*ddt_tau_rt )
            t6x1_lbl = r'$-\frac{1}{r} \partial_{\theta}[ \overline{\tau}_{r \theta} ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === (1/r)·∂θ[ρ·ur″·uθ″] & ∂r[ρ·ur″·uθ″]
            
            A = np.copy( r_urII_utII )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_r_urII_utII = np.copy( ddx2_A )
            t7x1 = np.copy( (1/r)*ddt_r_urII_utII )
            t7x1_lbl = r'$\frac{1}{r} \partial_{\theta}[ \overline{\rho u^{\prime \prime}_r  u^{\prime \prime}_\theta } ]$'
            
            ddr_r_urII_utII = np.copy( ddx1_A )
            t5x2 = np.copy( ddr_r_urII_utII )
            t5x2_lbl = r'$\partial_{r}[ \overline{\rho u^{\prime \prime}_r  u^{\prime \prime}_\theta } ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # === (1/r)·∂θ[ρ·uθ″·uθ″]
            
            A = np.copy( r_utII_utII )
            ddq1_A = gradient(A, 1., axis=0, acc=acc, edge_stencil=edge_stencil, d=1)
            ddq2_A = gradient(A, 1., axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddx1_A = ddq1_A*ddx1_q1 + ddq2_A*ddx1_q2
            ddx2_A = ddq1_A*ddx2_q1 + ddq2_A*ddx2_q2
            
            ddt_r_utII_utII = np.copy( ddx2_A )
            t7x2 = np.copy( (1/r)*ddt_r_utII_utII )
            t7x2_lbl = r'$\frac{1}{r} \partial_{\theta}[ \overline{\rho u^{\prime \prime}_\theta  u^{\prime \prime}_\theta } ]$'
            
            ddx1_A = None; del ddx1_A
            ddx2_A = None; del ddx2_A
            
            # ===
            
            ## +(1/r)·ρ·ur·ur
            t8x1 = np.copy( (1/r) * rho * ur_Fv * ur_Fv )
            t8x1_lbl = r'$\frac{1}{r} \bar{\rho} \tilde{u}_{r} \tilde{u}_{r}$'
            
            ## +(1/r)·ρ·ur″·ur″
            t9x1 = np.copy( (1/r) * r_urII_urII )
            t9x1_lbl = r'$\frac{1}{r} \overline{ \rho u^{\prime \prime}_r u^{\prime \prime}_r }$'
            
            ## -(1/r)·ρ·uθ·uθ
            t10x1 = np.copy( -1 * (1/r) * rho * ut_Fv * ut_Fv )
            t10x1_lbl = r'$-\frac{1}{r} \bar{\rho} \tilde{u}_{\theta} \tilde{u}_{\theta}$'
            
            ## -(1/r)·ρ·uθ″·uθ″
            t11x1 = np.copy( -1 * (1/r) * r_utII_utII )
            t11x1_lbl = r'$-\frac{1}{r} \overline{ \rho u^{\prime \prime}_\theta u^{\prime \prime}_\theta }$'
            
            ## -(1/r)·τ_rr
            t12x1 = np.copy( -1 * (1/r) * tau_rr )
            t12x1_lbl = r'$-\frac{1}{r} \overline{\tau}_{r r}$'
            
            ## +(1/r)·τ_θθ
            t13x1 = np.copy( (1/r) * tau_tt )
            t13x1_lbl = r'$\frac{1}{r} \overline{\tau}_{\theta \theta}$'
            
            # ===
            
            ## +(2/r)·ρ·ur·uθ
            t8x2 = np.copy( (2/r) * rho * ur_Fv * ut_Fv )
            t8x2_lbl = r'$\frac{2}{r} \bar{\rho} \tilde{u}_{r} \tilde{u}_{\theta}$'
            
            ## +(2/r)·ρ·ur·uθ
            t9x2 = np.copy( (2/r) * r_urII_utII  )
            t9x2_lbl = r'$\frac{2}{r} \overline{ \rho u^{\prime \prime}_r u^{\prime \prime}_\theta }$'
            
            ## -(2/r)·τ_rθ
            t10x2 = np.copy( -(2/r) * tau_rt )
            t10x2_lbl = r'$-\frac{2}{r} \overline{\tau}_{r \theta}$'
        
        # ===
        
        data_r = {'t1':np.copy(t1x1),
                  't2':np.copy(t2x1),
                  't3':np.copy(t3x1),
                  't4':np.copy(t4x1),
                  't5':np.copy(t5x1),
                  't6':np.copy(t6x1),
                  't7':np.copy(t7x1),
                  't8':np.copy(t8x1),
                  't9':np.copy(t9x1),
                  't10':np.copy(t10x1),
                  't11':np.copy(t11x1),
                  't12':np.copy(t12x1),
                  't13':np.copy(t13x1),
                  }
        
        data_t = {'t1':np.copy(t1x2),
                  't2':np.copy(t2x2),
                  't3':np.copy(t3x2),
                  't4':np.copy(t4x2),
                  't5':np.copy(t5x2),
                  't6':np.copy(t6x2),
                  't7':np.copy(t7x2),
                  't8':np.copy(t8x2),
                  't9':np.copy(t9x2),
                  't10':np.copy(t10x2),
                  }
        
        return data_r, data_t
    
    def calc_momentum_balance_terms(self, **kwargs):
        '''
        calculate terms of the 2D time-averaged momentum conservation equation in Cartesian coords
        '''
        
        verbose      = kwargs.get('verbose',True)
        local_csys   = kwargs.get('local_csys',False)
        acc          = kwargs.get('acc',8)
        edge_stencil = kwargs.get('edge_stencil','half')
        xis          = kwargs.get('xis',None)
        csys_type    = kwargs.get('csys_type','cartesian')
        
        if verbose: print('\n'+'ztmd.calc_momentum_balance_terms()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not (self.open_mode=='a') or (self.open_mode=='w') or (self.open_mode=='r+'):
            raise ValueError('not able to write to hdf5 file')
        
        if verbose: even_print('local_csys',str(local_csys))
        if verbose: even_print('csys_type',csys_type)
        
        if local_csys:
            if self.rectilinear:
                print('>>> WARNING: this grid is rectilinear. Using local csys is not needed.')
            if not ('csys/vtang' in self):
                raise ValueError('csys/vtang not in hdf5')
            if not ('csys/vnorm' in self):
                raise ValueError('csys/vnorm not in hdf5')
            
            ## read unit vectors (wall tangent, wall norm) from HDF5
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            if verbose: even_print('vtang',str(vtang.shape))
            if verbose: even_print('vnorm',str(vnorm.shape))
        
        if (csys_type=='cartesian'):
            gnx = 'data_momentum_x'
            gny = 'data_momentum_y'
            gns = [gnx,gny]
        elif (csys_type=='cylindrical'):
            gnr = 'data_momentum_r'
            gnt = 'data_momentum_theta'
            gns = [gnr,gnt]
        else:
            raise NotImplementedError
        
        terms_x = {
            't1':r'$\partial_{x}[\bar{\rho} \tilde{u} \tilde{u}]$',
            't2':r'$\partial_{y}[\bar{\rho} \tilde{u} \tilde{v}]$',
            't3':r'$\partial_{x}[\bar{p}]$',
            't4':r'$-\partial_{x}[\overline{\tau_{xx}}]$',
            't5':r'$\partial_{x}[\overline{\rho u^{\prime\prime} u^{\prime\prime}}]$',
            't6':r'$-\partial_{y}[\overline{\tau_{xy}}]$',
            't7':r'$\partial_{y}[\overline{\rho u^{\prime\prime} v^{\prime\prime}}]$',
            }
        
        terms_y = {
            't1':r'$\partial_{x}[\bar{\rho} \tilde{u} \tilde{v}]$',
            't2':r'$\partial_{y}[\bar{\rho} \tilde{v} \tilde{v}]$',
            't3':r'$\partial_{y}[\bar{p}]$',
            't4':r'$-\partial_{x}[\overline{\tau_{xy}}]$',
            't5':r'$\partial_{x}[\overline{\rho u^{\prime\prime} v^{\prime\prime}}]$',
            't6':r'$-\partial_{y}[\overline{\tau_{yy}}]$',
            't7':r'$\partial_{y}[\overline{\rho v^{\prime\prime} v^{\prime\prime}}]$',
            }
        
        terms_r = {
            't1':r'$\partial_{r}[ \bar{\rho} \tilde{u}_r \tilde{u}_r ]$',
            't2':r'$\frac{1}{r} \partial_{\theta}[ \bar{\rho} \tilde{u}_r \tilde{u}_\theta ]$',
            't3':r'$\partial_{r}[\bar{p}]$',
            't4':r'$-\partial_{r}[ \overline{\tau}_{r r} ]$',
            't5':r'$\partial_{r}[ \overline{ \rho u^{\prime \prime}_r  u^{\prime \prime}_r } ]$',
            't6':r'$-\frac{1}{r} \partial_{\theta}[ \overline{\tau}_{r \theta} ]$',
            't7':r'$\frac{1}{r} \partial_{\theta}[ \overline{\rho u^{\prime \prime}_r  u^{\prime \prime}_\theta } ]$',
            't8':r'$\frac{1}{r} \bar{\rho} \tilde{u}_{r} \tilde{u}_{r}$',
            't9':r'$\frac{1}{r} \overline{ \rho u^{\prime \prime}_r u^{\prime \prime}_r }$',
            't10':r'$-\frac{1}{r} \bar{\rho} \tilde{u}_{\theta} \tilde{u}_{\theta}$',
            't11':r'$-\frac{1}{r} \overline{ \rho u^{\prime \prime}_\theta u^{\prime \prime}_\theta }$',
            't12':r'$-\frac{1}{r} \overline{\tau}_{r r}$',
            't13':r'$\frac{1}{r} \overline{\tau}_{\theta \theta}$',
            }
        
        terms_theta = {
            't1':r'$\partial_{r}[ \bar{\rho} \tilde{u}_r \tilde{u}_\theta ]$',
            't2':r'$\frac{1}{r} \partial_{\theta}[ \bar{\rho} \tilde{u}_\theta \tilde{u}_\theta ]$',
            't3':r'$\frac{1}{r} \partial_{\theta}[\bar{p}]$',
            't4':r'$-\partial_{r}[ \overline{\tau}_{r \theta} ]$',
            't5':r'$\partial_{r}[ \overline{\rho u^{\prime \prime}_r  u^{\prime \prime}_\theta } ]$',
            't6':r'$-\frac{1}{r} \partial_{\theta}[ \overline{\tau}_{\theta \theta} ]$',
            't7':r'$\frac{1}{r} \partial_{\theta}[ \overline{\rho u^{\prime \prime}_\theta  u^{\prime \prime}_\theta } ]$',
            't8':r'$\frac{2}{r} \bar{\rho} \tilde{u}_{r} \tilde{u}_{\theta}$',
            't9':r'$\frac{2}{r} \overline{ \rho u^{\prime \prime}_r u^{\prime \prime}_\theta }$',
            't10':r'$-\frac{2}{r} \overline{\tau}_{r \theta}$',
            }
        
        ## create groups
        for gn in gns:
            
            if verbose: print(72*'-')
            
            if (gn in self):
                del self[gn]
                if verbose: even_print('deleted group',f'/{gn}/')
            
            grp = self.create_group(gn)
            if verbose: even_print('created group',f'{grp.name}/')
            
            ## set group attributes
            grp.attrs['local_csys'] = local_csys
            if verbose: even_print(f'{gn}.local_csys',str(grp.attrs['local_csys']))
        
        gn=None; del gn
        
        if verbose: print(72*'-')
        
        ## initialize dsets for [x] terms
        if (csys_type=='cartesian'):
            for dsn, tag in terms_x.items():
                shape = (self.ny,self.nx)
                if (f'{gnx}/{dsn}' in self): del self[f'{gnx}/{dsn}']
                dset = self.create_dataset(f'{gnx}/{dsn}', 
                                           shape=shape, 
                                           dtype=np.float32,
                                           chunks=None)
                if verbose: even_print(f'{gnx}/{dsn}',str(shape))
                dset.attrs['latex'] = tag
                if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
            if verbose: print(72*'-')
        
        ## initialize dsets for [y] terms
        if (csys_type=='cartesian'):
            for dsn, tag in terms_y.items():
                shape = (self.ny,self.nx)
                if (f'{gny}/{dsn}' in self): del self[f'{gny}/{dsn}']
                dset = self.create_dataset(f'{gny}/{dsn}', 
                                           shape=shape, 
                                           dtype=np.float32,
                                           chunks=None)
                if verbose: even_print(f'{gny}/{dsn}',str(shape))
                dset.attrs['latex'] = tag
                if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
            if verbose: print(72*'-')
        
        ## initialize dsets for [r] terms
        if (csys_type=='cylindrical'):
            for dsn, tag in terms_r.items():
                shape = (self.ny,self.nx)
                if (f'{gnr}/{dsn}' in self): del self[f'{gnr}/{dsn}']
                dset = self.create_dataset(f'{gnr}/{dsn}', 
                                           shape=shape, 
                                           dtype=np.float32,
                                           chunks=None)
                if verbose: even_print(f'{gnr}/{dsn}',str(shape))
                dset.attrs['latex'] = tag
                if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
            if verbose: print(72*'-')
        
        ## initialize dsets for [θ] terms
        if (csys_type=='cylindrical'):
            for dsn, tag in terms_theta.items():
                shape = (self.ny,self.nx)
                if (f'{gnt}/{dsn}' in self): del self[f'{gnt}/{dsn}']
                dset = self.create_dataset(f'{gnt}/{dsn}', 
                                           shape=shape, 
                                           dtype=np.float32,
                                           chunks=None)
                if verbose: even_print(f'{gnt}/{dsn}',str(shape))
                dset.attrs['latex'] = tag
                if verbose: even_print(f'{dsn}.latex',str(dset.attrs['latex']))
            if verbose: print(72*'-')
        
        ## copy data
        
        if (csys_type=='cylindrical') and hasattr(self,'crv_R'):
            crv_R = np.copy(self.crv_R)
            if verbose: even_print('crv_R',str(crv_R.shape))
        
        if verbose: print('reading data'+'\n'+'------------')
        u   = np.copy( self['data/u'][()].T   )
        v   = np.copy( self['data/v'][()].T   )
        #w   = np.copy( self['data/w'][()].T   )
        rho = np.copy( self['data/rho'][()].T )
        p   = np.copy( self['data/p'][()].T   )
        #T   = np.copy( self['data/T'][()].T   )
        mu  = np.copy( self['data/mu'][()].T  )
        
        if verbose: even_print('u',str(u.shape))
        if verbose: even_print('v',str(v.shape))
        if verbose: even_print('ρ',str(rho.shape))
        if verbose: even_print('p',str(p.shape))
        if verbose: even_print('μ',str(mu.shape))
        
        u_Fv = np.copy( self['data/u_Fv'][()].T )
        v_Fv = np.copy( self['data/v_Fv'][()].T )
        if verbose: even_print('u_Fv',str(u_Fv.shape))
        if verbose: even_print('v_Fv',str(v_Fv.shape))
        
        r_uII_uII = np.copy( self['data/r_uII_uII'][()].T )
        r_vII_vII = np.copy( self['data/r_vII_vII'][()].T )
        #r_wII_wII = np.copy( self['data/r_wII_wII'][()].T )
        r_uII_vII = np.copy( self['data/r_uII_vII'][()].T )
        #r_uII_wII = np.copy( self['data/r_uII_wII'][()].T )
        #r_vII_wII = np.copy( self['data/r_vII_wII'][()].T )
        if verbose: even_print('ρu″u″',str(r_uII_uII.shape))
        if verbose: even_print('ρu″v″',str(r_uII_vII.shape))
        if verbose: even_print('ρv″v″',str(r_vII_vII.shape))
        
        x = np.copy(self.x)
        y = np.copy(self.y)
        
        if (x.ndim==1) and (y.ndim==1):
            x,y = np.meshgrid(x,y, indexing='ij')
        if verbose: even_print('x',str(x.shape))
        if verbose: even_print('y',str(y.shape))
        if verbose: print(72*'-')
        
        # ===
        
        nx,ny = x.shape
        
        if local_csys:
            
            # ## calculate grid points for specific postions in char. lengths
            # s_vert_locs = [-100,-50,-25,0,+25,+50,+100]
            # svert  = (self.stang-250*self.lchar)/self.lchar
            # xis    = [ np.abs(s-svert).argmin() for s in s_vert_locs ]
            
            if (xis is None):
                xis = np.arange(self.nx)
                n_prog = self.nx
            else:
                n_prog = len(xis)
            
            if (csys_type=='cartesian'):
                desc='momentum balance [x,y]'
            elif (csys_type=='cylindrical'):
                desc='momentum balance [r,θ]'
            else:
                raise ValueError
            
            if verbose: progress_bar = tqdm(total=n_prog, ncols=100, desc=desc, leave=False, file=sys.stdout)
            
            ## iterate through axis=0 indices
            for xi in xis:
                
                rotation_angle_z = np.arcsin(vnorm[xi,0,0])
                #rotation_angle_z = np.arcsin(-vtang[xi,0,1])
                #if verbose: tqdm.write( even_print('rotation_angle_z', f'{np.degrees(rotation_angle_z):0.14f} [deg]', s=True) )
                
                rotation_matrix_1 = np.array( [[ np.cos(rotation_angle_z) , -np.sin(rotation_angle_z) , 0 ],
                                               [ np.sin(rotation_angle_z) ,  np.cos(rotation_angle_z) , 0 ],
                                               [ 0                        ,  0                        , 1 ]], dtype=np.float64 )
                
                ## rotation_matrix_2 = np.array( [[ vtang[xi,0,0] ,  vtang[xi,0,1] , 0 ],
                ##                                [ vnorm[xi,0,0] ,  vnorm[xi,0,1] , 0 ],
                ##                                [ 0             ,  0             , 1 ]], dtype=np.float64 )
                ## 
                ## ## just showing that these are the same
                ## np.testing.assert_allclose( rotation_matrix_1 , rotation_matrix_2 , rtol=1e-14 , atol=1e-14 )
                
                rotation_matrix = np.copy(rotation_matrix_1)
                
                ## translate to [cx,cy]
                cx = x[xi,0]
                cy = y[xi,0]
                translation_matrix_A = np.array( [[ 1 , 0 , -cx ],
                                                  [ 0 , 1 , -cy ],
                                                  [ 0 , 0 ,  1  ]], dtype=np.float64 )
                
                ## translate in [y] by -R (the local curvature radius)
                if (csys_type=='cylindrical'):
                    cx = 0.
                    cy = crv_R[xi]
                    translation_matrix_B = np.array( [[ 1 , 0 ,  0  ],
                                                      [ 0 , 1 , -cy ],
                                                      [ 0 , 0 ,  1  ]], dtype=np.float64 )
                else:
                    translation_matrix_B = np.identity(3, dtype=np.float64)
                
                ## form the full [3x3] matrix by multiplying together --> order matters!
                transformation_matrix = np.einsum( 'ij,jk,kl->il', translation_matrix_B , rotation_matrix, translation_matrix_A )
                
                # ===
                
                ## send to kernel
                if (csys_type=='cartesian'):
                    
                    data_x, data_y = ztmd.momentum_balance_kernel_cartesian(x=x,y=y,
                                                                            u=u,v=v,
                                                                            u_Fv=u_Fv,v_Fv=v_Fv,
                                                                            p=p, rho=rho, mu=mu,
                                                                            r_uII_uII=r_uII_uII,
                                                                            r_uII_vII=r_uII_vII,
                                                                            r_vII_vII=r_vII_vII,
                                                                            transformation_matrix=transformation_matrix,
                                                                            acc=acc,edge_stencil=edge_stencil )
                    
                    ## write to HDF5 (only data from single axis=0 index)
                    for dsn, arr in data_x.items():
                        dset = self[f'{gnx}/{dsn}']
                        dset[:,xi] = arr[xi,:] ## arrays in HDF5 are transposed
                    for dsn, arr in data_y.items():
                        dset = self[f'{gny}/{dsn}']
                        dset[:,xi] = arr[xi,:]
                
                elif (csys_type=='cylindrical'):
                    
                    data_r, data_t = ztmd.momentum_balance_kernel_cylindrical(x=x,y=y,
                                                                              u=u,v=v,
                                                                              u_Fv=u_Fv,v_Fv=v_Fv,
                                                                              p=p, rho=rho, mu=mu,
                                                                              r_uII_uII=r_uII_uII,
                                                                              r_uII_vII=r_uII_vII,
                                                                              r_vII_vII=r_vII_vII,
                                                                              transformation_matrix=transformation_matrix,
                                                                              acc=acc,edge_stencil=edge_stencil )
                    
                    ## write to HDF5 (only data from single axis=0 index)
                    for dsn, arr in data_r.items():
                        dset = self[f'{gnr}/{dsn}']
                        dset[:,xi] = arr[xi,:] ## arrays in HDF5 are transposed
                    for dsn, arr in data_t.items():
                        dset = self[f'{gnt}/{dsn}']
                        dset[:,xi] = arr[xi,:]
                
                else:
                    raise ValueError
                
                if verbose: progress_bar.update()
            if verbose: progress_bar.close()
        
        else:
            
            ## send to kernel
            if (csys_type=='cartesian'):
                
                data_x, data_y = ztmd.momentum_balance_kernel_cartesian(
                                                                    x=x,y=y,
                                                                    u=u,v=v,
                                                                    u_Fv=u_Fv,v_Fv=v_Fv,
                                                                    p=p, rho=rho, mu=mu,
                                                                    r_uII_uII=r_uII_uII,
                                                                    r_uII_vII=r_uII_vII,
                                                                    r_vII_vII=r_vII_vII,
                                                                    #transformation_matrix=transformation_matrix,
                                                                    acc=acc,edge_stencil=edge_stencil,
                                                                    )
                
                ## write to HDF5 (only data from single axis=0 index)
                for dsn, arr in data_x.items():
                    dset = self[f'{gnx}/{dsn}']
                    if verbose: even_print(dset.name,str(arr.T.shape))
                    dset[:,:] = arr.T ## arrays in HDF5 are transposed
                for dsn, arr in data_y.items():
                    dset = self[f'{gny}/{dsn}']
                    if verbose: even_print(dset.name,str(arr.T.shape))
                    dset[:,:] = arr.T ## arrays in HDF5 are transposed
            
            elif (csys_type=='cylindrical'):
                
                data_r, data_t = ztmd.momentum_balance_kernel_cylindrical(
                                                                        x=x,y=y,
                                                                        u=u,v=v,
                                                                        u_Fv=u_Fv,v_Fv=v_Fv,
                                                                        p=p, rho=rho, mu=mu,
                                                                        r_uII_uII=r_uII_uII,
                                                                        r_uII_vII=r_uII_vII,
                                                                        r_vII_vII=r_vII_vII,
                                                                        #transformation_matrix=transformation_matrix,
                                                                        acc=acc,edge_stencil=edge_stencil,
                                                                        )
                
                ## write to HDF5 (only data from single axis=0 index)
                for dsn, arr in data_r.items():
                    dset = self[f'{gnr}/{dsn}']
                    dset[:,xi] = arr[xi,:] ## arrays in HDF5 are transposed
                for dsn, arr in data_t.items():
                    dset = self[f'{gnt}/{dsn}']
                    dset[:,xi] = arr[xi,:]
            
            else:
                raise ValueError
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_momentum_balance_terms() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # === post-processing: 1D profiles
    
    def calc_bl_edge(self, **kwargs):
        '''
        determine the boundary layer edge location
        -----
        if 'method'=='u'         : 'edge' is where |du+/dy+|<ϵ
        if 'method'=='vorticity' : 'edge' is where |-ω+|<ϵ
        -----
        y_edge : the wall-normal edge location
        j_edge : the nearest index to y_edge
        -----
        ongrid : snap to [y] grid point
        '''
        
        verbose      = kwargs.get('verbose',True)
        method       = kwargs.get('method','vorticity') ## 'u','vorticity'
        epsilon      = kwargs.get('epsilon',5e-5)
        acc          = kwargs.get('acc',6)
        edge_stencil = kwargs.get('edge_stencil','full')
        interp_kind  = kwargs.get('interp_kind','cubic') ## 'linear','cubic'
        ongrid       = kwargs.get('ongrid',True) ## snap to grid point
        
        if verbose: print('\n'+'ztmd.calc_bl_edge()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check rectilinear/curvilinear
        if (self.x.ndim==1) and (self.y.ndim==1):
            if hasattr(self,'rectilinear'):
                if not self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if self.curvilinear:
                    raise AssertionError
        elif (self.x.ndim==2) and (self.y.ndim==2):
            if hasattr(self,'rectilinear'):
                if self.rectilinear:
                    raise AssertionError
            if hasattr(self,'curvilinear'):
                if not self.curvilinear:
                    raise AssertionError
        else:
            raise ValueError
        
        ## grids that don't have even s2 vectors
        if self.requires_wall_norm_interp:
            raise NotImplementedError
        
        if not any([(method=='u'),(method=='vorticity')]):
            raise ValueError(f"'method' should be one of: 'u','vorticity'")
        if not any([(interp_kind=='linear'),(interp_kind=='cubic')]):
            raise ValueError(f"'interp_kind' should be one of: 'linear','cubic'")
        
        if verbose: even_print('method',method)
        if verbose: even_print('epsilon','%0.1e'%(epsilon,))
        if verbose: even_print('acc',f'{acc:d}')
        if verbose: even_print('edge_stencil',edge_stencil)
        if verbose: even_print('1D interp kind',interp_kind)
        if verbose: even_print('ongrid',str(ongrid))
        
        # ===
        
        nx = self.nx
        ny = self.ny
        
        ## copy 2D datasets into memory
        u = np.copy( self['data/u'][()].T ) ## dimensional
        
        sc_u_in = np.copy( self['data_1Dx/sc_u_in'][()] ) ## uτ
        sc_l_in = np.copy( self['data_1Dx/sc_l_in'][()] ) ## δν = νw/uτ
        
        if self.rectilinear:
            
            x = np.copy( self['dims/x'][()] ) ## dimensional
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T ) ## 2D
            y = np.copy( self['dims/y'][()].T ) ## 2D
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            utang = np.copy( self['data/utang'][()].T )
            unorm = np.copy( self['data/unorm'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        # ===
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # ===
        
        if (method=='vorticity'): ## use -ωz i.e. where |-ωz|<ϵ
            if not ('data/vort_z' in self):
                raise ValueError('data/vort_z not in ztmd')
            vort_z = np.copy( self['data/vort_z'][()].T )
        
        # ===
        
        y_edge    = np.zeros(shape=(nx,)  , dtype=np.float64 )
        j_edge    = np.zeros(shape=(nx,)  , dtype=np.int32   )
        y_edge_2d = np.zeros(shape=(nx,2) , dtype=np.float64 )
        y_edge_g  = np.zeros(shape=(nx,)  , dtype=np.float64 )
        
        if (method=='vorticity'): ## ωz is already a derivative, dont need to pre-compute
            pass
        elif (method=='u'): ## pre-compute du/dy for efficiency
            if self.rectilinear:
                ddy_var = gradient(
                                u,
                                y_,
                                axis=1,
                                d=1,
                                acc=acc,
                                edge_stencil=edge_stencil,
                                )
            elif self.curvilinear:
                ddy_var = gradient(
                                utang,
                                y_,
                                axis=1,
                                d=1,
                                acc=acc,
                                edge_stencil=edge_stencil,
                                )
            else:
                raise ValueError
        else:
            raise ValueError
        
        if verbose: progress_bar = tqdm(total=nx, ncols=100, desc='y_edge', leave=False, file=sys.stdout)
        for i in range(nx):
            
            do_debug_plot = False
            #if (i==5000):
            #    do_debug_plot = True
            
            if (method=='u'): ## |du+/dy+|<ϵ
                
                ddy_u_      = np.copy( ddy_var[i,:] ) ## du/dy
                ddy_u_plus_ = np.copy( ddy_u_ / ( sc_u_in[i] / sc_l_in[i] ) ) ## du+/dy+ = (du/dy)/(uτ/δν) = (du/dy)/(uτ^2/νw)
                
                y_edge_ = calc_profile_edge_1d(
                                y=y_,
                                ddy_u=ddy_u_plus_,
                                ongrid=ongrid,
                                epsilon=epsilon,
                                acc=acc,
                                edge_stencil=edge_stencil,
                                interp_kind=interp_kind,
                                do_debug_plot=do_debug_plot,
                                )
            
            elif (method=='vorticity'): ## |-ωz+|<ϵ
                
                vort_z_      = np.copy( vort_z[i,:] ) ## ωz = (dv/dx)-(du/dy)
                vort_z_plus_ = np.copy( vort_z_ / ( sc_u_in[i] / sc_l_in[i] ) ) ## ωz+ = ωz/(uτ/δν) = ωz/(uτ^2/νw)
                
                y_edge_ = calc_profile_edge_1d(
                                y=y_,
                                ddy_u=-1*vort_z_plus_,
                                ongrid=ongrid,
                                epsilon=epsilon,
                                acc=acc,
                                edge_stencil=edge_stencil,
                                interp_kind=interp_kind,
                                do_debug_plot=do_debug_plot,
                                )
            
            else:
                raise ValueError
            
            # ===
            
            y_edge[i]   = y_edge_
            j_edge_     = np.abs( y_ - y_edge_ ).argmin()
            j_edge[i]   = j_edge_
            y_edge_g[i] = y_[j_edge_]
            
            if ongrid:
                y_edge[i] = y_[j_edge_]
            
            ## get the [x,y] coordinates of the 'edge line' --> shape=(nx,2)
            if self.rectilinear:
                pt_edge_ = np.array([self.x[i],y_edge_], dtype=np.float64)
            elif self.curvilinear:
                p0_ = np.array([self.x[i,0],self.y[i,0]], dtype=np.float64)
                vnorm_ = np.copy( vnorm[i,0,:] ) ## unit normal vec @ wall at this x
                pt_edge_ = p0_ + np.dot( y_edge_ , vnorm_ )
            else:
                raise ValueError
            
            y_edge_2d[i,:] = pt_edge_
            
            progress_bar.update()
        progress_bar.close()
        
        if ongrid:
            np.testing.assert_allclose(y_edge,y_edge_g,rtol=1e-5)
        
        if ('data_1Dx/y_edge' in self): del self['data_1Dx/y_edge']
        dset = self.create_dataset('data_1Dx/y_edge', data=y_edge, chunks=None)
        if verbose: even_print('data_1Dx/y_edge','%s'%str(y_edge.shape))
        
        if ('data_1Dx/y_edge_g' in self): del self['data_1Dx/y_edge_g']
        dset = self.create_dataset('data_1Dx/y_edge_g', data=y_edge_g, chunks=None)
        if verbose: even_print('data_1Dx/y_edge_g','%s'%str(y_edge_g.shape))
        
        if ('data_1Dx/y_edge_2d' in self): del self['data_1Dx/y_edge_2d']
        dset = self.create_dataset('data_1Dx/y_edge_2d', data=y_edge_2d, chunks=None)
        if verbose: even_print('data_1Dx/y_edge_2d','%s'%str(y_edge_2d.shape))
        
        if ('data_1Dx/j_edge' in self): del self['data_1Dx/j_edge']
        dset = self.create_dataset('data_1Dx/j_edge', data=j_edge, chunks=None)
        if verbose: even_print('data_1Dx/j_edge','%s'%str(j_edge.shape))
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_bl_edge() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_bl_edge_quantities(self, **kwargs):
        '''
        calculate field quantity values at [y_edge]
        - calculate friction coefficient: cf = 2·τw/(ρe·ue^2) = 2/(ρe+·(ue+)^2)
        '''
        
        verbose     = kwargs.get('verbose',True)
        interp_kind = kwargs.get('interp_kind','cubic') ## 'linear','cubic'
        
        if verbose: print('\n'+'ztmd.calc_bl_edge_quantities()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        # ===
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        y_edge   = np.copy( self['data_1Dx/y_edge'][()]   )
        y_edge_g = np.copy( self['data_1Dx/y_edge_g'][()] )
        j_edge   = np.copy( self['data_1Dx/j_edge'][()]   )
        
        ## was calc_bl_edge() run with ongrid=True?
        if np.allclose(y_edge,y_edge_g,rtol=1e-6):
            ongrid = True
        else:
            ongrid = False
        
        if ongrid:
            np.testing.assert_allclose(y_edge,y_edge_g,rtol=1e-6)
            np.testing.assert_allclose(
                y_edge_g,
                np.array([ self.y[j] for j in j_edge ],dtype=y_edge.dtype),
                rtol=1e-6,
                )
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # === make a numpy structured array
        
        names  = [ 'rho', 'u', 'v', 'w', 'T', 'p', 'vort_z', 'mu', 'nu', 'M' ]
        if ('data/u_inc' in self):
            names += [ 'u_inc' ]
        if ('data/psvel' in self):
            names += [ 'psvel' ]
        if ('data/utang' in self):
            names += [ 'utang' ]
        if ('data/unorm' in self):
            names += [ 'unorm' ]
        if ('data/umag' in self):
            names += [ 'umag' ]
        
        dtypes=[]
        for n in names:
            ds = self[f'data/{n}']
            dtypes.append( ds.dtype )
        
        names_edge = [ n+'_edge' for n in names ]
        
        data      = np.zeros(shape=(nx,ny), dtype={'names':names,      'formats':dtypes})
        data_edge = np.zeros(shape=(nx,),   dtype={'names':names_edge, 'formats':dtypes})
        
        ## populate 2D structured array with data to find edge for
        for scalar in data.dtype.names:
            data[scalar][:,:] = np.copy( self[f'data/{scalar}'][()].T )
        
        # === interpolate edge quantity for all vars
        
        if verbose: progress_bar = tqdm(total=nx*len(names), ncols=100, desc='edge quantities', leave=False, file=sys.stdout)
        for scalar in data.dtype.names:
            for i in range(nx):
                if ongrid:
                    je = j_edge[i]
                    data_edge_ = data[scalar][i,je]
                else:
                    data_y_    = np.copy( data[scalar][i,:] )
                    intrp_func = sp.interpolate.interp1d(y_, data_y_, kind=interp_kind, bounds_error=True)
                    data_edge_ = intrp_func(y_edge[i])
                data_edge[scalar+'_edge'][i] = data_edge_
                if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # === write
        
        for scalar in data_edge.dtype.names:
            if (f'data_1Dx/{scalar}' in self):
                del self[f'data_1Dx/{scalar}']
            data_ = np.copy( data_edge[scalar][:] )
            dset = self.create_dataset(f'data_1Dx/{scalar}', data=data_, chunks=None)
            if verbose: even_print(f'data_1Dx/{scalar}',str(dset.shape))
        
        # ===
        
        if False:
            plt.close('all')
            fig1 = plt.figure(figsize=(3*2,3), dpi=300)
            ax1 = plt.gca()
            ax1.plot( stang/self.lchar, data_edge['utang_edge']/self.U_inf, lw=0.5 )
            ax1.set_xlabel('stang')
            fig1.tight_layout(pad=0.25)
            fig1.tight_layout(pad=0.25)
            plt.show()
        
        # === cf (friction coefficient) = 2·τw/(ρe·ue^2) = 2/(ρe+·(ue+)^2)
        
        u_tau    = np.copy( self['data_1Dx/u_tau'][()]    )
        rho_wall = np.copy( self['data_1Dx/rho_wall'][()] )
        tau_wall = np.copy( self['data_1Dx/tau_wall'][()] )
        
        u_edge   = np.copy( data_edge['u_edge']   )
        rho_edge = np.copy( data_edge['rho_edge'] )
        
        if self.curvilinear:
            utang_edge = np.copy( data_edge['utang_edge'] )
        
        if self.rectilinear:
            
            ## assert cf formulas
            cf_1 = np.copy( 2. * (u_tau/u_edge)**2 * (rho_wall/rho_edge) )
            cf_2 = np.copy( 2. * tau_wall / (rho_edge*u_edge**2) )
            np.testing.assert_allclose(cf_1, cf_2, rtol=1e-6, atol=1e-8)
            cf_1 = None ; del cf_1
            cf_2 = None ; del cf_2
            
            cf_inf  = np.copy( 2. * tau_wall / ( self.rho_inf  * self.U_inf**2  ) )
            cf_edge = np.copy( 2. * tau_wall / (      rho_edge *      u_edge**2 ) )
            #np.testing.assert_allclose(cf_inf, cf_edge, rtol=0.01)
            
            ## take 'edge' cf rather than 'inf' cf
            cf = np.copy(cf_edge)
        
        elif self.curvilinear:
            
            # cf_1 = 2. * (u_tau/utang_edge)**2 * (rho_wall/rho_edge)
            # cf_2 = 2. * tau_wall / (rho_edge*utang_edge**2)
            # np.testing.assert_allclose(cf_1, cf_2, rtol=1e-6, atol=1e-8)
            # cf = np.copy(cf_2)
            
            cf_inf = np.copy( 2. * tau_wall / ( self.rho_inf * self.U_inf**2 ) )
            cf = np.copy(cf_inf)
        
        else:
            raise ValueError
        
        if ('data_1Dx/cf' in self): del self['data_1Dx/cf']
        self.create_dataset('data_1Dx/cf', data=cf, chunks=None)
        if verbose: even_print('data_1Dx/cf', '%s'%str(cf.shape))
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_bl_edge_quantities() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_d99(self, **kwargs):
        '''
        determine δ
        δ = δ99 = y[ u(y) == 0.99*u_edge ]
        'u' can be pseudovelocity or streamwise velocity (set with 'method')
        '''
        
        verbose     = kwargs.get('verbose',True)
        method      = kwargs.get('method','psvel') ## 'u','psvel'
        interp_kind = kwargs.get('interp_kind','cubic') ## 'linear','cubic'
        #rtol        = kwargs.get('rtol',1e-3) ## used by calc_d99_1d() for asserting u[y_edge]==u_edge
        rtol = 1e-3 ## now hardcoded rather than kwarg (inconsequential)
        
        if verbose: print('\n'+'ztmd.calc_d99()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        # ...
        
        if not any([(method=='u'),(method=='psvel')]):
            raise ValueError(f"'method' should be one of: 'u','psvel'")
        if not any([(interp_kind=='linear'),(interp_kind=='cubic')]):
            raise ValueError(f"'interp_kind' should be one of: 'linear','cubic'")
        
        if verbose: even_print('method',method)
        #if verbose: even_print('rtol','%0.1e'%(rtol,))
        if verbose: even_print('1D interp kind',interp_kind)
        
        # ===
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            ## copy dims into memory (1D)
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            ## copy dims into memory (2D)
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        ## the wall normal location of BL edge, as determined in e.g. ztmd.calc_bl_edge()
        y_edge   = np.copy( self['data_1Dx/y_edge'][()] )
        y_edge_g = np.copy( self['data_1Dx/y_edge_g'][()] )
        
        ## the index closest y_edge
        j_edge = np.copy( self['data_1Dx/j_edge'][()] )
        
        ## get pseudovelocity / u / utang
        if (method=='psvel'):
            var      = np.copy( self['data/psvel'][()].T )
            var_edge = np.copy( self['data_1Dx/psvel_edge'][()].T )
        elif (method=='u'):
            if self.rectilinear:
                var      = np.copy( self['data/u'][()].T )
                var_edge = np.copy( self['data_1Dx/u_edge'][()] )
            elif self.curvilinear:
                var      = np.copy( self['data/utang'][()].T )
                var_edge = np.copy( self['data_1Dx/utang_edge'][()] )
            else:
                raise ValueError
        else:
            raise ValueError
        
        # ===
        
        d99     = np.zeros(shape=(nx,),  dtype=np.float64 )
        d95     = np.zeros(shape=(nx,),  dtype=np.float64 )
        d99_2d  = np.zeros(shape=(nx,2), dtype=np.float64 )
        d95_2d  = np.zeros(shape=(nx,2), dtype=np.float64 )
        
        j99     = np.zeros(shape=(nx,),  dtype=np.int32   )
        d99g    = np.zeros(shape=(nx,),  dtype=np.float64 )
        
        if verbose: progress_bar = tqdm(total=nx, ncols=100, desc='δ', leave=False, file=sys.stdout)
        for i in range(nx):
            
            y_edge_   = y_edge[i]
            var_      = np.copy( var[i,:] )
            var_edge_ = var_edge[i]
            
            d99_   = calc_d99_1d(y=y_, u=var_, y_edge=y_edge_, u_edge=var_edge_, interp_kind=interp_kind, d95=False)
            d99[i] = d99_
            
            d95_   = calc_d99_1d(y=y_, u=var_, y_edge=y_edge_, u_edge=var_edge_, interp_kind=interp_kind, d95=True)
            d95[i] = d95_
            
            j99_    = np.abs( y_ - d99_ ).argmin()
            j99[i]  = j99_
            d99g[i] = y_[j99_]
            
            # ===
            
            ## get the [x,y] coordinates of the 'd99 line' --> shape=(nx,2)
            if self.rectilinear:
                pt_99_ = np.array([self.x[i],d99_], dtype=np.float64)
                pt_95_ = np.array([self.x[i],d95_], dtype=np.float64)
            elif self.curvilinear:
                p0_ = np.array([self.x[i,0],self.y[i,0]], dtype=np.float64)
                vnorm_ = np.copy( vnorm[i,0,:] ) ## unit normal vec @ wall at this x
                pt_99_ = p0_ + np.dot( d99_ , vnorm_ )
                pt_95_ = p0_ + np.dot( d95_ , vnorm_ )
            else:
                raise ValueError
            
            d99_2d[i,:] = pt_99_
            d95_2d[i,:] = pt_95_
            
            progress_bar.update()
        progress_bar.close()
        
        if ('data_1Dx/d99' in self): del self['data_1Dx/d99']
        dset = self.create_dataset('data_1Dx/d99', data=d99, chunks=None)
        if verbose: even_print('data_1Dx/d99','%s'%str(d99.shape))
        
        if ('data_1Dx/d95' in self): del self['data_1Dx/d95']
        dset = self.create_dataset('data_1Dx/d95', data=d95, chunks=None)
        if verbose: even_print('data_1Dx/d95','%s'%str(d95.shape))
        
        if ('data_1Dx/d99_2d' in self): del self['data_1Dx/d99_2d']
        dset = self.create_dataset('data_1Dx/d99_2d', data=d99_2d, chunks=None)
        if verbose: even_print('data_1Dx/d99_2d','%s'%str(d99_2d.shape))
        
        if ('data_1Dx/d95_2d' in self): del self['data_1Dx/d95_2d']
        dset = self.create_dataset('data_1Dx/d95_2d', data=d95_2d, chunks=None)
        if verbose: even_print('data_1Dx/d95_2d','%s'%str(d95_2d.shape))
        
        if ('data_1Dx/d99g' in self): del self['data_1Dx/d99g']
        dset = self.create_dataset('data_1Dx/d99g', data=d99g, chunks=None)
        if verbose: even_print('data_1Dx/d99g','%s'%str(d99g.shape))
        
        if ('data_1Dx/j99' in self): del self['data_1Dx/j99']
        dset = self.create_dataset('data_1Dx/j99', data=j99, chunks=None)
        if verbose: even_print('data_1Dx/j99','%s'%str(j99.shape))
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_d99() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_d99_quantities(self, **kwargs):
        '''
        calculate interpolated field quantity values at y=δ
        - sc_l_out = δ99
        - sc_u_out = u99
        - sc_t_out = u99/d99
        '''
        
        verbose     = kwargs.get('verbose',True)
        interp_kind = kwargs.get('interp_kind','cubic') ## 'linear','cubic'
        
        if verbose: print('\n'+'ztmd.calc_d99_quantities()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        # ...
        
        if not any([(interp_kind=='linear'),(interp_kind=='cubic')]):
            raise ValueError(f"'interp_kind' should be one of: 'linear','cubic'")
        
        if verbose: even_print('1D interp kind',interp_kind)
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            ## copy dims into memory
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        d99 = np.copy( self['data_1Dx/d99'][()] )
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # === make a structured array
        
        names  = [ 'rho', 'u', 'v', 'w', 'T', 'p', 'vort_z', 'mu', 'nu', 'M' ]
        if ('data/psvel' in self):
            names += [ 'psvel' ]
        if ('data/utang' in self):
            names += [ 'utang' ]
        if ('data/unorm' in self):
            names += [ 'unorm' ]
        if ('data/umag' in self):
            names += [ 'umag' ]
        
        dtypes=[]
        for n in names:
            ds = self[f'data/{n}']
            dtypes.append( ds.dtype )
        
        #names_99 = [ n+'99' for n in names ]
        names_99 = [ n+'_99' if ('_' in n) else n+'99' for n in names ]
        
        data    = np.zeros(shape=(nx,ny), dtype={'names':names,    'formats':dtypes})
        data_99 = np.zeros(shape=(nx,),   dtype={'names':names_99, 'formats':dtypes})
        
        # === populate structured array
        
        for scalar in data.dtype.names:
            data[scalar][:,:] = np.copy( self[f'data/{scalar}'][()].T )
        
        # === interpolate @ δ99 for all vars
        
        if verbose: progress_bar = tqdm(total=nx*len(names), ncols=100, desc='δ99 quantities', leave=False, file=sys.stdout)
        for ni,scalar in enumerate(data.dtype.names):
            for i in range(nx):
                
                data_y_    = np.copy( data[scalar][i,:] )
                intrp_func = sp.interpolate.interp1d(y_, data_y_, kind=interp_kind, bounds_error=True)
                
                d99_    = d99[i]
                data_99_ = intrp_func(d99_)
                
                data_99[names_99[ni]][i] = data_99_
                
                if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # === write
        
        for scalar in data_99.dtype.names:
            if (f'data_1Dx/{scalar}' in self):
                del self[f'data_1Dx/{scalar}']
            data_ = np.copy( data_99[scalar][:] )
            dset = self.create_dataset(f'data_1Dx/{scalar}', data=data_, chunks=None)
            if verbose: even_print(f'data_1Dx/{scalar}',str(dset.shape))
        
        # === outer scales: length, velocity & time
        
        sc_l_out = np.copy( d99 )
        
        if self.rectilinear:
            sc_u_out = np.copy( data_99['u99'] )
            sc_t_out = np.copy( d99/data_99['u99'] )
        elif self.curvilinear:
            sc_u_out = np.copy( data_99['utang99'] )
            sc_t_out = np.copy( d99/data_99['utang99'] )
        else:
            raise ValueError
        
        np.testing.assert_allclose(sc_t_out, sc_l_out/sc_u_out, rtol=1e-14, atol=1e-14)
        
        u_tau = np.copy( self['data_1Dx/u_tau'][()] )
        sc_t_eddy = np.copy( d99/u_tau )
        
        if ('data_1Dx/sc_u_out' in self): del self['data_1Dx/sc_u_out']
        self.create_dataset('data_1Dx/sc_u_out', data=sc_u_out, chunks=None)
        if verbose: even_print('data_1Dx/sc_u_out', '%s'%str(sc_u_out.shape))
        
        if ('data_1Dx/sc_l_out' in self): del self['data_1Dx/sc_l_out']
        self.create_dataset('data_1Dx/sc_l_out', data=sc_l_out, chunks=None)
        if verbose: even_print('data_1Dx/sc_l_out', '%s'%str(sc_l_out.shape))
        
        if ('data_1Dx/sc_t_out' in self): del self['data_1Dx/sc_t_out']
        self.create_dataset('data_1Dx/sc_t_out', data=sc_t_out, chunks=None)
        if verbose: even_print('data_1Dx/sc_t_out', '%s'%str(sc_t_out.shape))
        
        if ('data_1Dx/sc_t_eddy' in self): del self['data_1Dx/sc_t_eddy']
        self.create_dataset('data_1Dx/sc_t_eddy', data=sc_t_eddy, chunks=None)
        if verbose: even_print('data_1Dx/sc_t_eddy', '%s'%str(sc_t_eddy.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_d99_quantities() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_bl_integral_quantities(self, **kwargs):
        '''
        δ*=δ1, θ=δ2, Reτ, Reθ, Reδ2, H12, etc.
        '''
        
        verbose     = kwargs.get('verbose',True)
        interp_kind = kwargs.get('interp_kind','cubic') ## 'linear','cubic'
        
        if verbose: print('\n'+'ztmd.calc_bl_integral_quantities()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        # ...
        
        if not any([(interp_kind=='linear'),(interp_kind=='cubic')]):
            raise ValueError(f"'interp_kind' should be one of: 'linear','cubic'")
        
        if verbose: even_print('1D interp kind',interp_kind)
        
        nx = self.nx
        ny = self.ny
        
        if self.rectilinear:
            
            ## copy dims into memory (1D)
            x = np.copy( self['dims/x'][()] )
            y = np.copy( self['dims/y'][()] )
        
        elif self.curvilinear:
            
            if ('dims/snorm' not in self):
                raise AssertionError('dims/snorm not present')
            if ('dims/stang' not in self):
                raise AssertionError('dims/stang not present')
            
            snorm = np.copy( self['dims/snorm'][()] ) ## 1D
            stang = np.copy( self['dims/stang'][()] ) ## 1D
            
            if ('data/utang' not in self):
                raise AssertionError('data/utang not present')
            
            ## copy dims into memory (2D)
            x = np.copy( self['dims/x'][()].T )
            y = np.copy( self['dims/y'][()].T )
            
            ## copy csys datasets into memory
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            
            if (x.shape != (self.nx,self.ny)):
                raise ValueError('x.shape != (self.nx,self.ny)')
            if (y.shape != (self.nx,self.ny)):
                raise ValueError('y.shape != (self.nx,self.ny)')
        
        else:
            raise ValueError
        
        ## the local 1D wall-normal coordinate
        if self.rectilinear:
            y_ = np.copy(y)
        elif self.curvilinear:
            y_ = np.copy(snorm)
        else:
            raise ValueError
        
        # === copy 1D datasets into memory
        
        u_tau    = np.copy( self['data_1Dx/u_tau'][()]    )
        rho_wall = np.copy( self['data_1Dx/rho_wall'][()] )
        nu_wall  = np.copy( self['data_1Dx/nu_wall'][()]  )
        mu_wall  = np.copy( self['data_1Dx/mu_wall'][()]  )
        T_wall   = np.copy( self['data_1Dx/T_wall'][()]   )
        
        j_edge     = np.copy( self['data_1Dx/j_edge'][()] )
        y_edge     = np.copy( self['data_1Dx/y_edge'][()] )
        d99        = np.copy( self['data_1Dx/d99'][()]    )
        #j99        = np.copy( self['data_1Dx/j99'][()]    )
        
        if self.rectilinear:
            u_edge = np.copy( self['data_1Dx/u_edge'][()] )
        elif self.curvilinear:
            utang_edge = np.copy( self['data_1Dx/utang_edge'][()] )
        else:
            raise ValueError
        
        rho_edge = np.copy( self['data_1Dx/rho_edge'][()] )
        mu_edge  = np.copy( self['data_1Dx/mu_edge'][()]  )
        nu_edge  = np.copy( self['data_1Dx/nu_edge'][()]  )
        
        sc_l_out = np.copy( self['data_1Dx/sc_l_out'][()] )
        sc_u_out = np.copy( self['data_1Dx/sc_u_out'][()] )
        sc_t_out = np.copy( self['data_1Dx/sc_t_out'][()] )
        
        # === copy 2D datasets into memory
        
        u = np.copy( self['data/u'][()].T )
        v = np.copy( self['data/v'][()].T )
        
        vort_z = np.copy( self['data/vort_z'][()].T )
        nu     = np.copy( self['data/nu'][()].T     )
        T      = np.copy( self['data/T'][()].T      )
        rho    = np.copy( self['data/rho'][()].T    )
        
        if self.curvilinear:
            utang = np.copy( self['data/utang'][()].T )
        
        # ===
        
        theta_cmp = np.zeros(shape=(nx,), dtype=np.float64) ## momentum thickness
        theta_inc = np.zeros(shape=(nx,), dtype=np.float64)
        dstar_cmp = np.zeros(shape=(nx,), dtype=np.float64) ## displacement thickness
        dstar_inc = np.zeros(shape=(nx,), dtype=np.float64)
        
        if self.rectilinear:
            u_vd = np.zeros(shape=(nx,ny), dtype=np.float64) ## Van Driest scaled u/u_tang
        elif self.curvilinear:
            utang_vd = np.zeros(shape=(nx,ny), dtype=np.float64) ## Van Driest scaled u/u_tang
        else:
            raise ValueError
        
        if verbose: progress_bar = tqdm(total=nx, ncols=100, desc='θ,δ*', leave=False, file=sys.stdout)
        for i in range(nx):
            
            y_edge_ = y_edge[i]
            
            if self.rectilinear:
                u_      = np.copy( u[i,:] )
                u_edge_ = u_edge[i]
            elif self.curvilinear:
                u_      = np.copy( utang[i,:] )
                u_edge_ = utang_edge[i]
            else:
                raise ValueError
            
            rho_ = np.copy( rho[i,:] )
            
            # ===
            
            integrand_theta_cmp = (u_*rho_)/(u_edge_*rho_edge[i])*(1-(u_/u_edge_))
            integrand_dstar_cmp = (1-((u_*rho_)/(u_edge_*rho_edge[i])))
            
            theta_cmp_     = sp.integrate.cumulative_trapezoid(y=integrand_theta_cmp, x=y_, initial=0.)
            theta_cmp_func = sp.interpolate.interp1d(y_, theta_cmp_, kind=interp_kind)
            theta_cmp[i]   = theta_cmp_func(y_edge_)
            
            dstar_cmp_     = sp.integrate.cumulative_trapezoid(y=integrand_dstar_cmp, x=y_, initial=0.)
            dstar_cmp_func = sp.interpolate.interp1d(y_, dstar_cmp_, kind=interp_kind)
            dstar_cmp[i]   = dstar_cmp_func(y_edge_)
            
            # ===
            
            integrand_theta_inc = (u_/u_edge_)*(1-(u_/u_edge_))
            integrand_dstar_inc = 1-(u_/u_edge_)
            
            theta_inc_     = sp.integrate.cumulative_trapezoid(y=integrand_theta_inc, x=y_, initial=0.)
            theta_inc_func = sp.interpolate.interp1d(y_, theta_inc_, kind=interp_kind)
            theta_inc[i]   = theta_inc_func(y_edge_)
            
            dstar_inc_     = sp.integrate.cumulative_trapezoid(y=integrand_dstar_inc, x=y_, initial=0.)
            dstar_inc_func = sp.interpolate.interp1d(y_, dstar_inc_, kind=interp_kind)
            dstar_inc[i]   = dstar_inc_func(y_edge_)
            
            if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # ===
        
        if ('data_1Dx/theta_inc' in self): del self['data_1Dx/theta_inc']
        self.create_dataset('data_1Dx/theta_inc', data=theta_inc, chunks=None)
        if verbose: even_print('data_1Dx/theta_inc', '%s'%str(theta_inc.shape))
        
        if ('data_1Dx/dstar_inc' in self): del self['data_1Dx/dstar_inc']
        self.create_dataset('data_1Dx/dstar_inc', data=dstar_inc, chunks=None)
        if verbose: even_print('data_1Dx/dstar_inc', '%s'%str(dstar_inc.shape))
        
        if ('data_1Dx/theta_cmp' in self): del self['data_1Dx/theta_cmp']
        self.create_dataset('data_1Dx/theta_cmp', data=theta_cmp, chunks=None)
        if verbose: even_print('data_1Dx/theta_cmp', '%s'%str(theta_cmp.shape))
        
        if ('data_1Dx/dstar_cmp' in self): del self['data_1Dx/dstar_cmp']
        self.create_dataset('data_1Dx/dstar_cmp', data=dstar_cmp, chunks=None)
        if verbose: even_print('data_1Dx/dstar_cmp', '%s'%str(dstar_cmp.shape))
        
        # ===
        
        #theta   = np.copy(theta_cmp)
        #dstar   = np.copy(dstar_cmp)
        H12     = dstar_cmp/theta_cmp
        H12_inc = dstar_inc/theta_inc
        
        if self.rectilinear:
            Re_tau       = np.copy( d99 * u_tau / nu_wall )
            Re_theta     = np.copy( theta_cmp * u_edge / nu_edge )
            Re_d99       = np.copy( d99 * u_edge / nu_edge )
            Re_d2        = np.copy( theta_cmp * u_edge * rho_edge / mu_wall )
            Re_dstar     = np.copy( dstar_cmp * u_edge / nu_edge )
            Re_dstar_inc = np.copy( dstar_inc * u_edge / nu_edge )
            Re_theta_inc = np.copy( theta_inc * u_edge / nu_edge )
            
            ## Reδ2 = Reθ_inc = (μe/μw)·Reθ ≈ (μ∞/μw)·Reθ
            np.testing.assert_allclose(Re_d2, (mu_edge/mu_wall)*Re_theta, rtol=1e-6)
            
            ## H12 = δ*/θ = Reδ*/Reθ
            np.testing.assert_allclose(Re_dstar/Re_theta, H12, rtol=1e-6)
            
            ## H12_inc = δ*i/θi = Reδ*i/Reθi
            np.testing.assert_allclose(Re_dstar_inc/Re_theta_inc, H12_inc, rtol=1e-6)
            
            uplus_edge   = np.copy( u_edge   / u_tau    )
            rhoplus_edge = np.copy( rho_edge / rho_wall )
            muplus_edge  = np.copy( mu_edge  / mu_wall  )
            
            ## Reδ*/Reτ = (δ*/δ99)·(ue/uτ)·(ρe/ρw)·(μw/μe) = (δ*/δ99)·(ue+·ρe+/μe+)
            np.testing.assert_allclose(
                Re_dstar / Re_tau,
                ( dstar_cmp / d99 ) * uplus_edge * rhoplus_edge / muplus_edge,
                rtol=1e-6,
                )
            
            ## Reθ/Reτ = (θ/δ99)·(ue/uτ)·(ρe/ρw)·(μw/μe) = (θ/δ99)·(ue+·ρe+/μe+)
            np.testing.assert_allclose(
                Re_theta / Re_tau,
                ( theta_cmp / d99 ) * uplus_edge * rhoplus_edge / muplus_edge,
                rtol=1e-6,
                )
        
        elif self.curvilinear:
            Re_tau   = np.copy( d99 * u_tau / nu_wall )
            Re_theta = np.copy( theta_cmp * utang_edge / nu_edge )
            Re_d2    = np.copy( theta_cmp * utang_edge * rho_edge / mu_wall )
            Re_d99   = np.copy( d99 * utang_edge / nu_edge )
        
        else:
            raise ValueError
        
        # ===
        
        if ('H12' in locals()):
            if ('data_1Dx/H12' in self): del self['data_1Dx/H12']
            self.create_dataset('data_1Dx/H12', data=H12, chunks=None)
            if verbose: even_print('data_1Dx/H12', '%s'%str(H12.shape))
        
        if ('H12_inc' in locals()):
            if ('data_1Dx/H12_inc' in self): del self['data_1Dx/H12_inc']
            self.create_dataset('data_1Dx/H12_inc', data=H12_inc, chunks=None)
            if verbose: even_print('data_1Dx/H12_inc', '%s'%str(H12_inc.shape))
        
        if ('Re_tau' in locals()):
            if ('data_1Dx/Re_tau' in self): del self['data_1Dx/Re_tau']
            self.create_dataset('data_1Dx/Re_tau', data=Re_tau, chunks=None)
            if verbose: even_print('data_1Dx/Re_tau', '%s'%str(Re_tau.shape))
        
        if ('Re_theta' in locals()):
            if ('data_1Dx/Re_theta' in self): del self['data_1Dx/Re_theta']
            self.create_dataset('data_1Dx/Re_theta', data=Re_theta, chunks=None)
            if verbose: even_print('data_1Dx/Re_theta', '%s'%str(Re_theta.shape))
        
        if ('Re_d99' in locals()):
            if ('data_1Dx/Re_d99' in self): del self['data_1Dx/Re_d99']
            self.create_dataset('data_1Dx/Re_d99', data=Re_d99, chunks=None)
            if verbose: even_print('data_1Dx/Re_d99', '%s'%str(Re_d99.shape))
        
        if ('Re_d2' in locals()):
            if ('data_1Dx/Re_d2' in self): del self['data_1Dx/Re_d2']
            self.create_dataset('data_1Dx/Re_d2', data=Re_d2, chunks=None)
            if verbose: even_print('data_1Dx/Re_d2', '%s'%str(Re_d2.shape))
        
        if ('Re_dstar' in locals()):
            if ('data_1Dx/Re_dstar' in self): del self['data_1Dx/Re_dstar']
            self.create_dataset('data_1Dx/Re_dstar', data=Re_dstar, chunks=None)
            if verbose: even_print('data_1Dx/Re_dstar', '%s'%str(Re_dstar.shape))
        
        if ('Re_dstar_inc' in locals()):
            if ('data_1Dx/Re_dstar_inc' in self): del self['data_1Dx/Re_dstar_inc']
            self.create_dataset('data_1Dx/Re_dstar_inc', data=Re_dstar_inc, chunks=None)
            if verbose: even_print('data_1Dx/Re_dstar_inc', '%s'%str(Re_dstar_inc.shape))
        
        if ('Re_theta_inc' in locals()):
            if ('data_1Dx/Re_theta_inc' in self): del self['data_1Dx/Re_theta_inc']
            self.create_dataset('data_1Dx/Re_theta_inc', data=Re_theta_inc, chunks=None)
            if verbose: even_print('data_1Dx/Re_theta_inc', '%s'%str(Re_theta_inc.shape))
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_bl_integral_quantities() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    # ===
    
    def calc_u_inc(self, method='rho', **kwargs):
        '''
        calculate the 'incompressible' streamwise velocity profile
        Van Driest (1951) : https://doi.org/10.2514/8.1895
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_u_inc()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not any([(method=='rho'),(method=='T')]):
            raise ValueError(f"method {str(method)} not valid. options are: 'rho','T'")
        
        if verbose: even_print('method',method)
        
        if self.rectilinear:
            y = np.copy( self['dims/y'][()] )
        elif self.curvilinear:
            y = np.copy( self['dims/snorm'][()] )
        else:
            raise ValueError
        
        rho_wall = np.copy( self['data_1Dx/rho_wall'][()] )
        T_wall   = np.copy( self['data_1Dx/T_wall'][()]   )
        
        T   = np.copy( self['data/T'][()].T   )
        rho = np.copy( self['data/rho'][()].T )
        
        if self.rectilinear:
            u = np.copy( self['data/u'][()].T   )
        elif self.curvilinear:
            u = np.copy( self['data/utang'][()].T )
        else:
            raise ValueError
        
        u_inc = np.zeros(shape=(self.nx,self.ny), dtype=np.float64)
        if verbose: progress_bar = tqdm(total=self.nx, ncols=100, desc='calc u_inc', leave=False, file=sys.stdout)
        for i in range(self.nx):
            
            if (method=='T'):
                integrand_u_inc = np.copy( np.sqrt(T_wall[i]/T[i,:]) )
            elif (method=='rho'):
                integrand_u_inc = np.copy( np.sqrt(rho[i,:]/rho_wall[i]) )
            else:
                raise ValueError
            
            u_inc[i,:] = sp.integrate.cumulative_trapezoid(integrand_u_inc, u[i,:], initial=0)
            
            if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        if self.rectilinear:
            if ('data/u_inc' in self): del self['data/u_inc']
            self.create_dataset('data/u_inc', data=u_inc.T, chunks=None)
            if verbose: even_print('data/u_inc', '%s'%str(u_inc.shape))
        elif self.curvilinear:
            if ('data/utang_inc' in self): del self['data/utang_inc']
            self.create_dataset('data/utang_inc', data=u_inc.T, chunks=None)
            if verbose: even_print('data/utang_inc', '%s'%str(u_inc.shape))
        else:
            raise ValueError
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_u_inc() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_bl_edge_u_inc(self, **kwargs):
        '''
        determine the 'incompressible' BL edge location (edge of VDI u profile)
        '''
        
        ## having 2x different edges doesnt really make sense
        raise ValueError('ztmd.calc_bl_edge_u_inc() is not longer recommended')
        
        verbose      = kwargs.get('verbose',True)
        epsilon      = kwargs.get('epsilon',5e-5)
        acc          = kwargs.get('acc',6)
        edge_stencil = kwargs.get('edge_stencil','full')
        interp_kind  = kwargs.get('interp_kind','cubic')
        ongrid       = kwargs.get('ongrid',True) ## snap to grid point
        
        if verbose: print('\n'+'ztmd.calc_bl_edge_u_inc()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print('epsilon','%0.1e'%(epsilon,))
        if verbose: even_print('acc',f'{acc:d}')
        if verbose: even_print('edge_stencil',edge_stencil)
        if verbose: even_print('1D interp kind',interp_kind)
        
        if self.rectilinear:
            y     = np.copy( self['dims/y'][()] )
            u_inc = np.copy( self['data/u_inc'][()].T )
        elif self.curvilinear:
            y     = np.copy( self['dims/snorm'][()] )
            u_inc = np.copy( self['data/utang_inc'][()].T )
        else:
            raise ValueError
        
        sc_u_in = np.copy( self['data_1Dx/sc_u_in'][()] ) ## uτ
        sc_l_in = np.copy( self['data_1Dx/sc_l_in'][()] ) ## δν = νw/uτ
        
        #u_inc_max_x = np.amax(u_inc, axis=1)
        ddy_u = gradient(
                    u_inc,
                    y,
                    axis=1,
                    d=1,
                    acc=acc,
                    edge_stencil=edge_stencil,
                    )
        
        y_edge_inc = np.zeros(shape=(self.nx,), dtype=np.float64)
        u_inc_edge = np.zeros(shape=(self.nx,), dtype=np.float64)
        
        if verbose: progress_bar = tqdm(total=self.nx, ncols=100, desc='y_edge (u_inc)', leave=False, file=sys.stdout)
        for i in range(self.nx):
            
            do_debug_plot = False
            #if (i==10000):
            #    do_debug_plot = True
            
            ddy_u_      = np.copy( ddy_u[i,:] ) ## du/dy
            ddy_u_plus_ = np.copy( ddy_u_ / ( sc_u_in[i] / sc_l_in[i] ) ) ## du+/dy+ = (du/dy)/(uτ/δν) = (du/dy)/(uτ^2/νw)
            
            y_edge_inc_ = calc_profile_edge_1d(
                y=y,
                ddy_u=ddy_u_plus_,
                ongrid=ongrid,
                epsilon=epsilon,
                acc=acc,
                edge_stencil=edge_stencil,
                interp_kind=interp_kind,
                do_debug_plot=do_debug_plot,
                )
            
            y_edge_inc[i] = y_edge_inc_
            j_edge_       = np.abs( y - y_edge_inc_ ).argmin()
            
            u_inc_        = np.copy( u_inc[i,:] )
            
            if ongrid:
                u_inc_edge[i] = u_inc_[j_edge_]
            else:
                intrp_func    = sp.interpolate.interp1d(y, u_inc_, kind=interp_kind, bounds_error=True)
                u_inc_edge[i] = intrp_func(y_edge_inc_)
            
            progress_bar.update()
        progress_bar.close()
        
        #if ongrid:
        #    np.testing.assert_allclose(y_edge_inc,y_edge_inc_g,rtol=1e-5)
        
        if ('data_1Dx/y_edge_inc' in self): del self['data_1Dx/y_edge_inc']
        dset = self.create_dataset('data_1Dx/y_edge_inc', data=y_edge_inc, chunks=None)
        if verbose: even_print('data_1Dx/y_edge_inc','%s'%str(y_edge_inc.shape))
        
        if ('data_1Dx/u_inc_edge' in self): del self['data_1Dx/u_inc_edge']
        dset = self.create_dataset('data_1Dx/u_inc_edge', data=u_inc_edge, chunks=None)
        if verbose: even_print('data_1Dx/u_inc_edge','%s'%str(u_inc_edge.shape))
        
        # ===
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_bl_edge_u_inc() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_d99_inc(self, **kwargs):
        '''
        determine δ99 & δ95 for van Driest transformed u
        '''
        
        verbose     = kwargs.get('verbose',True)
        interp_kind = kwargs.get('interp_kind','cubic') ## 'linear','cubic'
        #rtol        = kwargs.get('rtol',1e-3)
        
        if verbose: print('\n'+'ztmd.calc_d99_inc()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        # ...
        
        if not any([(interp_kind=='linear'),(interp_kind=='cubic')]):
            raise ValueError(f"'interp_kind' should be one of: 'linear','cubic'")
        
        #if verbose: even_print('rtol','%0.1e'%(rtol,))
        if verbose: even_print('1D interp kind',interp_kind)
        
        #y_edge = np.copy( self['data_1Dx/y_edge_inc'][()] ) ## !! importing INCOMPRESSIBLE y_edge_inc as y_edge
        y_edge = np.copy( self['data_1Dx/y_edge'][()] ) ## !! importing REGULAR y_edge
        
        #print( np.mean( y_edge[2406:19124] - y_edge_inc[2406:19124] ) )
        
        if self.rectilinear:
            y = np.copy( self['dims/y'][()] )
            u = np.copy( self['data/u_inc'][()].T ) ## !! importing van Driest incompressible u_inc as 'u'
        elif self.curvilinear:
            y = np.copy( self['dims/snorm'][()] )
            u = np.copy( self['data/utang_inc'][()].T )
        else:
            raise ValueError
        
        d99_inc  = np.zeros(shape=(self.nx,), dtype=np.float64)
        d95_inc  = np.zeros(shape=(self.nx,), dtype=np.float64)
        u_inc_99 = np.zeros(shape=(self.nx,), dtype=np.float64)
        u_inc_95 = np.zeros(shape=(self.nx,), dtype=np.float64)
        
        if verbose: progress_bar = tqdm(total=self.nx, ncols=100, desc='δ', leave=False, file=sys.stdout)
        for i in range(self.nx):
            
            y_edge_ = y_edge[i]
            u_      = np.copy( u[i,:] )
            
            d99_inc_    = calc_d99_1d(y=y, u=u_, y_edge=y_edge_, interp_kind=interp_kind, d95=False)
            d99_inc[i]  = d99_inc_
            
            d95_inc_    = calc_d99_1d(y=y, u=u_, y_edge=y_edge_, interp_kind=interp_kind, d95=True)
            d95_inc[i]  = d95_inc_
            
            intrp_func  = sp.interpolate.interp1d(y, u_, kind=interp_kind, bounds_error=True)
            u_inc_99[i] = intrp_func(d99_inc_)
            u_inc_95[i] = intrp_func(d95_inc_)
            
            progress_bar.update()
        progress_bar.close()
        
        if ('data_1Dx/d99_inc' in self): del self['data_1Dx/d99_inc']
        dset = self.create_dataset('data_1Dx/d99_inc', data=d99_inc, chunks=None)
        if verbose: even_print('data_1Dx/d99_inc','%s'%str(d99_inc.shape))
        
        if ('data_1Dx/d95_inc' in self): del self['data_1Dx/d95_inc']
        dset = self.create_dataset('data_1Dx/d95_inc', data=d95_inc, chunks=None)
        if verbose: even_print('data_1Dx/d95_inc','%s'%str(d95_inc.shape))
        
        if ('data_1Dx/u_inc_99' in self): del self['data_1Dx/u_inc_99']
        dset = self.create_dataset('data_1Dx/u_inc_99', data=u_inc_99, chunks=None)
        if verbose: even_print('data_1Dx/u_inc_99','%s'%str(u_inc_99.shape))
        
        if ('data_1Dx/u_inc_95' in self): del self['data_1Dx/u_inc_95']
        dset = self.create_dataset('data_1Dx/u_inc_95', data=u_inc_95, chunks=None)
        if verbose: even_print('data_1Dx/u_inc_95','%s'%str(u_inc_95.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_d99_inc() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_wake_parameter(self, **kwargs):
        '''
        calculate the Coles wake parameter "Π"
        u+ = (1/κ)·ln(y+) + B + (2Π/κ)·w(y/δ)
        - at y=δ, w(y/δ)==1
        --> Π = ( u(δ) - (1/κ)·ln(δ+) - B ) · k / 2
        -----
        Coles (1956)            : https://doi.org/10.1017/S0022112056000135
        Pirozzoli (2004)        : https://doi.org/10.1063/1.1637604
        Smits & Dussauge (2006) : https://doi.org/10.1007/b137383
        Chauhan et al. (2009)   : https://doi.org/10.1088/0169-5983/41/2/021404
        Nagib et al. (2007)     : https://doi.org/10.1098/rsta.2006.1948
        '''
        
        verbose = kwargs.get('verbose',True)
        k       = kwargs.get('k',0.41) ## Von Kármán constant (κ)
        B       = kwargs.get('B',5.2) ## constant in log law eqn. : u+ = (1/κ)·ln(y+) + B
        
        ## see Nagib et al. (2007)
        #k       = kwargs.get('k',0.384) 
        #B       = kwargs.get('B',4.173) ## called 'B' in Nagib et al. (2007) 
        
        if verbose: print('\n'+'ztmd.calc_wake_parameter()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        # ...
        
        if verbose: even_print('κ',f'{k:0.5f}')
        if verbose: even_print('B',f'{B:0.5f}')
        
        if self.curvilinear:
            raise NotImplementedError('ztmd.calc_wake_parameter() has not been implemented for curved cases')
        
        y          = np.copy( self['dims/y'][()]           )
        u_tau      = np.copy( self['data_1Dx/u_tau'][()]   )
        sc_l_in    = np.copy( self['data_1Dx/sc_l_in'][()] ) ## δν = νw/uτ
        
        d99_inc    = np.copy( self['data_1Dx/d99_inc'][()]  ) ## 'incompressible' BL δ99 i.e. δ99_inc
        u_inc_99   = np.copy( self['data_1Dx/u_inc_99'][()] )
        
        #y_edge_inc = np.copy( self['data_1Dx/y_edge_inc'][()] ) ## 'incompressible' BL edge
        #u_inc_edge = np.copy( self['data_1Dx/u_inc_edge'][()] )
        
        # u_inc = np.copy( self['data/u_inc'][()].T ) ## 'incompressible' transformed u profile
        
        wake_parameter = np.zeros(shape=(self.nx,), dtype=np.float64) ## Π = (κ/2)·Δ(u/uτ)
        wake_strength  = np.zeros(shape=(self.nx,), dtype=np.float64) ## Δ(u/uτ) @ δ
        
        if verbose: progress_bar = tqdm(total=self.nx, ncols=100, desc='wake parameter', leave=False, file=sys.stdout)
        for i in range(self.nx):
            
            #up_ = u_inc_edge[i] / u_tau[i]  ## ue_inc+
            #yp_ = y_edge_inc[i] / sc_l_in[i]  ## ye_inc+
            
            up_ = u_inc_99[i] / u_tau[i] ## u99_inc+
            yp_ = d99_inc[i] / sc_l_in[i] ## δ99_inc+
            
            up_loglaw_99_   = (1/k)*np.log(yp_) + B
            wake_strength_  = up_ - up_loglaw_99_
            wake_parameter_ = wake_strength_ * (k/2)
            
            wake_strength[i]  = wake_strength_
            wake_parameter[i] = wake_parameter_
            
            # ===
            
            #u_edge_plus_  = u_inc_99[i] / u_tau[i]
            #delta_plus_   = d99_inc[i]  / sc_l_in[i]
            #wake_parameter2_ = (k/2)*( u_edge_plus_ - (1/k)*np.log(delta_plus_) - B )
            #wake_strength2_  = wake_parameter2_ * (2/k)
            #np.testing.assert_allclose(wake_parameter2_  , wake_parameter_ , rtol=1e-12, atol=1e-12)
            #np.testing.assert_allclose(wake_strength2_   , wake_strength_  , rtol=1e-12, atol=1e-12)
            
            progress_bar.update()
        progress_bar.close()
        
        if ('data_1Dx/wake_strength' in self): del self['data_1Dx/wake_strength']
        dset = self.create_dataset('data_1Dx/wake_strength', data=wake_strength, chunks=None)
        if verbose: even_print('data_1Dx/wake_strength','%s'%str(wake_strength.shape))
        
        if ('data_1Dx/wake_parameter' in self): del self['data_1Dx/wake_parameter']
        dset = self.create_dataset('data_1Dx/wake_parameter', data=wake_parameter, chunks=None)
        if verbose: even_print('data_1Dx/wake_parameter','%s'%str(wake_parameter.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_wake_parameter() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
    
    def calc_VDII(self, **kwargs):
        '''
        perform 'incompressible' cf,Reθ transform according to Van Driest (1956), yields
        Fc (compressibility factor)
        Reδ2 = (μe/μw)·Reθ
        cfi = Fc·cf
        often referred to as the 'Van Driest II' compressibility transform
        -----
        Van Driest 1956 'The Problem of Aerodynamic Heating'
        https://web.stanford.edu/~jurzay/ME356_files/vandriest_aeroheating.pdf
        White 2006 'Viscous Fluid Flow' 7-7 and 7-8 (p.547-556)
        '''
        
        verbose   = kwargs.get('verbose',True)
        adiabatic = kwargs.get('adiabatic',False)
        
        if verbose: print('\n'+'ztmd.calc_VDII()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if verbose: even_print('adiabatic',str(adiabatic))
        
        y        = np.copy( self['dims/y'][()]            )
        T_wall   = np.copy( self['data_1Dx/T_wall'][()]   )
        T_edge   = np.copy( self['data_1Dx/T_edge'][()]   )
        M_edge   = np.copy( self['data_1Dx/M_edge'][()]   )
        rho_edge = np.copy( self['data_1Dx/rho_edge'][()] )
        u_edge   = np.copy( self['data_1Dx/u_edge'][()]   )
        
        mu_wall  = np.copy( self['data_1Dx/mu_wall'][()]  )
        mu_edge  = np.copy( self['data_1Dx/mu_edge'][()]  )
        Re_theta = np.copy( self['data_1Dx/Re_theta'][()] )
        cf       = np.copy( self['data_1Dx/cf'][()]       )
        tau_wall = np.copy( self['data_1Dx/tau_wall'][()] )
        
        if adiabatic:
            Fc = np.copy( ((T_wall / T_edge)-1.) / (np.arcsin( (1.-(T_edge/T_wall))**0.5 ))**2 )
        else: 
            raise NotImplementedError
        
        # ===
        
        ## assert that cf is calculated with EDGE values
        cf_edge = np.copy( tau_wall / (0.5 *      rho_edge *     u_edge**2 ) )
        cf_inf  = np.copy( tau_wall / (0.5 * self.rho_inf  * self.U_inf**2 ) )
        np.testing.assert_allclose(cf, cf_edge, rtol=1e-6)
        cf_edge = None ; del cf_edge
        cf_inf = None  ; del cf_inf
        
        cf_inc = np.copy( Fc * cf )
        
        # ===
        
        if ('data_1Dx/cf_inc' in self): del self['data_1Dx/cf_inc']
        dset = self.create_dataset('data_1Dx/cf_inc', data=cf_inc, chunks=None)
        if verbose: even_print('data_1Dx/cf_inc','%s'%str(cf_inc.shape))
        
        if ('data_1Dx/Fc' in self): del self['data_1Dx/Fc']
        dset = self.create_dataset('data_1Dx/Fc', data=Fc, chunks=None)
        if verbose: even_print('data_1Dx/Fc','%s'%str(Fc.shape))
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_VDII() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def calc_peak_tauI(self, **kwargs):
        '''
        calculate peak τ′xx, τ′xy, τ′yy
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if verbose: print('\n'+'ztmd.calc_peak_tauI()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## check
        if self.rectilinear:
            pass
        elif self.curvilinear:
            raise NotImplementedError('ztmd.calc_peak_tauI() has not been implemented for curved cases')
        else:
            raise ValueError
        
        ## check
        if not ('data_1Dx/sc_l_in' in self):
            raise ValueError('data_1Dx/sc_l_in not found')
        if not ('data/r_uII_uII' in self):
            raise ValueError('data/r_uII_uII not found')
        if not ('data/r_uII_vII' in self):
            raise ValueError('data/r_uII_vII not found')
        if not ('data/r_vII_vII' in self):
            raise ValueError('data/r_vII_vII not found')
        
        r_uII_uII = np.copy( self['data/r_uII_uII'][()].T )
        r_uII_vII = np.copy( self['data/r_uII_vII'][()].T )
        r_vII_vII = np.copy( self['data/r_vII_vII'][()].T )
        
        y        = np.copy( self['dims/y'][()]            )
        tau_wall = np.copy( self['data_1Dx/tau_wall'][()] )
        sc_l_in  = np.copy( self['data_1Dx/sc_l_in'][()]  )
        sc_l_out = np.copy( self['data_1Dx/sc_l_out'][()] )
        nu_wall  = np.copy( self['data_1Dx/nu_wall'][()]  )
        u_tau    = np.copy( self['data_1Dx/u_tau'][()]    )
        d99      = np.copy( self['data_1Dx/d99'][()]      )
        
        np.testing.assert_allclose(sc_l_in  , nu_wall/u_tau , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(sc_l_out , d99           , rtol=1e-14, atol=1e-14)
        
        r_uII_uII_plus = np.copy( r_uII_uII / tau_wall[:,np.newaxis] )
        r_uII_vII_plus = np.copy( r_uII_vII / tau_wall[:,np.newaxis] )
        r_vII_vII_plus = np.copy( r_vII_vII / tau_wall[:,np.newaxis] )
        
        tau_xx_I_peak   = np.zeros((self.nx,), dtype=np.float64)
        tau_xx_I_peak_y = np.zeros((self.nx,), dtype=np.float64)
        
        tau_xy_I_peak   = np.zeros((self.nx,), dtype=np.float64)
        tau_xy_I_peak_y = np.zeros((self.nx,), dtype=np.float64)
        
        tau_yy_I_peak   = np.zeros((self.nx,), dtype=np.float64)
        tau_yy_I_peak_y = np.zeros((self.nx,), dtype=np.float64)
        
        # ===
        
        if verbose: progress_bar = tqdm(total=self.nx, ncols=100, desc='peak τ′', leave=False, file=sys.stdout)
        #for i in [5000,10000,15000]:
        for i in range(self.nx):
            
            y_plus_  = np.copy( y / sc_l_in[i]  )
            yovd_    = np.copy( y / sc_l_out[i] )
            
            r_uII_uII_plus_ = np.copy( r_uII_uII_plus[i,:] )
            r_uII_vII_plus_ = np.copy( r_uII_vII_plus[i,:] )
            r_vII_vII_plus_ = np.copy( r_vII_vII_plus[i,:] )
            
            # === τ′xx
            
            i_naive = np.argmax( r_uII_uII_plus_ )
            tau_xx_I_peak_naive_ = r_uII_uII_plus_[i_naive]
            
            func_tauIxx = sp.interpolate.CubicSpline( y_plus_ , r_uII_uII_plus_ , bc_type='natural', extrapolate=False )
            
            def __opt_find_peak(y_plus_pk,func):
                root = func(y_plus_pk,1)
                return root
            
            #print(i_naive)
            #print(y_plus_[i_naive])
            #time.sleep(1)
            
            bounds_ = ( max(y_plus_[i_naive]*0.9,y_plus_.min()) , min(y_plus_[i_naive]*1.1,y_plus_.max()) )
            
            sol = sp.optimize.least_squares(fun=__opt_find_peak,
                                            args=(func_tauIxx,),
                                            x0=y_plus_[i_naive],
                                            xtol=1e-15,
                                            ftol=1e-15,
                                            gtol=1e-15,
                                            method='dogbox',
                                            bounds=bounds_,
                                            )
            if not sol.success:
                raise ValueError
            
            y_plus_pk_ = float(sol.x[0])
            tau_xx_I_peak[i]   = func_tauIxx(y_plus_pk_) * tau_wall[i] ## re-dimensionalizing
            tau_xx_I_peak_y[i] = y_plus_pk_              * sc_l_in[i]  ## re-dimensionalizing
            
            # ===
            
            ## debug plot for τ′xx
            #if (i==5000) or (i==10000) or (i==15000):
            if 0:
                
                plt.close('all')
                fig1 = plt.figure(figsize=(3,2), dpi=400)
                ax1 = plt.gca()
                
                ax1.tick_params(axis='x', which='both', direction='in')
                ax1.tick_params(axis='y', which='both', direction='in')
                #ax1.xaxis.set_ticks_position('both')
                #ax1.yaxis.set_ticks_position('both')
                ax1.set_xscale('log',base=10)
                #ax1.set_yscale('log',base=10)
                
                #ax1.set_xlim(100,3000)
                #ax1.xaxis.set_major_locator(mpl.ticker.LogLocator(subs=(1,)))
                #ax1.xaxis.set_minor_locator(mpl.ticker.LogLocator(subs=np.linspace(1,9,9)))
                #ax1.xaxis.set_minor_formatter(mpl.ticker.NullFormatter())
                
                ax1.plot(
                y_plus_,
                r_uII_uII_plus_,
                c='k',
                zorder=19,
                lw=0.8,
                marker='o',
                ms=1.5,
                ls='none',
                )
                
                y_plus_dummy_          = np.logspace(np.log10(1),np.log10(1000),1000)
                r_uII_uII_plus_spline_ = func_tauIxx(y_plus_dummy_)
                
                ax1.plot(
                y_plus_dummy_,
                r_uII_uII_plus_spline_,
                c='blue',
                zorder=19,
                lw=0.8,
                #marker='o',
                #ms=1.5,
                #ls='none',
                )
                
                ax1.axvline(x=y_plus_pk_       , linestyle='solid', c='gray', zorder=1, lw=0.5)
                ax1.axhline(y=func_tauIxx(y_plus_pk_) , linestyle='solid', c='gray', zorder=1, lw=0.5)
                
                plt.show()
            
            # === τ′xy
            
            i_naive = np.argmin( r_uII_vII_plus_ ) ## ACHTUNG argmin(), NOT argmax() !!
            tau_xy_I_peak_naive_ = r_uII_vII_plus_[i_naive]
            
            func_tauIxy = sp.interpolate.CubicSpline( y_plus_ , r_uII_vII_plus_ , bc_type='natural', extrapolate=False )
            
            def __opt_find_peak(y_plus_pk,func):
                root = func(y_plus_pk,1)
                return root
            
            bounds_ = ( max(y_plus_[i_naive]*0.9,y_plus_.min()) , min(y_plus_[i_naive]*1.1,y_plus_.max()) )
            
            sol = sp.optimize.least_squares(fun=__opt_find_peak,
                                            args=(func_tauIxy,),
                                            x0=y_plus_[i_naive],
                                            xtol=1e-15,
                                            ftol=1e-15,
                                            gtol=1e-15,
                                            method='dogbox',
                                            bounds=bounds_,
                                            )
            if not sol.success:
                raise ValueError
            
            y_plus_pk_ = float(sol.x[0])
            tau_xy_I_peak[i]   = func_tauIxy(y_plus_pk_) * tau_wall[i] ## re-dimensionalizing
            tau_xy_I_peak_y[i] = y_plus_pk_              * sc_l_in[i]  ## re-dimensionalizing
            
            # ===
            
            ## debug plot for τ′xy
            #if (i==5000) or (i==10000) or (i==15000):
            if 0:
                
                plt.close('all')
                fig1 = plt.figure(figsize=(3,2), dpi=400)
                ax1 = plt.gca()
                
                ax1.tick_params(axis='x', which='both', direction='in')
                ax1.tick_params(axis='y', which='both', direction='in')
                #ax1.xaxis.set_ticks_position('both')
                #ax1.yaxis.set_ticks_position('both')
                ax1.set_xscale('log',base=10)
                #ax1.set_yscale('log',base=10)
                
                #ax1.set_xlim(100,3000)
                #ax1.xaxis.set_major_locator(mpl.ticker.LogLocator(subs=(1,)))
                #ax1.xaxis.set_minor_locator(mpl.ticker.LogLocator(subs=np.linspace(1,9,9)))
                #ax1.xaxis.set_minor_formatter(mpl.ticker.NullFormatter())
                
                ax1.plot(
                y_plus_,
                r_uII_vII_plus_,
                c='k',
                zorder=19,
                lw=0.8,
                marker='o',
                ms=1.5,
                ls='none',
                )
                
                y_plus_dummy_          = np.logspace(np.log10(1),np.log10(1000),1000)
                r_uII_vII_plus_spline_ = func_tauIxy(y_plus_dummy_)
                
                ax1.plot(
                y_plus_dummy_,
                r_uII_vII_plus_spline_,
                c='blue',
                zorder=19,
                lw=0.8,
                #marker='o',
                #ms=1.5,
                #ls='none',
                )
                
                ax1.axvline(x=y_plus_pk_       , linestyle='solid', c='gray', zorder=1, lw=0.5)
                ax1.axhline(y=func_tauIxy(y_plus_pk_) , linestyle='solid', c='gray', zorder=1, lw=0.5)
                
                plt.show()
            
            # === τ′yy
            
            i_naive = np.argmax( r_vII_vII_plus_ )
            tau_yy_I_peak_naive_ = r_vII_vII_plus_[i_naive]
            
            func_tauIyy = sp.interpolate.CubicSpline( y_plus_ , r_vII_vII_plus_ , bc_type='natural', extrapolate=False )
            
            def __opt_find_peak(y_plus_pk,func):
                root = func(y_plus_pk,1)
                return root
            
            bounds_ = ( max(y_plus_[i_naive]*0.9,y_plus_.min()) , min(y_plus_[i_naive]*1.1,y_plus_.max()) )
            
            sol = sp.optimize.least_squares(fun=__opt_find_peak,
                                            args=(func_tauIyy,),
                                            x0=y_plus_[i_naive],
                                            xtol=1e-15,
                                            ftol=1e-15,
                                            gtol=1e-15,
                                            method='dogbox',
                                            bounds=bounds_,
                                            )
            if not sol.success:
                raise ValueError
            
            y_plus_pk_ = float(sol.x[0])
            tau_yy_I_peak[i]   = func_tauIyy(y_plus_pk_) * tau_wall[i] ## re-dimensionalizing
            tau_yy_I_peak_y[i] = y_plus_pk_              * sc_l_in[i]  ## re-dimensionalizing
            
            # ===
            
            ## debug plot for τ′yy
            #if (i==5000) or (i==10000) or (i==15000):
            if 0:
                
                plt.close('all')
                fig1 = plt.figure(figsize=(3,2), dpi=400)
                ax1 = plt.gca()
                
                ax1.tick_params(axis='x', which='both', direction='in')
                ax1.tick_params(axis='y', which='both', direction='in')
                #ax1.xaxis.set_ticks_position('both')
                #ax1.yaxis.set_ticks_position('both')
                ax1.set_xscale('log',base=10)
                #ax1.set_yscale('log',base=10)
                
                #ax1.set_xlim(100,3000)
                #ax1.xaxis.set_major_locator(mpl.ticker.LogLocator(subs=(1,)))
                #ax1.xaxis.set_minor_locator(mpl.ticker.LogLocator(subs=np.linspace(1,9,9)))
                #ax1.xaxis.set_minor_formatter(mpl.ticker.NullFormatter())
                
                ax1.plot(
                y_plus_,
                r_vII_vII_plus_,
                c='k',
                zorder=19,
                lw=0.8,
                marker='o',
                ms=1.5,
                ls='none',
                )
                
                y_plus_dummy_          = np.logspace(np.log10(1),np.log10(1000),1000)
                r_vII_vII_plus_spline_ = func_tauIyy(y_plus_dummy_)
                
                ax1.plot(
                y_plus_dummy_,
                r_vII_vII_plus_spline_,
                c='blue',
                zorder=19,
                lw=0.8,
                #marker='o',
                #ms=1.5,
                #ls='none',
                )
                
                ax1.axvline(x=y_plus_pk_              , linestyle='solid', c='gray', zorder=1, lw=0.5)
                ax1.axhline(y=func_tauIyy(y_plus_pk_) , linestyle='solid', c='gray', zorder=1, lw=0.5)
                
                plt.show()
            
            # ===
            
            progress_bar.update()
        progress_bar.close()
        
        
        if ('data_1Dx/tau_xx_I_peak' in self): del self['data_1Dx/tau_xx_I_peak']
        dset = self.create_dataset('data_1Dx/tau_xx_I_peak', data=tau_xx_I_peak, chunks=None)
        if verbose: even_print('data_1Dx/tau_xx_I_peak','%s'%str(tau_xx_I_peak.shape))
        
        if ('data_1Dx/tau_xy_I_peak' in self): del self['data_1Dx/tau_xy_I_peak']
        dset = self.create_dataset('data_1Dx/tau_xy_I_peak', data=tau_xy_I_peak, chunks=None)
        if verbose: even_print('data_1Dx/tau_xy_I_peak','%s'%str(tau_xy_I_peak.shape))
        
        if ('data_1Dx/tau_yy_I_peak' in self): del self['data_1Dx/tau_yy_I_peak']
        dset = self.create_dataset('data_1Dx/tau_yy_I_peak', data=tau_yy_I_peak, chunks=None)
        if verbose: even_print('data_1Dx/tau_yy_I_peak','%s'%str(tau_yy_I_peak.shape))
        
        ## old
        if ('data_1Dx/tau_xx_I_peak_y_plus' in self):
            del self['data_1Dx/tau_xx_I_peak_y_plus']
        if ('data_1Dx/tau_xy_I_peak_y_plus' in self):
            del self['data_1Dx/tau_xy_I_peak_y_plus']
        if ('data_1Dx/tau_yy_I_peak_y_plus' in self):
            del self['data_1Dx/tau_yy_I_peak_y_plus']
        
        
        if ('data_1Dx/tau_xx_I_peak_y' in self): del self['data_1Dx/tau_xx_I_peak_y']
        dset = self.create_dataset('data_1Dx/tau_xx_I_peak_y', data=tau_xx_I_peak_y, chunks=None)
        if verbose: even_print('data_1Dx/tau_xx_I_peak_y','%s'%str(tau_xx_I_peak_y.shape))
        
        if ('data_1Dx/tau_xy_I_peak_y' in self): del self['data_1Dx/tau_xy_I_peak_y']
        dset = self.create_dataset('data_1Dx/tau_xy_I_peak_y', data=tau_xy_I_peak_y, chunks=None)
        if verbose: even_print('data_1Dx/tau_xy_I_peak_y','%s'%str(tau_xy_I_peak_y.shape))
        
        if ('data_1Dx/tau_yy_I_peak_y' in self): del self['data_1Dx/tau_yy_I_peak_y']
        dset = self.create_dataset('data_1Dx/tau_yy_I_peak_y', data=tau_yy_I_peak_y, chunks=None)
        if verbose: even_print('data_1Dx/tau_yy_I_peak_y','%s'%str(tau_yy_I_peak_y.shape))
        
        
        self.get_header(verbose=False)
        if verbose: print(72*'-')
        if verbose: print('total time : ztmd.calc_peak_tauI() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # === legacy / deprecated : functions for curved TBL cases where axis=1 mesh is not identical over axis=0
    
    def get_wall_norm_mesh(self, **kwargs):
        '''
        get a new 'grid' which is extruded in the normal direction from the wall
        - this grid is good for post-processing in the wall-normal direction ONLY
        - orthogonal to the wall-normal direction may have jumps or be folded
        '''
        
        verbose = kwargs.get('verbose',True)
        
        ## the wall-normal unit tangent and normal vectors
        if ('csys/vtang' in self) and ('csys/vnorm' in self):
            vtang = np.copy( self['csys/vtang'][()] )
            vnorm = np.copy( self['csys/vnorm'][()] )
            #wall_trafo_mat = np.stack((vtang,vnorm), axis=-1)
        else:
            raise AssertionError('no vnorm/vtang unit projection vector!')
        
        if ('data/wall_distance' in self):
            wall_dist = np.copy( self['data/wall_distance'][()].T )
        else:
            raise AssertionError('dset not present: data/wall_distance')
        
        xy2d_n1D          = np.zeros((self.nx,self.ny,2) , dtype=np.float64)
        wall_distance_n1D = np.zeros((self.nx,self.ny)   , dtype=np.float64)
        
        for i in range(self.nx):
            
            p0_ = np.array([self.x[i,0],self.y[i,0]], dtype=np.float64) ## wall point coordinate
            
            vnorm_ = np.copy( vnorm[i,0,:] ) ## unit normal vec @ wall at this x
            #vtang_ = np.copy( vtang[i,0,:] ) ## unit tangent vec @ wall at this x
            
            #x_  = np.copy( self.x[i,:] )
            #y_  = np.copy( self.y[i,:] )
            #dx_ = np.diff(x_,n=1)
            #dy_ = np.diff(y_,n=1)
            #ds_ = np.sqrt(dx_**2+dy_**2)
            #s_  = np.cumsum(np.concatenate(([0.,],ds_))) ## path length normal to wall @ this x
            
            s_ = np.copy( wall_dist[i,:] )
            
            wall_distance_n1D[i,:] = s_
            
            xy = p0_ + np.einsum( 'i,j->ij', s_, vnorm_ )
            
            xy2d_n1D[i,:,:] = xy
        
        if ('dims_2Dw/x' in self): del self['dims_2Dw/x']
        self.create_dataset('dims_2Dw/x', data=np.squeeze(xy2d_n1D[:,:,0]).T, chunks=None)
        
        if ('dims_2Dw/y' in self): del self['dims_2Dw/y']
        self.create_dataset('dims_2Dw/y', data=np.squeeze(xy2d_n1D[:,:,1]).T, chunks=None)
        
        if ('data_2Dw/wall_distance' in self): del self['data_2Dw/wall_distance']
        self.create_dataset('data_2Dw/wall_distance', data=wall_distance_n1D.T, chunks=None)
        
        self.attrs['requires_wall_norm_interp'] = True
        self.get_header(verbose=False)
        
        # ===
        
        if False: ## debug plot
            
            lwg = 0.12 ## line width grid
            
            xy2d1 = np.copy( np.stack((self.x,self.y), axis=-1) / self.lchar )
            xy2d2 = np.copy( xy2d_n1D / self.lchar )
            
            plt.close('all')
            mpl.style.use('dark_background')
            fig1 = plt.figure(figsize=(8,8/2), dpi=230)
            ax1 = fig1.gca()
            ax1.set_aspect('equal')
            ax1.tick_params(axis='x', which='both', direction='out')
            ax1.tick_params(axis='y', which='both', direction='out')
            ##
            # grid_ln_y = mpl.collections.LineCollection(xy2d1,                       linewidth=lwg, edgecolors='red', zorder=19)
            # grid_ln_x = mpl.collections.LineCollection(np.transpose(xy2d1,(1,0,2)), linewidth=lwg, edgecolors='red', zorder=19)
            # ax1.add_collection(grid_ln_y)
            # ax1.add_collection(grid_ln_x)
            ##
            grid_ln_y = mpl.collections.LineCollection(xy2d2,                       linewidth=lwg, edgecolors=ax1.xaxis.label.get_color(), zorder=19)
            grid_ln_x = mpl.collections.LineCollection(np.transpose(xy2d2,(1,0,2)), linewidth=lwg, edgecolors=ax1.xaxis.label.get_color(), zorder=19)
            ax1.add_collection(grid_ln_y)
            ax1.add_collection(grid_ln_x)
            ##
            ax1.set_xlabel(r'$x/\ell_{char}$')
            ax1.set_ylabel(r'$y/\ell_{char}$')
            ##
            ax1.set_xlim(xy2d1[:,:,0].min()-3,xy2d1[:,:,0].max()+3)
            ax1.set_ylim(xy2d1[:,:,1].min()-3,xy2d1[:,:,1].max()+3)
            ##
            #ax1.set_xlim(self.plot_xlim[0],self.plot_xlim[1])
            #ax1.set_ylim(self.plot_ylim[0],self.plot_ylim[1])
            ##
            ax1.xaxis.set_major_locator(mpl.ticker.MultipleLocator(20))
            ax1.xaxis.set_minor_locator(mpl.ticker.MultipleLocator(5))
            ax1.yaxis.set_major_locator(mpl.ticker.MultipleLocator(20))
            ax1.yaxis.set_minor_locator(mpl.ticker.MultipleLocator(5))
            ##
            fig1.tight_layout(pad=0.25)
            fig1.tight_layout(pad=0.25)
            ##
            #dpi_out = 2*2160/plt.gcf().get_size_inches()[1]
            #turbx.fig_trim_x(fig1, [ax1], offset_px=10, dpi=dpi_out)
            #fig1.savefig('grid.png', dpi=dpi_out)
            plt.show()
            pass
        
        return
    
    def interp_to_wall_norm_mesh(self, **kwargs):
        '''
        interpolate fields from original grid to 'wall-normal' grid
        '''
        
        verbose = kwargs.get('verbose',True)
        scalars = kwargs.get('scalars',None)
        
        if verbose: print('\n'+'turbx.ztmd.interp_to_wall_norm_mesh()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ## read: wall normal grid x
        if ('dims_2Dw/x' in self):
            x_wn = np.copy( self['dims_2Dw/x'][()].T )
        else:
            raise AssertionError('dset not present: dims_2Dw/x')
        
        ## read: wall normal grid y
        if ('dims_2Dw/y' in self):
            y_wn = np.copy( self['dims_2Dw/y'][()].T )
        else:
            raise AssertionError('dset not present: dims_2Dw/y')
        
        # === get list of scalars to interpolate
        
        scalars_2d_names = list(self['data'].keys())
        if verbose: even_print('n scalars found in data/','%i'%(len(scalars_2d_names),))
        
        if (scalars is None): ## take all scalars present in file
            
            ## dont interpolate ddx[] or ddy[] gradients or 'wall_distance'
            scalars_2d_names = [ s for s in scalars_2d_names if ('ddx_' not in s)    ]
            scalars_2d_names = [ s for s in scalars_2d_names if ('ddy_' not in s)    ]
            scalars_2d_names = [ s for s in scalars_2d_names if (s!='wall_distance') ]
        
        else: ## explicit scalar list was passed
            
            if not isinstance(scalars, list):
                raise ValueError("'scalars' should be type list")
            if not isinstance(scalars[0], str):
                raise ValueError("'scalars' should contain strings")
            
            scalars_2d_names = list(self['data'].keys())
            
            ## take only scalars which actually exist
            scalars_2d_names = [ s for s in scalars if (s in scalars_2d_names) ]
        
        # === interpolate
        
        if verbose: even_print('n scalars to be interpolated','%i'%(len(scalars_2d_names),))
        
        if True: ## interpolate
            
            x2d_A = self.x
            y2d_A = self.y
            x2d_B = x_wn
            y2d_B = y_wn
            
            if verbose: progress_bar = tqdm(total=len(scalars_2d_names), ncols=100, desc='interpolate 2D', leave=False, file=sys.stdout)
            for scalar_name in scalars_2d_names:
                
                ## copy data into memory
                scalar_data = np.copy( self['data/%s'%scalar_name][()].T  )
                
                ## do interpolation
                if verbose: tqdm.write(even_print('start interpolate',scalar_name,s=True))
                t_start = timeit.default_timer()
                scalar_data_wn = interp_2d_structured(x2d_A, y2d_A, x2d_B, y2d_B, scalar_data)
                if verbose: tqdm.write(even_print('done interpolating','%s'%format_time_string((timeit.default_timer() - t_start)),s=True))
                
                ## write to HDF5
                if ('data_2Dw/%s'%scalar_name in self):
                    del self['data_2Dw/%s'%scalar_name]
                self.create_dataset('data_2Dw/%s'%scalar_name, data=scalar_data_wn.T, chunks=None)
                
                ## clear from memory
                scalar_data    = None ; del scalar_data
                scalar_data_wn = None ; del scalar_data_wn
                
                progress_bar.update()
            progress_bar.close()
        
        self.attrs['requires_wall_norm_interp'] = True
        self.get_header(verbose=False)
        
        if verbose: print('\n'+72*'-')
        if verbose: print('total time : turbx.interp_to_wall_norm_mesh() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        
        return
    
    def add_grid_quality_metrics_2d(self, **kwargs):
        '''
        attach grid quality measures to ZTMD
        '''
        x = np.copy( self['dims/x'][()].T )
        y = np.copy( self['dims/y'][()].T )
        
        grid_quality_dict = get_grid_quality_metrics_2d(x,y,verbose=True)
        
        if ('data_cells/skew' in self): del self['data_cells/skew']
        dset = self.create_dataset('data_cells/skew', data=grid_quality_dict['skew'].T, chunks=None)
        
        if ('data/ds1avg' in self): del self['data/ds1avg']
        dset = self.create_dataset('data/ds1avg', data=grid_quality_dict['ds1avg'].T, chunks=None)
        
        if ('data/ds2avg' in self): del self['data/ds2avg']
        dset = self.create_dataset('data/ds2avg', data=grid_quality_dict['ds2avg'].T, chunks=None)
        
        return
    
    def add_cyl_coords(self, **kwargs):
        '''
        attach [θ,r] coords, calculated from [x,y]
        '''
        cx = kwargs.get('cx',0.)
        cy = kwargs.get('cy',0.)
        
        x = np.copy( self['dims/x'][()].T )
        y = np.copy( self['dims/y'][()].T )
        
        xy2d = np.stack((x,y), axis=-1)
        
        trz = rect_to_cyl(xy2d, cx=cx, cy=cy)
        
        if ('dims/theta' in self): del self['dims/theta']
        dset = self.create_dataset('dims/theta', data=trz[:,:,0].T, chunks=None)
        if ('dims/r' in self): del self['dims/r']
        dset = self.create_dataset('dims/r', data=trz[:,:,1].T, chunks=None)
        
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from ZTMD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        makeVectors = kwargs.get('makeVectors',True) ## write vectors (e.g. velocity, vorticity) to XDMF
        makeTensors = kwargs.get('makeTensors',True) ## write 3x3 tensors (e.g. stress, strain) to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if verbose: print('\n'+'ztmd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for scalar in ['x','y','r','theta']:
            if ('dims/'+scalar in self):
                data = self['dims/'+scalar]
                dataset_precision_dict[scalar] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[scalar] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[scalar] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        ## refresh header
        self.get_header(verbose=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')
        
        # === write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            if not os.path.isfile(fname_xdmf): ## if doesnt exist...
                Path(fname_xdmf).touch() ## touch XDMF file
                perms_h5 = oct(os.stat(self.fname).st_mode)[-3:] ## get permissions of ZTMD file
                os.chmod(fname_xdmf, int(perms_h5, base=8)) ## change permissions of XDMF file
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                if self.curvilinear:
                    xdmf_str=f'''
                            <Topology TopologyType="3DSMesh" NumberOfElements="{self.ny:d} {self.nx:d}"/>
                            <Geometry GeometryType="X_Y_Z">
                            <DataItem Dimensions="{self.nx:d} {self.ny:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                                {fname_base}:/dims/{'x'}
                            </DataItem>
                            <DataItem Dimensions="{self.nx:d} {self.ny:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                                {fname_base}:/dims/{'y'}
                            </DataItem>
                            </Geometry>
                            '''
                else:
                    xdmf_str=f'''
                            <Topology TopologyType="3DRectMesh" NumberOfElements="1 {self.ny:d} {self.nx:d}"/>
                            <Geometry GeometryType="VxVyVz">
                            <DataItem Dimensions="{self.nx:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                                {fname_base}:/dims/{'x'}
                            </DataItem>
                            <DataItem Dimensions="{self.ny:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                                {fname_base}:/dims/{'y'}
                            </DataItem>
                            <DataItem Dimensions="1" Format="XML">
                                0.0
                            </DataItem>
                            </Geometry>
                            '''
                    
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # === .xdmf : <Grid> per 2D coordinate array
                    
                    if self.curvilinear:
                        
                        for scalar in ['x','y','r','theta']:
                            
                            dset_hf_path = 'dims/%s'%scalar
                            
                            if (dset_hf_path in self):
                                
                                ## get optional 'label' for Paraview (currently inactive)
                                #if scalar in scalar_names:
                                if False:
                                    scalar_name = scalar_names[scalar]
                                else:
                                    scalar_name = scalar
                                
                                xdmf_str=f'''
                                        <!-- ===== scalar : {scalar} ===== -->
                                        <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                        <DataItem Dimensions="{self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                            {fname_base}:/{dset_hf_path}
                                        </DataItem>
                                        </Attribute>
                                        '''
                                
                                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        
                        dset_hf_path = 'data/%s'%scalar
                        
                        ## get optional 'label' for Paraview (currently inactive)
                        #if scalar in scalar_names:
                        if False:
                            scalar_name = scalar_names[scalar]
                        else:
                            scalar_name = scalar
                        
                        if self.curvilinear:
                            xdmf_str=f'''
                                    <!-- ===== scalar : {scalar} ===== -->
                                    <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                    <DataItem Dimensions="{self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                        {fname_base}:/{dset_hf_path}
                                    </DataItem>
                                    </Attribute>
                                    '''
                        else:
                            xdmf_str=f'''
                                    <!-- ===== scalar : {scalar} ===== -->
                                    <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                    <DataItem Dimensions="1 {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                        {fname_base}:/{dset_hf_path}
                                    </DataItem>
                                    </Attribute>
                                    '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : <Grid> per scalar (cell-centered values)
                    
                    if ('data_cells' in self):
                        scalars_cells = list(self['data_cells'].keys())
                        for scalar in scalars_cells:
                            
                            dset_hf_path = 'data_cells/%s'%scalar
                            dset = self[dset_hf_path]
                            dset_precision = dset.dtype.itemsize
                            scalar_name = scalar
                            
                            if (dset.dtype.name=='float32') or (dset.dtype.name=='float64'):
                                dset_numbertype = 'Float'
                            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                                dset_numbertype = 'Integer'
                            else:
                                raise TypeError('dtype not recognized, please update script accordingly')
                            
                            xdmf_str=f'''
                                     <!-- ===== scalar : {scalar} ===== -->
                                     <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Cell">
                                       <DataItem Dimensions="{(self.ny-1):d} {(self.nx-1):d}" NumberType="{dset_numbertype}" Precision="{dset_precision:d}" Format="HDF">
                                         {fname_base}:/{dset_hf_path}
                                       </DataItem>
                                     </Attribute>
                                     '''
                            
                            xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    xdmf_str='''
                             <!-- ===== end scalars ===== -->
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if verbose: print('--w-> %s'%fname_xdmf_base)
        return

class spd(h5py.File):
    '''
    Surface Polydata (SPD)
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.spd()')
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
        
        ## spd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 16    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2     )
        perms          = kwargs.pop('perms'          , '640' )
        
        if not isinstance(stripe_count, int):
            raise ValueError
        if not isinstance(stripe_size_mb, int):
            raise ValueError
        if not isinstance(perms, str):
            raise ValueError
        if not len(perms)==3:
            raise ValueError
        if not re.fullmatch(r'\d{3}',perms):
            raise ValueError
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using romio --> causes crash
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        
        ## set ROMIO hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                
                mpi_info.Set('romio_ds_write' , 'disable'   )
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                mpi_info.Set('romio_cb_read'  , 'automatic' )
                mpi_info.Set('romio_cb_write' , 'automatic' )
                #mpi_info.Set('romio_cb_read'  , 'enable' )
                #mpi_info.Set('romio_cb_write' , 'enable' )
                #mpi_info.Set('cb_buffer_size' , str(int(round(16*1024**2))) ) ## collective buffer 16 [MB]
                mpi_info.Set('cb_buffer_size' , str(int(round(64*1024**2))) ) ## collective buffer 64 [MB]
                
                ## force collective I/O
                #mpi_info.Set('romio_no_indep_rw' , 'true' )
                
                ## cb_nodes: number of aggregator processes
                #mpi_info.Set('cb_nodes' , str(int(round(1*self.n_ranks))) )
                mpi_info.Set('cb_nodes' , str(max(1,int(round(self.n_ranks/4)))) )
                
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        ## | rdcc_nbytes:
        ## | ------------
        ## | Integer setting the total size of the raw data chunk cache for this dataset in bytes.
        ## | In most cases increasing this number will improve performance, as long as you have 
        ## | enough free memory. The default size is 1 MB
        
        ## --> gets passed to H5Pset_chunk_cache
        if ('rdcc_nbytes' not in kwargs):
            #kwargs['rdcc_nbytes'] = int(16*1024**2) ## 16 [MB]
            kwargs['rdcc_nbytes'] = int(256*1024**2) ## 256 [MB]
            #kwargs['rdcc_nbytes'] = int(512*1024**2) ## 512 [MB]
            #kwargs['rdcc_nbytes'] = int(1024*1024**2) ## 1024 [MB]
        
        ## | rdcc_nslots:
        ## | ------------
        ## | Integer defining the number of chunk slots in the raw data chunk cache for this dataset.
        ## | Default is 521
        ## | Should be ~= ( rdcc_nbytes / chunk size )
        
        if ('rdcc_nslots' not in kwargs):
            kwargs['rdcc_nslots'] = 2048
        
        ## spd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       --> Read only, file must exist
                                  r+      --> Read/write, file must exist
                                  w       --> Create file, truncate if exists
                                  w- or x --> Create file, fail if exists
                                  a       --> Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with spd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError()
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(0.1)
                Path(self.fname).touch() ## touch a new file
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                else:
                    #print('striping with lfs not permitted on this filesystem')
                    pass
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        self.mod_avail_tqdm = ('tqdm' in sys.modules)
        
        ## call actual h5py.File.__init__()
        super(spd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose)
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #return self
        return super(spd, self).__enter__()
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        if (self.rank==0):
            if exception_type is not None:
                print('\nsafely closed SPD HDF5 due to exception')
                print(72*'-')
                print('exception type : '+exception_type.__name__)
            if exception_value is not None:
                print('exception_value : '+str(exception_value))
            if exception_traceback is not None:
                print(72*'-')
                #print('exception_traceback : '+str(exception_traceback))
                print('exception_traceback : \n'+traceback.format_exc().rstrip())
            if exception_type is not None:
                print(72*'-')
        return super(spd, self).__exit__()
    
    def get_header(self,**kwargs):
        '''
        initialize header attributes of SPD class instance
        '''
        
        verbose = kwargs.get('verbose',True)
        
        if (self.rank!=0):
            verbose=False
        
        # === udef (header vector dset based) --> the 'old' way but still present in RGD,CGD
        
        if ('header' in self):
            
            udef_real = np.copy(self['header/udef_real'][:])
            udef_char = np.copy(self['header/udef_char'][:]) ## the unpacked numpy array of |S128 encoded fixed-length character objects
            udef_char = [s.decode('utf-8') for s in udef_char] ## convert it to a python list of utf-8 strings
            self.udef = dict(zip(udef_char, udef_real)) ## make dict where keys are udef_char and values are udef_real
            
            # === characteristic values
            
            self.Ma          = self.udef['Ma']
            self.Re          = self.udef['Re']
            self.Pr          = self.udef['Pr']
            self.kappa       = self.udef['kappa']
            self.R           = self.udef['R']
            self.p_inf       = self.udef['p_inf']
            self.T_inf       = self.udef['T_inf']
            self.mu_Suth_ref = self.udef['mu_Suth_ref']
            self.T_Suth_ref  = self.udef['T_Suth_ref']
            self.S_Suth      = self.udef['S_Suth']
            #self.C_Suth      = self.udef['C_Suth']
            
            self.C_Suth = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## [kg/(m·s·√K)]
            self.udef['C_Suth'] = self.C_Suth
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            
            # === characteristic values : derived
            
            ## mu_inf_1 = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            ## mu_inf_2 = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            ## mu_inf_3 = self.C_Suth*self.T_inf**(3/2)/(self.T_inf+self.S_Suth)
            ## if not np.isclose(mu_inf_1, mu_inf_2, rtol=1e-14):
            ##     raise AssertionError('inconsistency in Sutherland calc --> check')
            ## if not np.isclose(mu_inf_2, mu_inf_3, rtol=1e-14):
            ##     raise AssertionError('inconsistency in Sutherland calc --> check')
            ## mu_inf = self.mu_inf = mu_inf_2
            
            self.mu_inf    = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            self.rho_inf   = self.p_inf/(self.R*self.T_inf)
            self.nu_inf    = self.mu_inf/self.rho_inf
            self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            self.U_inf     = self.Ma*self.a_inf
            self.cp        = self.R*self.kappa/(self.kappa-1.)
            self.cv        = self.cp/self.kappa
            self.recov_fac = self.Pr**(1/3)
            self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            self.tchar = self.lchar / self.U_inf
            self.uchar = self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            if verbose: even_print('tchar'           , '%0.6E [s]'        % self.tchar     )
            #if verbose: print(72*'-'+'\n')
            #if verbose: print(72*'-')
            
            # === write the 'derived' udef variables to a dict attribute of the SPD instance
            self.udef_deriv = { 'rho_inf':self.rho_inf,
                                'mu_inf':self.mu_inf,
                                'nu_inf':self.nu_inf,
                                'a_inf':self.a_inf,
                                'U_inf':self.U_inf,
                                'cp':self.cp,
                                'cv':self.cv,
                                'recov_fac':self.recov_fac,
                                'Taw':self.Taw,
                                'lchar':self.lchar,
                              }
        
        else:
            #print("dset 'header' not in SPD")
            pass
        
        # === udef (attr based)
        
        header_attr_str_list = ['Ma','Re','Pr','kappa','R','p_inf','T_inf','S_Suth','mu_Suth_ref','T_Suth_ref'] ## ,'C_Suth'
        if all([ attr_str in self.attrs.keys() for attr_str in header_attr_str_list ]):
            header_attr_based = True
        else:
            header_attr_based = False
        
        if header_attr_based:
            
            ## set all attributes
            for attr_str in header_attr_str_list:
                setattr( self, attr_str, self.attrs[attr_str] )
            
            self.C_Suth = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## [kg/(m·s·√K)]
            #self.udef['C_Suth'] = self.C_Suth
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            
            # === characteristic values : derived
            
            ## mu_inf_1 = 14.58e-7*self.T_inf**1.5/(self.T_inf+110.4)
            ## mu_inf_2 = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            ## mu_inf_3 = self.C_Suth*self.T_inf**(3/2)/(self.T_inf+self.S_Suth)
            ## if not np.isclose(mu_inf_1, mu_inf_2, rtol=1e-14):
            ##     raise AssertionError('inconsistency in Sutherland calc --> check')
            ## if not np.isclose(mu_inf_2, mu_inf_3, rtol=1e-14):
            ##     raise AssertionError('inconsistency in Sutherland calc --> check')
            ## mu_inf = self.mu_inf = mu_inf_2
            
            self.mu_inf    = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            self.rho_inf   = self.p_inf/(self.R*self.T_inf)
            self.nu_inf    = self.mu_inf/self.rho_inf
            self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
            self.U_inf     = self.Ma*self.a_inf
            self.cp        = self.R*self.kappa/(self.kappa-1.)
            self.cv        = self.cp/self.kappa
            self.recov_fac = self.Pr**(1/3)
            self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            self.lchar     = self.Re*self.nu_inf/self.U_inf
            
            self.tchar = self.lchar / self.U_inf
            self.uchar = self.U_inf
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            if verbose: even_print('tchar'           , '%0.6E [s]'        % self.tchar     )
            #if verbose: print(72*'-'+'\n')
            if verbose: print(72*'-')
            
            # === write the 'derived' udef variables to a dict attribute of the RGD instance
            self.udef_deriv = { 'rho_inf':self.rho_inf,
                                'mu_inf':self.mu_inf,
                                'nu_inf':self.nu_inf,
                                'a_inf':self.a_inf,
                                'U_inf':self.U_inf,
                                'cp':self.cp,
                                'cv':self.cv,
                                'recov_fac':self.recov_fac,
                                'Taw':self.Taw,
                                'lchar':self.lchar,
                              }
        
        #if ('duration_avg' in self.attrs.keys()):
        #    self.duration_avg = self.attrs['duration_avg']
        #if ('nx' in self.attrs.keys()):
        #    self.nx = self.attrs['nx']
        #if ('ny' in self.attrs.keys()):
        #    self.ny = self.attrs['ny']
        
        # if ('p_inf' in self.attrs.keys()):
        #     self.p_inf = self.attrs['p_inf']
        # if ('lchar' in self.attrs.keys()):
        #     self.lchar = self.attrs['lchar']
        # if ('U_inf' in self.attrs.keys()):
        #     self.U_inf = self.attrs['U_inf']
        # if ('Re' in self.attrs.keys()):
        #     self.Re = self.attrs['Re']
        # if ('T_inf' in self.attrs.keys()):
        #     self.T_inf = self.attrs['T_inf']
        # if ('rho_inf' in self.attrs.keys()):
        #     self.rho_inf = self.attrs['rho_inf']
        
        if 0: ## could potentially be big
            if ('dims/xyz' in self):
                self.xyz = np.copy( self['dims/xyz'][()] )
        if ('dims/stang' in self):
            self.stang = np.copy( self['dims/stang'][()] )
        if ('dims/snorm' in self):
            self.snorm = np.copy( self['dims/snorm'][()] )
        if ('dims/crv_R' in self):
            self.crv_R = np.copy( self['dims/crv_R'][()] )
        
        if ('n_quads' in self.attrs.keys()):
            self.n_quads = int( self.attrs['n_quads'] )
        if ('n_pts' in self.attrs.keys()):
            self.n_pts = int( self.attrs['n_pts'] )
        if ('ni' in self.attrs.keys()):
            self.ni = int( self.attrs['ni'] )
        if ('nj' in self.attrs.keys()):
            self.nj = int( self.attrs['nj'] )
        
        if ('nt' in self.attrs.keys()):
            self.nt = int( self.attrs['nt'] )
        
        if ('dims/t' in self):
            self.t = t = np.copy( self['dims/t'][()] )
        if hasattr(self,'t'):
            if (self.t.ndim!=1):
                raise ValueError('self.t.ndim!=1')
            nt = self.t.shape[0]
            
            if hasattr(self,'nt'):
                if not isinstance(self.nt, (int,np.int32,np.int64)):
                    raise TypeError('self.nt is not type int')
                if (self.nt != nt):
                    raise ValueError('self.nt != nt')
            else:
                #self.attrs['nt'] = nt
                self.nt = nt
        
        ## check n_quads / n_pts is consistent with xyz
        ## if xyz exists and attrs n_quads/n_pts do not exist, set them
        if hasattr(self,'xyz'):
            if (self.xyz.ndim!=3):
                raise ValueError('self.xyz.ndim!=3')
            ni,nj,three = self.xyz.shape
            
            if hasattr(self,'ni'):
                if not isinstance(self.ni, (int,np.int32,np.int64)):
                    raise TypeError('self.ni is not type int')
                if (self.ni != ni):
                    raise ValueError('self.ni != ni')
            else:
                #self.attrs['ni'] = ni
                self.ni = ni
            
            if hasattr(self,'nj'):
                if not isinstance(self.nj, (int,np.int32,np.int64)):
                    raise TypeError('self.nj is not type int')
                if (self.nj != nj):
                    raise ValueError('self.nj != nj')
            else:
                #self.attrs['nj'] = nj
                self.nj = nj
            
            if hasattr(self,'n_quads'):
                if not isinstance(self.n_quads, (int,np.int32,np.int64)):
                    raise TypeError('self.n_quads is not type int')
                if (self.n_quads != (ni-1)*(nj-1)):
                    raise ValueError('self.n_quads != (ni-1)*(nj-1)')
            else:
                #self.attrs['n_quads'] = (ni-1)*(nj-1)
                self.n_quads = (ni-1)*(nj-1)
            
            if hasattr(self,'n_pts'):
                if not isinstance(self.n_pts, (int,np.int32,np.int64)):
                    raise TypeError('self.n_pts is not type int')
                if (self.n_pts != ni*nj):
                    raise ValueError('self.n_pts != ni*nj')
            else:
                #self.attrs['n_pts'] = ni*nj
                self.n_pts = ni*nj
        
        if any([hasattr(self,'ni'), hasattr(self,'nj'), hasattr(self,'n_quads'), hasattr(self,'n_pts') ]):
            if verbose and hasattr(self,'nt'):      even_print('nt',      f'{self.nt:d}')
            if verbose and hasattr(self,'ni'):      even_print('ni',      f'{self.ni:d}')
            if verbose and hasattr(self,'nj'):      even_print('nj',      f'{self.nj:d}')
            if verbose and hasattr(self,'n_quads'): even_print('n_quads', f'{self.n_quads:d}')
            if verbose and hasattr(self,'n_pts'):   even_print('n_pts',   f'{self.n_pts:d}')
            #if verbose: print(72*'-')
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys())
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self['data/%s'%scalar].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    def gen_unstruct_xyz(self,**kwargs):
        '''
        convert structured grid coords (data/xyz) to unstructured:
        dims/quads
        dims/pts
        '''
        
        verbose    = kwargs.get( 'verbose'    , True )
        #indexing   = kwargs.get( 'indexing'   , 'xy' ) ## 'xy', 'ij'
        chunk_kb   = kwargs.get( 'chunk_kb'   , 1*1024 ) ## 1 [MB]
        chunk_base = kwargs.get( 'chunk_base' , 2 )
        
        if self.usingmpi:
            raise ValueError('spd.gen_unstruct_xyz() should not be run in MPI mode')
        
        if ('dims/xyz' not in self):
            raise ValueError('dims/xyz not in file')
        
        if verbose: print('\n'+'spd.gen_unstruct_xyz()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        xyz = np.copy( self['dims/xyz'][()] )
        if verbose:
            even_print('dims/xyz',str(xyz.shape))
        
        if (xyz.ndim!=3):
            raise ValueError('xyz.ndim!=3')
        
        ni,nj,three = xyz.shape
        
        n_quads = (ni-1)*(nj-1)
        n_pts   = ni*nj
        nt      = self.nt
        
        if verbose:
            even_print('ni'      , f'{ni:d}' )
            even_print('nj'      , f'{nj:d}' )
            even_print('n_quads' , f'{n_quads:d}' )
            even_print('n_pts'   , f'{n_pts:d}' )
            even_print('nt'      , f'{nt:d}' )
        
        # ===
        
        indexing = 'xy'
        
        xi, yi = np.meshgrid(np.arange(ni,dtype=np.int64), np.arange(nj,dtype=np.int64), indexing=indexing)
        
        inds_list = np.stack((xi,yi), axis=2)
        inds_list = np.reshape(inds_list, (ni*nj,2), order='C')
        inds_list = np.ravel_multi_index((inds_list[:,0],inds_list[:,1]), (ni,nj), order='F')
        inds_list = np.reshape(inds_list, (ni,nj), order='C')
        
        if verbose:
            progress_bar = tqdm(
                total=(ni-1)*(nj-1),
                ncols=100,
                desc='quads',
                smoothing=1.,
                leave=False,
                file=sys.stdout,
                bar_format="{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}\n\033[F\r",
                ascii="░█",
                )
        
        ## quad index array
        quads = np.zeros(((ni-1),(nj-1),4), dtype=np.int64)
        for i in range(ni-1):
            for j in range(nj-1):
                
                ## Counter-Clockwise (CCW)
                quads[i,j,0] = inds_list[i,   j  ]
                quads[i,j,1] = inds_list[i+1, j  ]
                quads[i,j,2] = inds_list[i+1, j+1]
                quads[i,j,3] = inds_list[i,   j+1]
                
                ## Clockwise (CW)
                #quads[i,j,0] = inds_list[i,   j  ]
                #quads[i,j,1] = inds_list[i,   j+1]
                #quads[i,j,2] = inds_list[i+1, j+1]
                #quads[i,j,3] = inds_list[i+1, j  ]
                
                if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        # ===
        
        if (indexing=='xy'):
            order='C'
        elif (indexing=='ij'):
            order = 'F'
        else:
            raise ValueError
        
        # === dims_unstruct/quads
        
        ## flatten quad index vector
        quads = np.reshape(quads, ((ni-1)*(nj-1),4), order=order)
        
        dsn = 'dims/quads'
        if (dsn in self):
            del self[dsn]
        
        shape = quads.shape
        dtype = quads.dtype
        itemsize = quads.dtype.itemsize
        chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None), size_kb=chunk_kb, base=chunk_base, itemsize=itemsize)
        ds = self.create_dataset(
                            dsn,
                            shape=shape,
                            chunks=chunks,
                            dtype=dtype,
                            )
        
        chunk_kb_ = np.prod(ds.chunks)*itemsize / 1024. ## actual
        if verbose:
            even_print('chunk shape (n_quads,4)','%s'%str(ds.chunks))
            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        ## write
        ds[:,:] = quads
        
        # === dims_unstruct/pts
        
        ## flatten point coordinate vector
        pts = np.reshape(xyz, (ni*nj,3), order=order)
        
        dsn = 'dims/pts'
        if (dsn in self):
            del self[dsn]
        
        shape = pts.shape
        dtype = pts.dtype
        itemsize = pts.dtype.itemsize
        chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None), size_kb=chunk_kb, base=chunk_base, itemsize=itemsize)
        ds = self.create_dataset(
                            dsn,
                            shape=shape,
                            chunks=chunks,
                            dtype=dtype,
                            )
        
        chunk_kb_ = np.prod(ds.chunks)*itemsize / 1024. ## actual
        if verbose:
            even_print('chunk shape (n_pts,3)','%s'%str(ds.chunks))
            even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: even_print(dsn,'%s'%str(ds.shape))
        
        ## write
        ds[:,:] = pts
        
        if verbose: print(72*'-')
        if verbose: print('total time : spd.gen_unstruct_xyz() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    @staticmethod
    def copy(fn_spd_src, fn_spd_tgt, **kwargs):
        '''
        copy header info, selected scalars, and [i,j,t] range to new SPD file
        - currently copies complete [i,j] range
        - if [i,j] range clipping were to be implemented, taking data_unstruct would be difficult
        --> this currently does NOT work in serial mode
        '''
        
        #comm    = MPI.COMM_WORLD
        rank    = MPI.COMM_WORLD.Get_rank()
        n_ranks = MPI.COMM_WORLD.Get_size()
        
        if (rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'spd.copy()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        rx       = kwargs.get('rx',None)
        ry       = kwargs.get('ry',None)
        rz       = kwargs.get('rz',None)
        
        ri       = kwargs.get('ri',1)
        rj       = kwargs.get('rj',1)
        
        rt       = kwargs.get('rt',1)
        force    = kwargs.get('force',False) ## overwrite or raise error if exists
        
        ti_min   = kwargs.get('ti_min',None)
        ti_max   = kwargs.get('ti_max',None)
        scalars  = kwargs.get('scalars',None)
        
        i_min  = kwargs.get( 'i_min'  , None )
        i_max  = kwargs.get( 'i_max'  , None )
        j_min  = kwargs.get( 'j_min'  , None )
        j_max  = kwargs.get( 'j_max'  , None )
        ti_min = kwargs.get( 'ti_min' , None )
        ti_max = kwargs.get( 'ti_max' , None )
        
        ct = kwargs.get('ct',1) ## 'chunks' in time
        
        chunk_kb         = kwargs.get('chunk_kb',2*1024) ## h5 chunk size: default 2 [MB]
        chunk_constraint = kwargs.get('chunk_constraint',(None,None,1)) ## the 'constraint' parameter for sizing h5 chunks (i,j,t)
        chunk_base       = kwargs.get('chunk_base',2)
        
        stripe_count   = kwargs.pop('stripe_count'   , 16 ) ## for initializing SPD file
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2  )
        
        xi_step = kwargs.get('xi_step',1)
        yi_step = kwargs.get('yi_step',1)
        zi_step = kwargs.get('zi_step',1)
        
        if (rx is not None):
            raise ValueError('rx not a valid option for spd.copy(). accepted are: ri,rj')
        if (ry is not None):
            raise ValueError('ry not a valid option for spd.copy(). accepted are: ri,rj')
        if (rz is not None):
            raise ValueError('rz not a valid option for spd.copy(). accepted are: ri,rj')
        
        if (i_min is not None):
            raise NotImplementedError('i/j_min/max not yet supported')
        if (i_max is not None):
            raise NotImplementedError('i/j_min/max not yet supported')
        if (j_min is not None):
            raise NotImplementedError('i/j_min/max not yet supported')
        if (j_max is not None):
            raise NotImplementedError('i/j_min/max not yet supported')
        
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (ri*rj!=n_ranks):
            raise AssertionError('ri*rj!=n_ranks')
        if not os.path.isfile(fn_spd_src):
            raise FileNotFoundError('%s not found!'%fn_spd_src)
        if os.path.isfile(fn_spd_tgt) and not force:
            raise FileExistsError('%s already exists. delete it or use \'force=True\' kwarg'%fn_spd_tgt)
        
        # ===
        
        with spd(fn_spd_src, 'r', comm=MPI.COMM_WORLD, driver='mpio') as hf_src:
            with spd(fn_spd_tgt, 'w',
                     force=force,
                     comm=MPI.COMM_WORLD,
                     driver='mpio',
                     stripe_count=stripe_count,
                     stripe_size_mb=stripe_size_mb) as hf_tgt:
                
                ni      = hf_src.ni
                nj      = hf_src.nj
                n_quads = hf_src.n_quads
                n_pts   = hf_src.n_pts
                nt      = hf_src.nt
                
                ## report info from source file
                fsize = os.path.getsize(hf_src.fname)/1024**3
                if verbose: even_print(os.path.basename(hf_src.fname),'%0.1f [GB]'%fsize)
                if verbose: even_print('ni','%i'%hf_src.ni)
                if verbose: even_print('nj','%i'%hf_src.nj)
                if verbose: even_print('nt','%i'%hf_src.nt)
                if verbose: even_print('n_quads','%i'%hf_src.n_quads)
                if verbose: even_print('n_pts','%i'%hf_src.n_pts)
                if verbose: print(72*'-')
                
                # ===
                
                ## get OUTPUT times
                t_  = np.copy(hf_src['dims/t'][()])
                ti_ = np.arange(t_.shape[0], dtype=np.int32)
                if (ti_min is None):
                    ti_min = ti_.min()
                if (ti_max is None):
                    ti_max = ti_.max()
                ti  = np.copy(ti_[ti_min:ti_max+1])
                if (ti.shape[0]==0):
                    raise ValueError('ti_min/ti_max combo yields no times')
                ti1 = ti.min()
                ti2 = ti.max()+1
                t   = np.copy(t_[ti1:ti2])
                nt  = t.shape[0]
                
                if (ti_min<0):
                    if verbose: even_print('ti_min', f'{ti_min:d} / {ti1:d}')
                else:
                    if verbose: even_print('ti_min', f'{ti_min:d}')
                
                if (ti_max<0):
                    if verbose: even_print('ti_max', f'{ti_max:d} / {ti2:d}')
                else:
                    if verbose: even_print('ti_max', f'{ti_max:d}')
                
                if verbose: even_print('t range', f'{ti.shape[0]:d}/{ti_.shape[0]:d}')
                
                # ===
                
                ## time chunk ranges
                if (ct>nt):
                    raise ValueError('ct>nt')
                
                tfi = np.arange(ti1,ti2,dtype=np.int64)
                ctl_ = np.array_split(tfi,ct)
                ctl = [[b[0],b[-1]+1] for b in ctl_ ]
                
                if verbose: print(72*'-')
                
                # ===
                
                ## copy over attributes
                for key,val in hf_src.attrs.items():
                    hf_tgt.attrs[key] = val
                
                # === get rank distribution over (i,j) dims
                
                comm2d = hf_src.comm.Create_cart(dims=[ri,rj], periods=[False,False], reorder=False)
                t2d = comm2d.Get_coords(rank)
                
                ril_ = np.array_split(np.arange(hf_src.ni,dtype=np.int64),ri)
                rjl_ = np.array_split(np.arange(hf_src.nj,dtype=np.int64),rj)
                
                ril = [[b[0],b[-1]+1] for b in ril_ ]
                rjl = [[b[0],b[-1]+1] for b in rjl_ ]
                
                ri1, ri2 = ril[t2d[0]]; nir = ri2 - ri1
                rj1, rj2 = rjl[t2d[1]]; njr = rj2 - rj1
                
                # === copy over non-attribute metadata
                
                ## 'dims/xyz' : 3D polydata grid coordinates : shape=(ni,nj,3)
                dsn = f'dims/xyz'
                dset = hf_src[dsn]
                dtype = dset.dtype
                float_bytes = dtype.itemsize
                with dset.collective:
                    xyz = np.copy( dset[ri1:ri2,rj1:rj2,:] )
                shape  = (ni,nj,3)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,3), size_kb=chunk_kb, base=4, itemsize=float_bytes)
                data_gb = float_bytes * ni * nj / 1024**3
                if verbose:
                    even_print(f'initializing {dsn}','%0.1f [GB]'%(data_gb,))
                dset = hf_tgt.create_dataset(dsn, dtype=xyz.dtype, shape=shape, chunks=chunks)
                chunk_kb_ = np.prod(dset.chunks)*float_bytes / 1024. ## actual
                if verbose:
                    even_print('chunk shape (i,j,3)',str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                with dset.collective:
                    dset[ri1:ri2,rj1:rj2,:] = xyz
                
                if verbose: print(72*'-')
                
                ## copy over [t]
                dsn = f'dims/t'
                ds = hf_tgt.create_dataset(dsn, chunks=None, data=t)
                if verbose: even_print(dsn,str(ds.shape))
                hf_tgt.attrs['nt'] = t.shape[0]
                
                ## copy over additional [dims/<>] dsets
                for dsn in [ 'dims/stang', 'dims/snorm', 'dims/crv_R', 'dims/x', 'dims/y', 'dims/z' ]:
                    if (dsn in hf_src):
                        data = np.copy(hf_src[dsn][()])
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,str(ds.shape))
                    else:
                        if verbose: even_print(dsn,'not found')
                
                ## copy over additional [csys/<>] dsets
                for dsn in [ 'csys/vtang', 'csys/vnorm' ]:
                    if (dsn in hf_src):
                        data = np.copy(hf_src[dsn][()])
                        ds = hf_tgt.create_dataset(dsn, data=data, chunks=None)
                        if verbose: even_print(dsn,str(ds.shape))
                    else:
                        if verbose: even_print(dsn,'not found')
                
                if verbose: print(72*'-')
                hf_tgt.get_header(verbose=verbose)
                if verbose: print(72*'-')
                
                # === initialize datasets in target file
                
                for scalar in hf_src.scalars:
                    
                    dsn = f'data/{scalar}'
                    ds = hf_src[dsn]
                    dtype = ds.dtype
                    float_bytes = dtype.itemsize
                    
                    data_gb = ni * nj * nt * float_bytes / 1024**3
                    shape   = (ni,nj,nt)
                    chunks  = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, base=chunk_base, itemsize=float_bytes)
                    
                    if verbose:
                        even_print(f'initializing data/{scalar}','%0.1f [GB]'%(data_gb,))
                    if (dsn in hf_tgt):
                        del hf_tgt[dsn]
                    dset = hf_tgt.create_dataset(
                                            dsn,
                                            shape=shape,
                                            dtype=dtype,
                                            chunks=chunks,
                                            )
                    
                    chunk_kb_ = np.prod(dset.chunks)*4 / 1024. ## actual
                    if verbose:
                        even_print('chunk shape (i,j,t)','%s'%str(dset.chunks))
                        even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if verbose: print(72*'-')
                
                # === main loop
                
                data_gb_read  = 0.
                data_gb_write = 0.
                t_read  = 0.
                t_write = 0.
                
                if verbose:
                    progress_bar = tqdm(
                        total=len(ctl)*hf_src.n_scalars,
                        ncols=100,
                        desc='copy',
                        smoothing=1.,
                        leave=False,
                        file=sys.stdout,
                        bar_format="{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}\n\033[F\r",
                        ascii="░█",
                        )
                
                for scalar in hf_src.scalars:
                    dset_src = hf_src[f'data/{scalar}']
                    dset_tgt = hf_tgt[f'data/{scalar}']
                    
                    dtype = dset_src.dtype
                    float_bytes = dtype.itemsize
                    
                    for ctl_ in ctl:
                        ct1, ct2 = ctl_
                        ntc = ct2 - ct1
                        
                        ## read
                        hf_src.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_src.collective:
                            data = np.copy( dset_src[ri1:ri2,rj1:rj2,ct1:ct2] )
                        hf_src.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * ni * nj * ntc / 1024**3
                        
                        t_read       += t_delta
                        data_gb_read += data_gb
                        
                        if verbose:
                            tqdm.write(even_print(f'read: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        ## write
                        hf_tgt.comm.Barrier()
                        t_start = timeit.default_timer()
                        with dset_tgt.collective:
                            dset_tgt[ri1:ri2,rj1:rj2,:] = data
                        hf_tgt.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = float_bytes * ni * nj * ntc / 1024**3
                        
                        t_write       += t_delta
                        data_gb_write += data_gb
                        
                        if verbose:
                            tqdm.write(even_print(f'write: {scalar}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        if verbose: progress_bar.update()
                
                if verbose: progress_bar.close()
        
        if verbose: print(72*'-')
        if verbose: print('total time : spd.copy() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # === wall [y] pln post-processing
    
    def import_eas4_special_wall(self, fn_eas4_list, **kwargs):
        '''
        directly export a wall SPD file from EAS4s
        - only parallelize in [x]
        '''
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        if verbose: print('\n'+'spd.import_eas4_special_wall()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not self.usingmpi:
            if verbose: print('this function has not been implemented in non-MPI mode')
            sys.exit(1)
        
        if not h5py.h5.get_config().mpi:
            if verbose: print('h5py was not compiled for parallel usage! exiting.')
            sys.exit(1)
        
        acc          = kwargs.get('acc',4)
        edge_stencil = kwargs.get('edge_stencil','full')
        
        ## delete EAS4s after import --> DANGER!
        delete_after_import = kwargs.get('delete_after_import',False)
        
        ## check that the passed iterable of EAS4 files is OK
        if not hasattr(fn_eas4_list, '__iter__'):
            raise AssertionError('first arg \'fn_eas4_list\' must be iterable')
        for fn_eas4 in fn_eas4_list:
            if not os.path.isfile(fn_eas4):
                raise FileNotFoundError('%s not found!'%fn_eas4)
        
        ## n ranks per direction
        rx = kwargs.get('rx',1)
        if (rx != self.n_ranks):
            raise AssertionError('rx != self.n_ranks')
        
        ## nx = ni
        if not hasattr(self,'ni'):
            raise ValueError('attribute ni not found.')
        else:
            self.nx = self.ni
        
        ## nz = nj
        if not hasattr(self,'nj'):
            raise ValueError('attribute nj not found.')
        else:
            self.nz = self.nj
        
        if self.usingmpi:
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            rxl  = [[b[0],b[-1]+1] for b in rxl_ ]
            rx1, rx2 = rxl[self.rank]
            nxr = rx2 - rx1
        else:
            rx1,rx2 = 0,self.nx
            nxr = self.nx
        
        ## get all time info & check
        if (self.rank==0):
            t = np.array([], dtype=np.float64)
            for fn_eas4 in fn_eas4_list:
                with eas4(fn_eas4, 'r', verbose=False) as hf_eas4:
                    t_ = np.copy(hf_eas4.t)
                t = np.concatenate((t,t_))
        else:
            t = np.array([], dtype=np.float64) ## 't' must exist on all ranks
        
        self.comm.Barrier()
        t = self.comm.bcast(t, root=0)
        nt = t.shape[0]
        self.nt = nt
        self.t = t
        assert t.dtype == np.float64, f'Array dtype is {str(t.dtype)}, expected np.float64'
        
        if verbose: even_print('n EAS4 files','%i'%len(fn_eas4_list))
        if verbose: even_print('nt all files','%i'%nt)
        if verbose: even_print('delete after import',str(delete_after_import))
        if verbose: print(72*'-')
        
        ## check [t] & Δt
        if (nt>1):
            
            ## check no zero distance elements
            if (np.diff(t).size - np.count_nonzero(np.diff(t))) != 0.:
                raise AssertionError('t arr has zero-distance elements')
            else:
                if verbose: even_print('check: Δt!=0','passed')
            
            ## check monotonically increasing
            if not np.all(np.diff(t) > 0.):
                raise AssertionError('t arr not monotonically increasing')
            else:
                if verbose: even_print('check: t mono increasing','passed')
            
            ## check constant Δt
            dt0 = np.diff(t)[0]
            if not np.all(np.isclose(np.diff(t), dt0, rtol=1e-3)):
                if (self.rank==0): print(np.diff(t))
                raise AssertionError('t arr not uniformly spaced')
            else:
                if verbose: even_print('check: constant Δt','passed')
            
            ## write [t]
            self.comm.Barrier()
            if 'dims/t' in self:
                del self['dims/t']
            self.create_dataset('dims/t', data=t, chunks=None, dtype=np.float64)
            self.attrs['nt'] = nt
            self.attrs['dt'] = dt0
            self.attrs['duration'] = dt0 * (nt - 1)
            self.flush()
            self.comm.Barrier()
            if verbose: print(72*'-')
        
        else:
            return ## nothing to do
        
        # ==============================================================
        # initialize
        # ==============================================================
        
        scalars = [ 'tau_uy','tau_vy','tau_wy',
                    'u_tau','v_tau','w_tau',
                    'T','mu','nu','rho','p', ]
        
        dtype = np.dtype(np.float32)
        shape = (self.nx, self.nz, self.nt)
        data_gb = np.prod(shape) * dtype.itemsize / 1024**3
        
        ## HARDCODED!!! TODO!!!
        #chunks = ( 8, self.nz, min(8,self.nt) )
        #chunks = ( 79, self.nz, 1 )
        #chunks = ( 79, self.nz, 8 ) ## (x,z,t)
        #chunks = ( 1, self.nz, 150 ) ## (x,z,t)
        #chunks = ( 2, self.nz, 150 ) ## (x,z,t)
        chunks = ( 4, self.nz, 50 ) ## (x,z,t)
        #chunks = ( 8, self.nz, 50 ) ## (x,z,t)
        
        ## initialize output datasets
        for scalar in scalars:
            
            dsn = f'data/{scalar}'
            if not (dsn in self):
                self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if verbose:
                    even_print(f'initializing data/{scalar}',f'{str(shape)} / {data_gb:0.1f} [GB]')
                
                dset = self.create_dataset(
                                        dsn,
                                        shape=shape,
                                        dtype=dtype,
                                        chunks=chunks,
                                        )
                
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose: even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]')
                
                chunk_kb_ = np.prod(dset.chunks) * dset.dtype.itemsize / 1024. ## actual
                if verbose:
                    even_print('chunk shape (x,z,t)','%s'%str(dset.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
        
        if verbose: print(72*'-')
        
        # ==============================================================
        # read,process,write
        # ==============================================================
        
        if verbose:
            progress_bar = tqdm(
                #total=self.nt,
                total=len(fn_eas4_list),
                ncols=100,
                desc='import',
                smoothing=0.,
                leave=False,
                file=sys.stdout,
                bar_format="{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}\n\033[F\r",
                ascii="░█",
                )
        
        tii = 0 ## counter full series
        
        for fn_eas4 in fn_eas4_list:
            
            with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=self.comm) as hf_eas4:
                
                ## dimensional coordinates
                x = np.copy( hf_eas4.x * hf_eas4.lchar )
                y = np.copy( hf_eas4.y * hf_eas4.lchar )
                z = np.copy( hf_eas4.z * hf_eas4.lchar )
                
                nx = hf_eas4.nx
                ny = hf_eas4.ny
                nz = hf_eas4.nz
                nt = hf_eas4.nt ## overwriting above!
                
                if verbose:
                    tqdm.write(even_print(os.path.basename(fn_eas4), f'{(os.path.getsize(fn_eas4)/1024**3):0.1f} [GB]', s=True))
                
                ## rank-local data read buffer for this EAS4
                scalars = ['rho','u','v','w','T','p']
                formats = [ np.dtype(np.float32) for s in scalars ]
                data    = np.zeros(shape=(nxr,ny,nz,nt), dtype={'names':scalars, 'formats':formats})
                
                ## read (full file)
                self.comm.Barrier()
                t_start = timeit.default_timer()
                for ti in range(hf_eas4.nt): ## timesteps in EAS4
                    for scalar in scalars:
                        dset_path = f'Data/DOMAIN_000000/ts_{ti:06d}/par_{hf_eas4.scalar_n_map[scalar]:06d}'
                        dset = hf_eas4[dset_path]
                        with dset.collective:
                            data[scalar][:,:,:,ti] = dset[rx1:rx2,:,:]
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                data_gb = os.path.getsize(fn_eas4)/1024**3
                if verbose:
                    #msg = even_print(f'read {os.path.basename(fn_eas4)}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]', s=True)
                    msg = even_print(f'read', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]', s=True)
                    tqdm.write(msg)
                
                ## re-dimensionalize EAS4 data
                data['rho'][:,:,:,:] *= hf_eas4.rho_inf
                data['u'][:,:,:,:]   *= hf_eas4.U_inf
                data['v'][:,:,:,:]   *= hf_eas4.U_inf
                data['w'][:,:,:,:]   *= hf_eas4.U_inf
                data['T'][:,:,:,:]   *= hf_eas4.T_inf
                data['p'][:,:,:,:]   *= hf_eas4.rho_inf * hf_eas4.U_inf**2
                
                mu = np.copy( hf_eas4.mu_Suth_ref * ( data['T'][:,:,:,:] / hf_eas4.T_Suth_ref )**(3/2) * ((hf_eas4.T_Suth_ref+hf_eas4.S_Suth)/(data['T'][:,:,:,:]+hf_eas4.S_Suth)) )
                nu = np.copy( mu / data['rho'][:,:,:,:] )
            
            # ==========================================================
            
            ## dimensional wall strains
            ddy_u      = gradient( data['u'][:,:,:,:], y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_v      = gradient( data['v'][:,:,:,:], y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_w      = gradient( data['w'][:,:,:,:], y, axis=1, acc=acc, edge_stencil=edge_stencil, d=1)
            ddy_u_wall = np.copy( ddy_u[:,0,:,:] )
            ddy_v_wall = np.copy( ddy_v[:,0,:,:] )
            ddy_w_wall = np.copy( ddy_w[:,0,:,:] )
            
            if (ddy_u_wall.ndim!=3): ## (x,z,t)
                raise ValueError
            if (ddy_u_wall.shape!=(nxr,nz,nt)): ## (x,z,t)
                raise ValueError
            
            ## dimensional wall quantities
            rho_wall = np.copy( data['rho'][:,0,:,:] )
            mu_wall  = np.copy( mu[:,0,:,:]          )
            nu_wall  = np.copy( nu[:,0,:,:]          )
            T_wall   = np.copy( data['T'][:,0,:,:]   )
            p_wall   = np.copy( data['p'][:,0,:,:]   )
            
            if (mu_wall.ndim!=3): ## (x,z,t)
                raise ValueError
            if (mu_wall.shape!=(nxr,nz,nt)): ## (x,z,t)
                raise ValueError
            
            if (rho_wall.ndim!=3): ## (x,z,t)
                raise ValueError
            if (rho_wall.shape!=(nxr,nz,nt)): ## (x,z,t)
                raise ValueError
            
            tau_uy = np.copy( mu_wall * ddy_u_wall ) ## INSTANTANEOUS τw
            tau_vy = np.copy( mu_wall * ddy_v_wall )
            tau_wy = np.copy( mu_wall * ddy_w_wall )
            
            u_tau = np.copy( np.sign(tau_uy) * np.sqrt( np.abs(tau_uy) / rho_wall ) ) ## INSTANTANEOUS uτ
            v_tau = np.copy( np.sign(tau_vy) * np.sqrt( np.abs(tau_vy) / rho_wall ) ) ## INSTANTANEOUS vτ
            w_tau = np.copy( np.sign(tau_wy) * np.sqrt( np.abs(tau_wy) / rho_wall ) ) ## INSTANTANEOUS wτ
            
            #grp      = self['data']
            #scalars  = list(grp.keys())
            scalars = [ 'tau_uy','tau_vy','tau_wy',
                        'u_tau','v_tau','w_tau',
                        'T','mu','nu','rho','p', ]
            
            ## 3D [scalar][x,z,t] numpy structured array
            ## rank-local data buffer for write
            formats  = [ np.dtype(np.float32) for s in scalars ]
            data4spd = np.zeros(shape=(nxr,hf_eas4.nz,hf_eas4.nt), dtype={'names':scalars, 'formats':formats})
            data4spd['tau_uy'][:,:,:] = tau_uy   / ( self.rho_inf * self.U_inf**2 )
            data4spd['tau_vy'][:,:,:] = tau_vy   / ( self.rho_inf * self.U_inf**2 )
            data4spd['tau_wy'][:,:,:] = tau_wy   / ( self.rho_inf * self.U_inf**2 )
            data4spd['T'][:,:,:]      = T_wall   / self.T_inf
            data4spd['rho'][:,:,:]    = rho_wall / self.rho_inf
            data4spd['p'][:,:,:]      = p_wall   / ( self.rho_inf * self.U_inf**2 )
            data4spd['mu'][:,:,:]     = mu_wall  / self.mu_inf
            data4spd['nu'][:,:,:]     = nu_wall  / self.nu_inf
            data4spd['u_tau'][:,:,:]  = u_tau    / self.U_inf
            data4spd['v_tau'][:,:,:]  = v_tau    / self.U_inf
            data4spd['w_tau'][:,:,:]  = w_tau    / self.U_inf
            
            tiA = tii
            tiB = tiA + nt
            tii += nt ## increment tii by this EAS4's nt
            
            ## write
            self.comm.Barrier()
            t_start = timeit.default_timer()
            #for scalar in data4spd.dtype.names:
            for scalar in scalars:
                dset = self[f'data/{scalar}']
                with dset.collective:
                    dset[rx1:rx2,:,tiA:tiB] = data4spd[scalar][:,:,:]
            self.comm.Barrier()
            t_delta = timeit.default_timer() - t_start
            
            ## report write speed
            data_gb = self.n_ranks * data4spd.nbytes / 1024**3
            if verbose:
                msg = even_print(f'write', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]', s=True)
                tqdm.write(msg)
            
            ## delete source file
            if delete_after_import:
                if os.path.isfile('do_delete.txt'):
                    if (self.rank==0):
                        tqdm.write(even_print('deleting', fn_eas4, s=True))
                        os.remove(fn_eas4)
                    self.comm.Barrier()
            
            if verbose: progress_bar.update()
        if verbose: progress_bar.close()
        
        ## report file
        self.comm.Barrier()
        if verbose:
            print(72*'-')
            even_print( os.path.basename(self.fname), f'{(os.path.getsize(self.fname)/1024**3):0.1f} [GB]')
        if verbose: print(72*'-')
        if verbose: print('total time : spd.import_eas4_special_wall() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_spectrum_wall(self, **kwargs):
        '''
        calculate FFT cospectrum in [z,t] at every [x]
        - designed for analyzing unsteady, pre-computed wall quantities ([y] plane)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'spd.calc_spectrum_wall()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not self.usingmpi:
            raise NotImplementedError('function is not implemented for non-MPI usage')
        
        h5py_is_mpi_build = h5py.h5.get_config().mpi
        if not h5py_is_mpi_build:
            if verbose: print('h5py was not compiled for parallel usage! exiting.')
            sys.exit(1)
        
        ri = kwargs.get('ri',1)
        rj = kwargs.get('rj',1)
        rt = kwargs.get('rt',1)
        
        #n_threads = kwargs.get('n_threads',1)
        try:
            n_threads = int(os.environ.get('OMP_NUM_THREADS'))
        except TypeError: ## not set
            n_threads = os.cpu_count()
        
        #fn_dat_fft = kwargs.get('fn_dat_fft',None) ## filename for output pickle (.dat) file
        fn_h5_fft  = kwargs.get('fn_h5_fft',None) ## filename for output HDF5 (.h5) file
        ti_min     = kwargs.get('ti_min',None)
        
        overlap_fac_nom = kwargs.get('overlap_fac_nom',0.50)
        n_win           = kwargs.get('n_win',8)
        window_type     = kwargs.get('window_type','hann') ## 'tukey','hann'
        
        ## check data distribution
        if (rj!=1):
            raise AssertionError('rj!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (ri*rj*rt != self.n_ranks):
            raise AssertionError('ri*rj*rt != self.n_ranks')
        if (ri>self.ni):
            raise AssertionError('ri>self.ni')
        if (self.ni%ri!=0):
            raise AssertionError('ni currently needs to be divisible by the n ranks')
        
        ## distribute data over [x]
        ril_ = np.array_split(np.arange(self.ni,dtype=np.int64),min(ri,self.ni))
        ril = [[b[0],b[-1]+1] for b in ril_ ]
        ri1,ri2 = ril[self.rank]
        nir = ri2 - ri1
        
        ## output filename : HDF5 (.h5)
        if (fn_h5_fft is None): ## automatically determine name
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            #fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_fft_h5_base = fname_root+'_fft.h5'
            fn_h5_fft = str(PurePosixPath(fname_path, fname_fft_h5_base))
        if (Path(fn_h5_fft).suffix != '.h5'):
            raise ValueError(f"fn_h5_fft='{str(fn_h5_fft)}' must end in .h5")
        if os.path.isfile(fn_h5_fft):
            if (fn_h5_fft == self.fname):
                raise ValueError(f"fn_h5_fft='{str(fn_h5_fft)}' cannot be same as input filename.")
        
        if verbose: even_print( 'fn_h5'      , self.fname )
        if verbose: even_print( 'fn_h5_fft'  , fn_h5_fft  )
        if verbose: print(72*'-')
        self.comm.Barrier()
        
        # ===
        
        ## the data dictionary to be pickled or written to .h5 later
        data = {}
        
        ## freestream data
        lchar   = self.lchar   ; data['lchar']   = self.lchar
        U_inf   = self.U_inf   ; data['U_inf']   = self.U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = self.rho_inf
        T_inf   = self.T_inf   ; data['T_inf']   = self.T_inf
        mu_inf  = self.mu_inf  ; data['mu_inf']  = self.mu_inf
        data['p_inf'] = self.p_inf
        data['Ma']    = self.Ma
        data['Pr']    = self.Pr
        
        ## read in 1D coordinate arrays & re-dimensionalize
        x = np.copy( self['dims/x'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        data['x'] = x
        data['z'] = z
        data['t'] = t
        
        nx = self.ni
        ni = self.ni
        nz = self.nj
        nj = self.nj
        data['ni'] = ni
        data['nx'] = nx
        data['nj'] = nj
        data['nz'] = nz
        
        nt = self.nt
        data['nt'] = nt
        
        ## assert constant Δz
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-6)):
            raise NotImplementedError('Δz not constant')
        
        ## get Δt, dimensional [s]
        if hasattr(self,'dt'):
            dt = self.dt * self.tchar
            np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-12, atol=1e-12)
        else:
            dt = t[1] - t[0] ## already dimensional
        
        if hasattr(self,'duration'):
            t_meas = self.duration * self.tchar
            np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-12, atol=1e-12)
        else:
            t_meas = t[-1] - t[0] ## already dimensional
            self.duration = t_meas / self.tchar ## non-dimensionalize for attribute
        
        zrange = z.max() - z.min()
        
        data['t'] = t
        data['dt'] = dt
        data['dz'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print( 'Δt/tchar'       , f'{dt/self.tchar:0.8f}' )
        if verbose: even_print( 'Δt'             , f'{dt:0.3e} [s]'        )
        if verbose: even_print( 'duration/tchar' , f'{self.duration:0.1f}' )
        if verbose: even_print( 'duration'       , f'{self.duration*self.tchar:0.3e} [s]' )
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('Δt'     , f'{dt    :0.5e} [s]' )
            even_print('t_meas' , f'{t_meas:0.5e} [s]' )
            even_print('Δz'     , f'{dz0   :0.5e} [m]' )
            even_print('zrange' , f'{zrange:0.5e} [m]' )
            print(72*'-')
        
        win_len, overlap = get_overlapping_window_size(nt, n_win, overlap_fac_nom)
        overlap_fac = overlap / win_len
        tw, n_win, n_pad = get_overlapping_windows(t, win_len, overlap)
        t_meas_per_win = (win_len-1)*dt
        data['win_len']        = win_len
        data['overlap_fac']    = overlap_fac
        data['overlap']        = overlap
        data['n_win']          = n_win
        data['t_meas_per_win'] = t_meas_per_win
        
        if verbose:
            even_print('overlap_fac (nominal)' , f'{overlap_fac_nom:0.5f}'  )
            even_print('n_win'                 , f'{n_win:d}'               )
            even_print('win_len'               , f'{win_len:d}'             )
            even_print('overlap'               , f'{overlap:d}'             )
            even_print('overlap_fac'           , f'{overlap_fac:0.5f}'      )
            even_print('n_pad'                 , f'{n_pad:d}'               )
            #even_print('t_win/(δ99/uτ)'        , '%0.3f [-]'%t_eddy_per_win )
            print(72*'-')
        
        ## temporal [t] frequency (f) vector (Short Time Fourier Transform)
        freq_full = sp.fft.fftfreq(n=win_len, d=dt)
        fp        = np.where(freq_full>0) ## indices of positive values
        freq      = np.copy(freq_full[fp])
        df        = freq[1] - freq[0]
        nf        = freq.size
        
        data['freq'] = freq
        data['df']   = df
        data['nf']   = nf
        
        if verbose:
            even_print('freq min',f'{freq.min():0.1f} [Hz]')
            even_print('freq max',f'{freq.max():0.1f} [Hz]')
            even_print('df',f'{df:0.1f} [Hz]')
            even_print('nf',f'{nf:d}')
            #period_eddy = (1/freq) / (d99/u_tau)
            #period_plus = (1/freq) / sc_t_in
            #even_print('min : period+ = (1/f)/(νw/uτ²)'     , '%0.5f [-]'%period_plus.min())
            #even_print('max : period_eddy = (1/f)/(δ99/uτ)' , '%0.5f [-]'%period_eddy.max())
            print(72*'-')
        
        ## spatial [x] wavenumber (kx) and wavelength (λx)
        # λx  = u/f
        # kx  = 2·π·f/u
        # λx+ = λx/(ν/uτ)
        # kx+ = (2·π·f/u)·(ν/uτ)
        na = np.newaxis
        
        ## kx = 2·π·f/u --> [y,f]
        #kx = np.copy( 2*np.pi*freq[na,:] / u_avg[:,na] )
        
        ## λx = u/f --> [y,f]
        #lx = np.copy( u_avg[:,na] / freq[na,:] )
        
        #data['kx'] = kx
        #data['lx'] = lx
        
        ## spatial [z] wavenumber (kz) vector
        kz_full = sp.fft.fftfreq(n=nz, d=dz0) * ( 2 * np.pi )
        kzp     = np.where(kz_full>0) ## indices of positive values
        kz      = np.copy(kz_full[kzp])
        dkz     = kz[1] - kz[0]
        nkz     = kz.shape[0]
        
        ## wavenumber vector should be size nz//2-1
        if (nkz!=nz//2-1):
            raise ValueError
        
        data['kz']  = kz
        data['dkz'] = dkz
        data['nkz'] = nkz
        
        if verbose:
            even_print('kz min',f'{kz.min():0.1f} [1/m]')
            even_print('kz max',f'{kz.max():0.1f} [1/m]')
            even_print('dkz',f'{dkz:0.1f} [1/m]')
            even_print('nkz',f'{nkz:d}')
            #print(72*'-')
        
        ## wavelength λz = (2·π)/kz
        lz = np.copy( 2 * np.pi / kz )
        data['lz'] = lz
        
        # ==============================================================
        # prepare buffers, etc.
        # ==============================================================
        
        do_density_weighting = False ## deactivated for now
        
        ## cospectrum pairs
        ## [ str:var1, str:var2 ]
        fft_combis = [
        [ 'u_tau'  , 'u_tau'  ], ## [ uτ  , uτ  ] --> FFT[ uτ′  , uτ′  ]
        #[ 'u_tau'  , 'v_tau'  ], ## [ uτ  , vτ  ] --> FFT[ uτ′  , vτ′  ]
        [ 'u_tau'  , 'w_tau'  ], ## [ uτ  , wτ  ] --> FFT[ uτ′  , wτ′  ]
        [ 'tau_uy' , 'tau_uy' ], ## [ τuy , τuy ] --> FFT[ τuy′ , τuy′ ]
        #[ 'tau_uy' , 'tau_vy' ], ## [ τuy , τvy ] --> FFT[ τuy′ , τvy′ ]
        [ 'tau_uy' , 'tau_wy' ], ## [ τuy , τwy ] --> FFT[ τuy′ , τwy′ ]
        ]
        
        ## generate FFT cospectrum scalar names
        scalars = []
        for fft_combi in fft_combis:
            #s1,s2,do_density_weighting = fft_combi
            s1,s2 = fft_combi
            if do_density_weighting:
                raise NotImplementedError
            else:
                scalars.append(f"{s1.replace('_','')}I_{s2.replace('_','')}I")
        
        scalars_dtypes = [ np.dtype(np.float64) for s in scalars ]
        
        ## generate AVG scalar names
        scalars_Re_avg = []
        #scalars_Fv_avg = []
        for fft_combi in fft_combis:
            #s1,s2,do_density_weighting = fft_combi
            s1,s2 = fft_combi
            if do_density_weighting and ('rho' not in scalars_Re_avg):
                scalars_Re_avg.append('rho')
            if do_density_weighting:
                #if (s1 not in scalars_Fv_avg):
                #    scalars_Fv_avg.append(s1)
                #if (s2 not in scalars_Fv_avg):
                #    scalars_Fv_avg.append(s2)
                raise NotImplementedError
            else:
                if (s1 not in scalars_Re_avg):
                    scalars_Re_avg.append(s1)
                if (s2 not in scalars_Re_avg):
                    scalars_Re_avg.append(s2)
        
        ## numpy formatted arrays: buffers for PSD & other data (rank-local)
        Ekz        = np.zeros(shape=(nir,nkz ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        Ef         = np.zeros(shape=(nir,nf  ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        covariance = np.zeros(shape=(nir,    ) , dtype={'names':scalars, 'formats':scalars_dtypes})
        avg_Re     = np.zeros(shape=(nir,    ) , dtype={'names':scalars_Re_avg, 'formats':[ np.dtype(np.float64) for s in scalars_Re_avg ]})
        #avg_Fv     = np.zeros(shape=(nir,    ) , dtype={'names':scalars_Fv_avg, 'formats':[ np.dtype(np.float64) for s in scalars_Fv_avg ]})
        
        if verbose:
            even_print('n turb spectrum scalar combinations' , '%i'%(len(fft_combis),))
            print(72*'-')
        
        ## no window for [z]
        window_z = np.ones(nz, dtype=np.float64)
        sum_sqrt_win_z = np.sum(np.sqrt(window_z))
        if verbose:
            #even_print('sum(sqrt(window_z))'     , '%0.5f'%(sum_sqrt_win_z,))
            even_print('sum(sqrt(window_z)) / nz', '%0.5f'%(sum_sqrt_win_z/nz,))
        
        ## window function for [t]
        if (window_type=='tukey'):
            window_t = sp.signal.windows.tukey(win_len,alpha=0.5,sym=False) ## α=0:rectangular, α=1:Hann
        elif (window_type=='hann'):
            window_t = sp.signal.windows.hann(win_len,sym=False)
        elif (window_type is None):
            window_t = np.ones(win_len, dtype=np.float64)
        else:
            raise ValueError
        
        if verbose:
            even_print('window type [t]', '\'%s\''%str(window_type))
        
        ## sum of sqrt of window: needed for normalization
        sum_sqrt_win_t = np.sum(np.sqrt(window_t))
        if verbose:
            #even_print('sum(sqrt(window_t))'          , '%0.5f'%(sum_sqrt_win_t,))
            even_print('sum(sqrt(window_t)) / win_len', '%0.5f'%(sum_sqrt_win_t/win_len,))
        
        # ==============================================================
        # main loop
        # ==============================================================
        
        if verbose:
            progress_bar = tqdm(
                total=len(fft_combis)*nir,
                ncols=100,
                desc='fft',
                smoothing=0.,
                leave=False,
                file=sys.stdout,
                bar_format="{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}\n\033[F\r",
                ascii="░█",
                )
        
        for cci,cc in enumerate(fft_combis): ## fft pairs
            
            if verbose: tqdm.write(72*'-')
            
            scalar_L, scalar_R = cc
            
            msg = f'FFT[{scalar_L}′,{scalar_R}′]'
            if verbose:
                tqdm.write(even_print('computing',msg,s=True,))
            
            dset_L = self[f'data/{scalar_L}']
            dset_R = self[f'data/{scalar_R}']
            data_gb_1x = self.n_ranks * 1 * self.nj * self.nt * dset_L.dtype.itemsize / 1024**3
            
            scalar = scalars[cci]
            
            ## assert scalar name
            scalar_ = f"{scalar_L.replace('_','')}I_{scalar_R.replace('_','')}I"
            if (scalar != scalar_):
                raise ValueError
            
            ## assert scalar name
            if do_density_weighting:
                #if (f'r{scalar_L}II_r{scalar_R}II' != scalar ):
                #    raise ValueError
                raise NotImplementedError
            else:
                if (f"{scalar_L.replace('_','')}I_{scalar_R.replace('_','')}I" != scalar ):
                    raise ValueError
            
            ## [x] loop (rank-local)
            for ii in range(ri1,ri2):
                
                iii = ii - ri1
                
                data_L = np.zeros( (nz,nt) , dtype=np.float64 )
                data_R = np.zeros( (nz,nt) , dtype=np.float64 )
                
                self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## read data L
                n_scalars_read = 1 ## initialize
                scalar_str = scalar_L ## initialize
                with dset_L.collective:
                    data_L[:,:] = np.copy( dset_L[ii,:,:] ).astype(np.float64)
                self.comm.Barrier()
                
                ## read data R (if != data L)
                if (scalar_L==scalar_R):
                    data_R[:,:] = np.copy( data_L )
                else:
                    n_scalars_read += 1
                    scalar_str += f',{scalar_R}'
                    with dset_R.collective:
                        data_R[:,:] = np.copy( dset_R[ii,:,:] ).astype(np.float64)
                    self.comm.Barrier()
                
                self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = data_gb_1x * n_scalars_read
                
                if verbose:
                    tqdm.write(even_print(f'read: {scalar_str}', f'{data_gb:0.3f} [GB]  {t_delta:0.3f} [s]  {data_gb/t_delta:0.3f} [GB/s]', s=True))
                
                ## data_L and data_R shape should be (nz,nt)
                if ( data_L.shape != (nz,nt) ) or ( data_R.shape != (nz,nt) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                # === redimensionalize
                
                if scalar_L in ['tau_uy','tau_vy','tau_wy',]:
                    data_L *= rho_inf * U_inf**2
                elif scalar_L in ['u_tau','v_tau','w_tau',]:
                    data_L *= U_inf
                else:
                    raise ValueError
                
                if scalar_R in ['tau_uy','tau_vy','tau_wy',]:
                    data_R *= rho_inf * U_inf**2
                elif scalar_R in ['u_tau','v_tau','w_tau',]:
                    data_R *= U_inf
                else:
                    raise ValueError
                
                ## data_L and data_R shape should be (nz,nt)
                if ( data_L.shape != (nz,nt) ) or ( data_R.shape != (nz,nt) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                # === compute mean-removed data
                
                ## avg(□) or avg(ρ·□)/avg(ρ) in [t]
                if do_density_weighting:
                    #rho_avg     = np.mean(        rho , axis=-1, dtype=np.float64, keepdims=True)
                    #data_L_avg  = np.mean( rho*data_L , axis=-1, dtype=np.float64, keepdims=True)
                    #data_L_avg /= rho_avg
                    #data_R_avg  = np.mean( rho*data_R , axis=-1, dtype=np.float64, keepdims=True)
                    #data_R_avg /= rho_avg
                    raise NotImplementedError
                else:
                    data_L_avg = np.mean( data_L , axis=-1, dtype=np.float64, keepdims=True) ## (nz,1)
                    data_R_avg = np.mean( data_R , axis=-1, dtype=np.float64, keepdims=True) ## (nz,1)
                
                ## data_L_avg and data_R_avg shape should be (nz,1)
                if ( data_L_avg.shape != (nz,1) ) or ( data_R_avg.shape != (nz,1) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                ## Reynolds prime □′ or Favre prime □″ --> shape (nz,nt)
                data_L -= data_L_avg
                data_R -= data_R_avg
                
                ## data_L and data_R shape should be (nz,nt)
                if ( data_L.shape != (nz,nt) ) or ( data_R.shape != (nz,nt) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                ## assert stationarity / definition averaging
                ## avg(□′)==0 or avg(ρ·□″)==0
                if do_density_weighting:
                    #a_ = np.mean(rho*data_L, axis=-1, dtype=np.float64, keepdims=True)
                    #b_ = np.mean(rho*data_R, axis=-1, dtype=np.float64, keepdims=True)
                    raise NotImplementedError
                else:
                    a_ = np.mean(data_L, axis=-1, dtype=np.float64, keepdims=True) ## average in [t] --> (nz,1)
                    b_ = np.mean(data_R, axis=-1, dtype=np.float64, keepdims=True)
                if not np.allclose( a_, np.zeros_like(a_), atol=1e-6 ) or not np.allclose( b_, np.zeros_like(b_), atol=1e-6 ):
                    print(f'rank {self.rank:d}: avg(□′)!=0 or avg(ρ·□″)!=0')
                    self.comm.Abort(1)
                
                ## covariance: <□′·□′> OR <ρ□″·ρ□″> --> note that this is NOT the typical Favre <ρ·□″□″>
                if do_density_weighting:
                    #covariance_ = np.mean( rho*data_L * rho*data_R , axis=-1 , dtype=np.float64, keepdims=True)
                    raise NotImplementedError
                else:
                    covariance_ = np.mean( data_L*data_R , axis=-1 , dtype=np.float64, keepdims=True) ## average in [t] --> (nz,1)
                
                
                if ( covariance_.shape != (nz,1) ):
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                
                ## write this chunk/scalar's covariance to covariance buffer
                ## avg over [z,t] --> np.float64()
                covariance[scalar][iii] = np.mean( covariance_ , axis=(0,1) , dtype=np.float64)
                
                ## write (rank-local) 1D [x] average
                if do_density_weighting:
                    #avg_Fv[scalar_L][iii] = np.mean( data_L_avg , axis=(0,1) , dtype=np.float64) )
                    #avg_Fv[scalar_R][iii] = np.mean( data_R_avg , axis=(0,1) , dtype=np.float64) )
                    #avg_Re['rho'][iii]    = np.mean( rho_avg    , axis=(0,1) , dtype=np.float64) )
                    raise ValueError
                else:
                    avg_Re[scalar_L][iii] = np.mean( data_L_avg , axis=(0,1) , dtype=np.float64)
                    avg_Re[scalar_R][iii] = np.mean( data_R_avg , axis=(0,1) , dtype=np.float64)
                
                # ===============================================================================
                # At this point you have 4D [x,y,z,t] [□′,□′] or [ρ·□″,ρ·□″] data
                # ===============================================================================
                
                ## do [z] FFT for every [t]
                Ekz_buf = np.zeros((nt,nkz), dtype=np.float64)
                for ti in range(nt):
                    uL = np.copy( data_L[:,ti] )
                    uR = np.copy( data_R[:,ti] )
                    A1 = sp.fft.fft( uL * window_z )[kzp] / sum_sqrt_win_z
                    A2 = sp.fft.fft( uR * window_z )[kzp] / sum_sqrt_win_z
                    Ekz_buf[ti,:] = 2. * np.real(A1*np.conj(A2)) / dkz
                
                Ekz[scalar][iii,:] = np.mean(Ekz_buf, axis=0, dtype=np.float64) ## mean across [t] --> (nkz,)
                
                ## do [t] FFT for every [z]
                Ef_buf = np.zeros((nz,nf), dtype=np.float64)
                #for zi in range(self.nj):
                for zi in range(nz):
                    uL = np.copy( data_L[zi,:] )
                    uR = np.copy( data_R[zi,:] )
                    
                    uL_, nw, n_pad = get_overlapping_windows(uL, win_len, overlap)
                    uR_, nw, n_pad = get_overlapping_windows(uR, win_len, overlap)
                    
                    ## STFT buffer
                    ## compute fft for each overlapped [t] window segment
                    Ef_buf_win = np.zeros((nw,nf), dtype=np.float64)
                    for wi in range(nw):
                        A1 = sp.fft.fft( uL_[wi,:] * window_t )[fp] / sum_sqrt_win_t
                        A2 = sp.fft.fft( uR_[wi,:] * window_t )[fp] / sum_sqrt_win_t
                        Ef_buf_win[wi,:] = 2. * np.real(A1*np.conj(A2)) / df
                    Ef_buf[zi,:] = np.mean(Ef_buf_win, axis=0, dtype=np.float64) ## mean across windows --> (nf,)
                
                Ef[scalar][iii,:] = np.mean(Ef_buf, axis=0, dtype=np.float64) ## mean across [z] --> (nf,)
                
                self.comm.Barrier() ## [x] loop ('ii' within this rank's range)
                if verbose: progress_bar.update()
        
        self.comm.Barrier()
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        # ==============================================================
        # write HDF5 (.h5) file
        # ==============================================================
        
        ## open on rank 0 and write attributes, dimensions, etc.
        if (self.rank==0):
            with h5py.File(fn_h5_fft, 'w') as hfw:
                
                ## write floats,ints as top-level attributes
                for key,val in data.items():
                    if isinstance(data[key], (int,np.int32,np.int64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], (float,np.float32,np.float64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], np.ndarray):
                        pass
                    else:
                        print(f'key {key} is type {str(type(data[key]))}')
                        self.comm.Abort(1)
                
                ## write numpy arrays
                hfw.create_dataset( 'dims/x'    , data=x    ) ## [m]
                hfw.create_dataset( 'dims/z'    , data=z    ) ## [m]
                hfw.create_dataset( 'dims/t'    , data=t    ) ## [s]
                hfw.create_dataset( 'dims/freq' , data=freq ) ## [1/s] | [Hz]
                hfw.create_dataset( 'dims/kz'   , data=kz   ) ## [1/m]
                hfw.create_dataset( 'dims/lz'   , data=lz   ) ## [m]
        
        self.comm.Barrier()
        time.sleep(2)
        self.comm.Barrier()
        
        with h5py.File(fn_h5_fft, 'a', driver='mpio', comm=self.comm) as hfw:
            
            ## initialize datasets : covariance,Ekz,Ef
            for scalar in scalars:
                hfw.create_dataset( f'covariance/{scalar}' , shape=(nx,) , dtype=np.float64 , chunks=(1,) )
                self.comm.Barrier()
                hfw.create_dataset( f'Ekz/{scalar}' , shape=(nx,nkz) , dtype=np.float64 , chunks=(1,nkz) )
                self.comm.Barrier()
                hfw.create_dataset( f'Ef/{scalar}' , shape=(nx,nf ) , dtype=np.float64 , chunks=(1,nf) )
                self.comm.Barrier()
            
            ## initialize datasets : 1D [x] mean
            for scalar in avg_Re.dtype.names:
                hfw.create_dataset( f'avg/Re/{scalar}', shape=(nx,), dtype=np.float64, chunks=(1,) )
                self.comm.Barrier()
            
            ## collectively write covariance,Ekz,Ef
            for scalar in scalars:
                
                dset = hfw[f'covariance/{scalar}']
                with dset.collective:
                    dset[ri1:ri2] = covariance[scalar][:]
                self.comm.Barrier()
                
                dset = hfw[f'Ekz/{scalar}']
                with dset.collective:
                    dset[ri1:ri2,:] = Ekz[scalar][:,:]
                self.comm.Barrier()
                
                dset = hfw[f'Ef/{scalar}']
                with dset.collective:
                    dset[ri1:ri2,:] = Ef[scalar][:,:]
                self.comm.Barrier()
            
            ## collectively write 1D [y] avgs
            for scalar in avg_Re.dtype.names:
                
                dset = hfw[f'avg/Re/{scalar}']
                with dset.collective:
                    dset[ri1:ri2] = avg_Re[scalar][:]
                self.comm.Barrier()
        
        self.comm.Barrier()
        
        ## report file
        if verbose:
            even_print( os.path.basename(fn_h5_fft), f'{(os.path.getsize(fn_h5_fft)/1024**2):0.2f} [MB]' )
            print(72*'-')
        
        ## report file contents
        self.comm.Barrier()
        if (self.rank==0):
            with h5py.File(fn_h5_fft,'r') as hfr:
                h5_print_contents(hfr)
        self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : spd.calc_spectrum_wall() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def calc_statistics_wall(self, **kwargs):
        '''
        calculate statistics for an unsteady wall measurement
          which has dimensions (nx,1,nz)
        - mean
        - covariance
        - skewness, kurtosis
        - probability distribution function (PDF)
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        if verbose: print('\n'+'spd.calc_statistics_wall()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        if not self.usingmpi:
            raise NotImplementedError('function is not implemented for non-MPI usage')
        
        h5py_is_mpi_build = h5py.h5.get_config().mpi
        if not h5py_is_mpi_build:
            if verbose: print('h5py was not compiled for parallel usage! exiting.')
            sys.exit(1)
        
        ri = kwargs.get('ri',1)
        rj = kwargs.get('rj',1)
        rt = kwargs.get('rt',1)
        
        #n_threads = kwargs.get('n_threads',1)
        try:
            n_threads = int(os.environ.get('OMP_NUM_THREADS'))
        except TypeError: ## not set
            n_threads = os.cpu_count()
        
        fn_h5_stats = kwargs.get('fn_h5_stats',None) ## filename for output HDF5 (.h5) file
        n_bins      = kwargs.get('n_bins',512) ## n bins for histogram (PDF) calculation
        
        ## check data distribution
        if (rj!=1):
            raise AssertionError('rj!=1')
        if (rt!=1):
            raise AssertionError('rt!=1')
        if (ri*rj*rt != self.n_ranks):
            raise AssertionError('ri*rj*rt != self.n_ranks')
        if (ri>self.ni):
            raise AssertionError('ri>self.ni')
        if (self.ni%ri!=0):
            raise AssertionError('ni currently needs to be divisible by the n ranks')
        
        ## distribute data over i/[x]
        ril_ = np.array_split(np.arange(self.ni,dtype=np.int64),min(ri,self.ni))
        ril = [[b[0],b[-1]+1] for b in ril_ ]
        ri1,ri2 = ril[self.rank]
        nir = ri2 - ri1
        
        ## output filename : HDF5 (.h5)
        if (fn_h5_stats is None): ## automatically determine name
            fname_path = os.path.dirname(self.fname)
            fname_base = os.path.basename(self.fname)
            fname_root, fname_ext = os.path.splitext(fname_base)
            #fname_root = re.findall(r'io\S+_mpi_[0-9]+', fname_root)[0]
            fname_stats_h5_base = fname_root+'_stats.h5'
            fn_h5_stats = str(PurePosixPath(fname_path, fname_stats_h5_base))
        if (Path(fn_h5_stats).suffix != '.h5'):
            raise ValueError(f"fn_h5_stats='{str(fn_h5_stats)}' must end in .h5")
        if os.path.isfile(fn_h5_stats):
            if (fn_h5_stats == self.fname):
                raise ValueError(f"fn_h5_stats='{str(fn_h5_stats)}' cannot be same as input filename.")
        
        if verbose: even_print( 'fn_h5'        , self.fname   )
        if verbose: even_print( 'fn_h5_stats'  , fn_h5_stats  )
        if verbose: print(72*'-')
        if verbose: even_print( 'ni' , f'{self.ni:d}' )
        if verbose: even_print( 'nj' , f'{self.nj:d}' )
        if verbose: even_print( 'nt' , f'{self.nt:d}' )
        if verbose: print(72*'-')
        self.comm.Barrier()
        
        # ===
        
        ## the data dictionary to be pickled or written to .h5 later
        data = {}
        
        ## freestream data
        lchar = self.lchar     ; data['lchar']   = self.lchar
        U_inf = self.U_inf     ; data['U_inf']   = self.U_inf
        rho_inf = self.rho_inf ; data['rho_inf'] = self.rho_inf
        T_inf = self.T_inf     ; data['T_inf']   = self.T_inf
        mu_inf = self.mu_inf   ; data['mu_inf']  = self.mu_inf
        data['p_inf'] = self.p_inf
        data['Ma']    = self.Ma
        data['Pr']    = self.Pr
        
        ## read in 1D coordinate arrays & re-dimensionalize
        x = np.copy( self['dims/x'][()] * self.lchar )
        z = np.copy( self['dims/z'][()] * self.lchar )
        t = np.copy( self['dims/t'][()] * self.tchar )
        data['x'] = x
        data['z'] = z
        data['t'] = t
        
        nx = self.ni
        ni = self.ni
        nz = self.nj
        nj = self.nj
        data['ni'] = ni
        data['nx'] = nx
        data['nj'] = nj
        data['nz'] = nz
        
        nt = self.nt
        data['nt'] = nt
        
        ## assert constant Δz
        dz0 = np.diff(z)[0]
        if not np.all(np.isclose(np.diff(z), dz0, rtol=1e-6)):
            raise NotImplementedError('Δz not constant')
        
        ## get Δt, dimensional [s]
        if hasattr(self,'dt'):
            dt = self.dt * self.tchar
            np.testing.assert_allclose(dt, t[1]-t[0], rtol=1e-12, atol=1e-12)
        else:
            dt = t[1] - t[0] ## already dimensional
        
        if hasattr(self,'duration'):
            t_meas = self.duration * self.tchar
            np.testing.assert_allclose(t_meas, t.max()-t.min(), rtol=1e-12, atol=1e-12)
        else:
            t_meas = t[-1] - t[0] ## already dimensional
            self.duration = t_meas / self.tchar ## non-dimensionalize for attribute
        
        zrange = z.max() - z.min()
        
        data['t'] = t
        data['dt'] = dt
        data['dz'] = dz0
        data['zrange'] = zrange
        
        if verbose: even_print( 'Δt/tchar'       , f'{dt/self.tchar:0.8f}' )
        if verbose: even_print( 'Δt'             , f'{dt:0.3e} [s]'        )
        if verbose: even_print( 'duration/tchar' , f'{self.duration:0.1f}' )
        if verbose: even_print( 'duration'       , f'{self.duration*self.tchar:0.3e} [s]' )
        if verbose: print(72*'-')
        
        ## report
        if verbose:
            even_print('Δt'     , f'{dt    :0.5e} [s]' )
            even_print('t_meas' , f'{t_meas:0.5e} [s]' )
            even_print('Δz'     , f'{dz0   :0.5e} [m]' )
            even_print('zrange' , f'{zrange:0.5e} [m]' )
            #print(72*'-')
        
        # ==============================================================
        # prepare buffers, maps, etc.
        # ==============================================================
        
        ## key    = str:scalar name
        ## value  = tuple:recipe,
        ##          bool:do_mean,
        ##          bool:do_pdf,
        ##          bool:do_skew_kurt
        ##
        ## recipe elements:
        ##    - if tuple : ( str:scalar , bool:ρ weighting ) --> always mean-removed
        ##    - if str   : str:scalar
        
        scalars_dict = {
            
            'u_tau'     : [ ( 'u_tau',                  ) ,  True, True, False ], ## uτ
            'utauI'     : [ (          ('u_tau',False), ) , False, True,  True ], ## uτ′
            'r_utauII'  : [ (   'rho', ('u_tau',True ), ) , False, True,  True ], ## ρ·uτ″
            
            'w_tau'     : [ ( 'w_tau',                  ) ,  True, True, False ], ## wτ
            'wtauI'     : [ (          ('w_tau',False), ) , False, True,  True ], ## wτ′
            'r_wtauII'  : [ (   'rho', ('w_tau',True ), ) , False, True,  True ], ## ρ·wτ″
            
            'utauIutauI' : [ ( ('u_tau',False), ('u_tau',False) ), True, True, False ], ## uτ′uτ′
            'utauIvtauI' : [ ( ('u_tau',False), ('v_tau',False) ), True, True, False ], ## uτ′vτ′
            'utauIwtauI' : [ ( ('u_tau',False), ('w_tau',False) ), True, True, False ], ## uτ′wτ′
            
            'tau_uy'       : [ (  'tau_uy',                          ) ,  True, True, False ], ## τuy
            'tauuyI'       : [ ( ('tau_uy',False) ,                  ) , False, True,  True ], ## τ′uy
            'tauuyItauuyI' : [ ( ('tau_uy',False) , ('tau_uy',False) ) ,  True, True, False ], ## τ′uy·τ′uy
            
            'tau_wy'       : [ (  'tau_wy',                          ) ,  True, True, False ], ## τwy
            'tauwyI'       : [ ( ('tau_wy',False) ,                  ) , False, True,  True ], ## τ′wy
            'tauwyItauwyI' : [ ( ('tau_wy',False) , ('tau_wy',False) ) ,  True, True, False ], ## τ′wy·τ′wy
            
            'TITI'     : [ (        ('T',False), ('T',False) ) , True, True, False ], ## T′T′
            'TIITII'   : [ (        ('T',True ), ('T',True ) ) , True, True, False ], ## T″T″
            'r_TIITII' : [ ( 'rho', ('T',True ), ('T',True ) ) , True, True, False ], ## ρ·T″T″
            
            'rho'      : [ ( 'rho',                ) ,  True, True, False, ], ## ρ
            'rhoI'     : [ (        ('rho',False), ) , False, True,  True, ], ## ρ′
            
            'p'        : [ ( 'p',                ) ,  True, True, False, ], ## p
            'pI'       : [ (        ('p',False), ) , False, True,  True, ], ## p′
            
            'T'        : [ ( 'T',                ) ,  True, True, False, ], ## T
            'TI'       : [ (        ('T',False), ) , False, True,  True, ], ## T′
            'TII'      : [ (        ('T',True ), ) , False, True, False, ], ## T″
            'r_TII'    : [ ( 'rho', ('T',True ), ) , False, True,  True, ], ## ρ·T″
            
            }
        
        dtype_unsteady = np.float64
        float_bytes = 8
        
        scalars_avg  = []
        scalars_pdf  = []
        scalars_hos_ = []
        for s,ss in scalars_dict.items():
            recipe, do_mean, do_pdf, do_skew_kurt = ss
            if do_mean:
                scalars_avg.append(s)
            if do_pdf:
                scalars_pdf.append(s)
            if do_skew_kurt:
                scalars_hos_.append(s)
        
        scalars_hos=[]
        for s_ in ['skew','kurt']:
            for ss_ in scalars_hos_:
                scalars_hos.append(f'{ss_}_{s_}')
        
        ## rank-local buffsres
        data_avg  = np.zeros(shape=(nir,),          dtype={'names':scalars_avg, 'formats':[ np.float64 for sss in scalars_avg ]})
        data_bins = np.zeros(shape=(nir,n_bins+1),  dtype={'names':scalars_pdf, 'formats':[ np.float64 for sss in scalars_pdf ]})
        data_pdf  = np.zeros(shape=(nir,n_bins),    dtype={'names':scalars_pdf, 'formats':[ np.float64 for sss in scalars_pdf ]})
        data_hos  = np.zeros(shape=(nir,),          dtype={'names':scalars_hos, 'formats':[ np.float64 for sss in scalars_hos ]})
        
        # ==============================================================
        # main loop
        # ==============================================================
        
        if verbose:
            progress_bar = tqdm(
                total=len(scalars_dict)*nir,
                ncols=100,
                desc='statistics',
                smoothing=0.,
                leave=False,
                file=sys.stdout,
                bar_format="{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}\n\033[F\r",
                #ascii="░▒▓█",
                ascii="░█",
                )
        
        for s,ss in scalars_dict.items(): ## e.g. where s='r_uIIvII' & ss=[ ( 'rho', ('u',True ), ('v',True ) ), True,True,False ]
            
            if verbose: tqdm.write(72*'-')
            
            recipe, do_mean, do_pdf, do_skew_kurt = ss
            
            if verbose:
                tqdm.write(even_print('computing',s,s=True,))
            
            ## should ρ be read?
            read_rho = False
            for s_ in recipe:
                if isinstance(s_, str) and (s_=='rho'):
                    read_rho = True
                elif isinstance(s_, str) and (s_!='rho'):
                    pass
                elif isinstance(s_, tuple):
                    if not isinstance(s_[1], bool):
                        raise ValueError
                    if s_[1]: ## i.e. density-weighted
                        read_rho = True
                else:
                    raise ValueError
            
            ## [x] loop (rank-local)
            for ii in range(ri1,ri2):
                
                iii = ii - ri1
                
                ## read ρ
                if read_rho:
                    
                    dset = self[f'data/rho']
                    self.comm.Barrier()
                    t_start = timeit.default_timer()
                    if self.usingmpi:
                        with dset.collective:
                            rho = np.copy( dset[ii,:,:] )
                    else:
                        rho = np.copy( dset[ii,:,:] )
                    self.comm.Barrier()
                    t_delta = timeit.default_timer() - t_start
                    data_gb = self.n_ranks * 1 * self.nj * self.nt * dset.dtype.itemsize / 1024**3
                    if verbose:
                        tqdm.write(even_print(f'read: ρ', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                    
                    rho = rho.astype(np.float64) ## cast to double
                    
                    ## re-dimensionalize
                    rho *= self.rho_inf
                    
                    if ( rho.shape != (nj,nt) ):
                        raise ValueError
                        print(f'rank {self.rank:d}: shape violation')
                        self.comm.Abort(1)
                    
                    ## ρ mean in [t] --> leave (j,1)
                    rho_avg = np.mean(rho, axis=-1, dtype=np.float64, keepdims=True)
                
                else:
                    rho     = None ; del rho
                    rho_avg = None ; del rho_avg
                
                ## product buffer for multiplication --> !!! notices ones() here and not zeros() !!!
                #data_accum = np.ones(shape=(ni,nj,nt), dtype=dtype_unsteady) ## chunk range context
                data_accum = np.ones(shape=(nj,nt), dtype=dtype_unsteady) ## chunk range context
                
                ## read unsteady scalar data, remove mean, <density weight>, multiply
                for sss in recipe:
                    
                    if isinstance(sss, str) and (sss=='rho'): ## ρ
                        
                        ## multiply product accumulator, ρ was already read and is already dimensional
                        data_accum *= rho
                    
                    elif isinstance(sss, str) and (sss!='rho'): ## scalar which will NOT be mean-removed
                        
                        ## read
                        dset = self[f'data/{sss}']
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                data_X = np.copy( dset[ii,:,:] )
                        else:
                            data_X = np.copy( dset[ii,:,:] )
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = self.n_ranks * 1 * self.nj * self.nt * dset.dtype.itemsize / 1024**3
                        if verbose:
                            tqdm.write(even_print(f'read: {sss}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        data_X = data_X.astype(np.float64) ## cast to double
                        
                        if ( data_X.shape != (nj,nt) ):
                            raise ValueError
                            print(f'rank {self.rank:d}: shape violation')
                            self.comm.Abort(1)
                        
                        ## redimensionalize
                        if sss in ['tau_uy','tau_vy','tau_wy']:
                            data_X *= self.rho_inf * self.U_inf**2
                        elif sss in ['u_tau','v_tau','w_tau']:
                            data_X *= self.U_inf
                        elif sss in ['T',]:
                            data_X *= self.T_inf
                        elif sss in ['rho',]:
                            data_X *= self.rho_inf
                        elif sss in ['p',]:
                            data_X *= self.rho_inf * self.U_inf**2
                        else:
                            raise ValueError(f"condition needed for redimensionalizing '{str(sss)}'")
                        
                        ## MULTIPLY product accumulator
                        data_accum *= data_X
                    
                    elif isinstance(sss, tuple): ## scalar which WILL be mean-removed (with- or without ρ-weighting)
                        
                        if (len(sss)!=2):
                            raise ValueError
                        
                        sn, do_density_weighting = sss ## e.g. ('u',True)
                        
                        ## read
                        dset = self[f'data/{sn}']
                        self.comm.Barrier()
                        t_start = timeit.default_timer()
                        if self.usingmpi:
                            with dset.collective:
                                data_X = np.copy( dset[ii,:,:] )
                        else:
                            data_X = np.copy( dset[ii,:,:] )
                        self.comm.Barrier()
                        t_delta = timeit.default_timer() - t_start
                        data_gb = self.n_ranks * 1 * self.nj * self.nt * dset.dtype.itemsize / 1024**3
                        if verbose:
                            tqdm.write(even_print(f'read: {sn}', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)), s=True))
                        
                        data_X = data_X.astype(np.float64) ## cast to double
                        
                        ## redimensionalize
                        if sn in ['tau_uy','tau_vy','tau_wy']:
                            data_X *= self.rho_inf * self.U_inf**2
                        elif sn in ['u_tau','v_tau','w_tau']:
                            data_X *= self.U_inf
                        elif sn in ['T',]:
                            data_X *= self.T_inf
                        elif sn in ['rho',]:
                            data_X *= self.rho_inf
                        elif sn in ['p',]:
                            data_X *= self.rho_inf * self.U_inf**2
                        else:
                            raise ValueError(f"condition needed for redimensionalizing '{str(sn)}'")
                        
                        ## avg(□) or avg(ρ·□)/avg(ρ)
                        if do_density_weighting:
                            data_X_mean = np.mean( rho*data_X , axis=-1, dtype=np.float64, keepdims=True) ## (nz,1)
                            data_X_mean /= rho_avg
                        else:
                            data_X_mean = np.mean( data_X , axis=-1, dtype=np.float64, keepdims=True) ## (nz,1)
                        
                        ## Reynolds prime □′ or Favre prime □″
                        data_X -= data_X_mean
                        
                        ## assert avg(□′)==0 or avg(ρ·□″)==0
                        if do_density_weighting:
                            a_ = np.mean(rho*data_X, axis=-1, dtype=np.float64, keepdims=True)
                        else:
                            a_ = np.mean(data_X, axis=-1, dtype=np.float64, keepdims=True)
                        if not np.allclose( a_, np.zeros_like(a_), atol=1e-3 ):
                            print(f'rank {self.rank:d}: avg(□′)!=0 or avg(ρ·□″)!=0')
                            self.comm.Abort(1)
                        
                        ## MULTIPLY product accumulator by □′ | □″
                        data_accum *= data_X
                    
                    else:
                        raise ValueError
                
                # ===============================================================================
                # At this point you have the UNSTEADY 2D [j,t] data according to 'recipe'
                # ===============================================================================
                
                #xiA = cx1 - rx1
                #xiB = cx2 - rx1
                
                self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## mean in [t] --> leave [j,1]
                d_mean = np.mean( data_accum, axis=-1, keepdims=True, dtype=np.float64)
                #if ( d_mean.shape != (ni,nj,1,) ):
                if ( d_mean.shape != (nj,1) ):
                    raise ValueError
                    print(f'rank {self.rank:d}: shape violation')
                    self.comm.Abort(1)
                
                if do_mean: ## mean in [j] --> leave float
                    data_avg[s][iii] = np.mean( d_mean, axis=(0,1), dtype=np.float64)
                
                if do_pdf:
                    
                    d_ = np.copy( data_accum.ravel() )
                    if ( d_.shape != (nj*nt,) ):
                        raise ValueError
                        print(f'rank {self.rank:d}: shape violation')
                        self.comm.Abort(1)
                    pdf_ , bin_edges_ = np.histogram( d_ , bins=n_bins , density=True )
                    data_bins[s][iii,:] = bin_edges_
                    data_pdf[s][iii,:]  = pdf_
                
                if do_skew_kurt:
                    
                    d_ = np.copy( data_accum.ravel() ) ## □′ or □″, although mean ==0 is not checked
                    if ( d_.shape != (nj*nt,) ):
                        raise ValueError
                        print(f'rank {self.rank:d}: shape violation')
                        self.comm.Abort(1)
                    
                    ## ## is it interpretable as a Reynolds mean-removed quantity? (□′)
                    ## interpretable_as_Re_mean = False
                    ## a_ = np.mean(d_, axis=3, dtype=np.float64, keepdims=True) ## mean in [t]
                    ## np.testing.assert_equal( a_.shape, (nx,1,nz,1,) )
                    ## if np.allclose(a_, np.zeros_like(a_), atol=1e-10):
                    ##     interpretable_as_Re_mean = True
                    ## 
                    ## ## is it interpretable as a Favre mean-removed quantity? (□″)
                    ## interpretable_as_Fv_mean = False
                    ## if read_rho and not interpretable_as_Re_mean:
                    ##     #rho_ = np.copy( rho[:,yii,:,:] )
                    ##     rho_ = np.zeros(shape=(nx,1,nz,nt), dtype=dtype_unsteady)
                    ##     rho_[:,:,:,:] = rho[:,yii,:,:][:,np.newaxis,:,:] ## [x,1,z,t]
                    ##     np.testing.assert_equal( rho_.shape, (nx,1,nz,nt) )
                    ##     
                    ##     a_ = np.mean(d_*rho_, axis=3, dtype=np.float64, keepdims=True) ## mean in [t]
                    ##     np.testing.assert_equal( a_.shape, (nx,1,nz,1,) )
                    ##     if np.allclose(a_, np.zeros_like(a_), atol=1e-10):
                    ##         interpretable_as_Fv_mean = True
                    ## 
                    ## if (not interpretable_as_Re_mean) and (not interpretable_as_Fv_mean):
                    ##     print(f'{s} has non-zero Favre and Reynolds mean, i.e. is not <□′>!=0 and <ρ□″>!=0')
                    ##     self.comm.Abort(1)
                    
                    d_std = np.sqrt( np.mean( d_**2 , dtype=np.float64 ) )
                    
                    if np.isclose(d_std, 0., atol=1e-08):
                        d_skew = 0.
                        d_kurt = 0.
                    else:
                        d_skew = np.mean( d_**3 , dtype=np.float64 ) / d_std**3
                        d_kurt = np.mean( d_**4 , dtype=np.float64 ) / d_std**4
                    
                    data_hos[f'{s}_skew'][iii] = d_skew
                    data_hos[f'{s}_kurt'][iii] = d_kurt
                
                self.comm.Barrier() ## [x] loop ('ii' within this rank's range)
                if verbose: progress_bar.update()
        
        self.comm.Barrier()
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        # ==============================================================
        # write HDF5 (.h5) file
        # ==============================================================
        
        ## open on rank 0 and write attributes, dimensions, etc.
        if (self.rank==0):
            with h5py.File(fn_h5_stats, 'w') as hfw:
                
                ## write floats,ints as top-level attributes
                for key,val in data.items():
                    if isinstance(data[key], (int,np.int32,np.int64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], (float,np.float32,np.float64)):
                        hfw.attrs[key] = val
                    elif isinstance(data[key], np.ndarray):
                        pass
                    else:
                        print(f'key {key} is type {str(type(data[key]))}')
                        self.comm.Abort(1)
                
                ## write numpy arrays
                hfw.create_dataset( 'dims/x' , data=x ) ## [m]
                hfw.create_dataset( 'dims/z' , data=z ) ## [m]
                hfw.create_dataset( 'dims/t' , data=t ) ## [s]
        
        self.comm.Barrier()
        time.sleep(2.)
        self.comm.Barrier()
        
        with h5py.File(fn_h5_stats, 'a', driver='mpio', comm=self.comm) as hfw:
            
            # === initialize datasets
            
            for scalar in scalars_avg:
                hfw.create_dataset( f'avg/{scalar}', shape=(nx,), dtype=np.float64, chunks=(1,) )
                self.comm.Barrier()
            
            for scalar in scalars_pdf:
                hfw.create_dataset( f'bins/{scalar}', shape=(nx,n_bins+1), dtype=np.float64, chunks=(1,n_bins+1) )
                self.comm.Barrier()
            
            for scalar in scalars_pdf:
                hfw.create_dataset( f'pdf/{scalar}', shape=(nx,n_bins), dtype=np.float64, chunks=(1,n_bins) )
                self.comm.Barrier()
            
            for scalar in scalars_hos:
                hfw.create_dataset( f'hos/{scalar}', shape=(nx,), dtype=np.float64, chunks=(1,) )
                self.comm.Barrier()
            
            # === collectively write data
            
            for scalar in scalars_avg:
                dset = hfw[f'avg/{scalar}']
                with dset.collective:
                    dset[ri1:ri2] = data_avg[scalar][:]
                self.comm.Barrier()
            
            for scalar in scalars_pdf:
                dset = hfw[f'bins/{scalar}']
                with dset.collective:
                    dset[ri1:ri2,:] = data_bins[scalar][:,:]
                self.comm.Barrier()
            
            for scalar in scalars_pdf:
                dset = hfw[f'pdf/{scalar}']
                with dset.collective:
                    dset[ri1:ri2,:] = data_pdf[scalar][:,:]
                self.comm.Barrier()
            
            for scalar in scalars_hos:
                dset = hfw[f'hos/{scalar}']
                with dset.collective:
                    dset[ri1:ri2] = data_hos[scalar][:]
                self.comm.Barrier()
        
        ## report file size
        if verbose:
            even_print( os.path.basename(fn_h5_stats), f'{(os.path.getsize(fn_h5_stats)/1024**2):0.2f} [MB]' )
            print(72*'-')
        
        ## report file contents
        self.comm.Barrier()
        if (self.rank==0):
            with h5py.File(fn_h5_stats,'r') as hfr:
                h5_print_contents(hfr)
        self.comm.Barrier()
        
        if verbose: print(72*'-')
        if verbose: print('total time : spd.calc_statistics_wall() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    # === Paraview
    
    def make_xdmf(self, **kwargs):
        '''
        generate an XDMF/XMF2 from SPD for processing with Paraview
        -----
        --> https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if 'dims/quads' not in self:
            raise ValueError('dims/quads not in file')
        if 'dims/pts' not in self:
            raise ValueError('dims/pts not in file')
        
        ## this should be added to spd.get_header()
        dsn = 'dims/quads'
        n_quads,four = self[dsn].shape
        dsn = 'dims/pts'
        n_pts,three = self[dsn].shape
        self.n_quads = n_quads
        self.n_pts   = n_pts
        
        if verbose: print('\n'+'spd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for dsn in ['pts','quads']:
            if (f'dims/{dsn}' in self):
                data = self[f'dims/{dsn}']
                dataset_precision_dict[dsn] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[dsn] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[dsn] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        ## refresh header
        self.get_header(verbose=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')

        # === write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            if not os.path.isfile(fname_xdmf): ## if doesnt exist...
                Path(fname_xdmf).touch() ## touch XDMF file
                perms_h5 = oct(os.stat(self.fname).st_mode)[-3:] ## get permissions of SPD file
                os.chmod(fname_xdmf, int(perms_h5, base=8)) ## change permissions of XDMF file
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
                
                xdmf_str=f'''
                         <Topology TopologyType="Quadrilateral" NumberOfElements="{n_quads:d}">
                             <DataItem Dimensions="{n_quads:d} 4" NumberType="{dataset_numbertype_dict['quads']}" Precision="{dataset_precision_dict['quads']}" Format="HDF">
                                 {self.fname_base}:/dims/quads
                             </DataItem>
                         </Topology>
                         <Geometry GeometryType="XYZ">
                             <DataItem Dimensions="{n_pts:d} 3" NumberType="{dataset_numbertype_dict['pts']}" Precision="{dataset_precision_dict['pts']}" Format="HDF">
                                 {self.fname_base}:/dims/pts
                             </DataItem>
                         </Geometry>
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # ===
                
                xdmf_str='''
                         <!-- ==================== time series ==================== -->
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                # === the time series
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 4*' '))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str = f'''
                                <!-- ==================== ts = {ti:d} ==================== -->
                                '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # ===
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                    
                    # === .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        dset_hf_path = f'data/{scalar}'
                        scalar_name = scalar
                        
                        xdmf_str=f'''
                                 <!-- {scalar} -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem ItemType="HyperSlab" Dimensions="{self.ni:d} {self.nj:d}" Type="HyperSlab">
                                     <DataItem Dimensions="3 3" NumberType="Integer" Format="XML">
                                       {0:<9d} {0:<9d} {ti:d}
                                       {1:<9d} {1:<9d} {1:d}
                                       {self.ni:<9d} {self.nj:<9d} {1:d}
                                     </DataItem>
                                     <DataItem Dimensions="{self.ni:d} {self.nj:d} {self.nt:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                       {fname_base}:/{dset_hf_path}
                                     </DataItem>
                                   </DataItem>
                                 </Attribute>
                                 '''
                        
                        xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 8*' '))
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 6*' '))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(textwrap.indent(textwrap.dedent(xdmf_str.strip('\n')), 0*' '))
        
        if verbose: print('--w-> %s'%fname_xdmf_base)
        return

# meta HDF5 & h5py
# ======================================================================

def h5_chunk_sizer(nxi, **kwargs):
    '''
    solve for HDF5 dataset chunk size
    -----
    nxi --> an iterable (tuple,list,etc.) containing the target shape of the full HDF5 dataset
    constraint --> an iterable with arguments <int> , None , 'full'/-1
    -----
    constraint = (1,None,None,None) ## [t] convention
    constraint = (None,-1,1,-1) ## [y] convention
    '''
    
    size_kb    = kwargs.get('size_kb'    , 2*1024 ) ## target chunk size in [KB] --> default = 2 [MB]
    itemsize   = kwargs.get('itemsize'   , 4      ) ## dtype.itemsize --> default single precision i.e. 4 [B]
    constraint = kwargs.get('constraint' , None   ) ## iterable of nxi constraints --> int,None,'full'/-1
    base       = kwargs.get('base'       , 2      ) ## axis chunk size = ceil[size/(<int>*base)] where <int> is incremented
    
    ## check inputs
    if not hasattr(nxi, '__iter__'):
        raise AssertionError('\'nxi\' must be an iterable')
    if constraint is None:
        constraint = [ None for i in range(len(nxi)) ]
    if not hasattr(constraint, '__iter__'):
        raise AssertionError('\'constraint\' must be an iterable')
    if not (len(nxi)==len(constraint)):
        raise ValueError('nxi and constraint must have same size')
    if not isinstance(base, int):
        raise TypeError('\'base\' must be type int')
    
    if True: ## increment divisor on largest axis, with divisor=<int>*base
        
        nxi_ = [ n for n in nxi ] ## copy the input shape
        div = [ 1 for i in range(len(nxi)) ] ## divisor vector, initialize with int ones
        i_flexible = [ i for i in range(len(nxi)) if constraint[i] is None ] ## list of axes indices which are 'flexible'
        
        while True:
            
            #div_last = div ## no! pointer, not a copy
            div_last = [ div[i] for i in range(len(nxi)) ]
            
            chunks = []
            for i in range(len(nxi)):
                if (constraint[i] is None): ## axis is 'flexible', so divide by divisor
                    cas = max( int(round(nxi_[i]/div[i])) , 1 ) ## chunk axis shape is solved for
                elif (constraint[i] == 'full') or (constraint[i] == -1):
                    cas = nxi_[i] ## chunk axis shape is == dset axis shape
                else:
                    
                    ## this axis is neither 'flexible' nor 'full', so
                    ## assert that the constraint is a positive integer
                    if not isinstance(constraint[i], int):
                        raise ValueError('constraint[%i]=%s is not type int'%(i,str(constraint[i])))
                    if not (constraint[i]>0):
                        raise ValueError('constraint[%i]=%i is not >0'%(i,constraint[i]))
                    
                    cas = constraint[i] ## chunk axis shape is just the constraint
                
                chunks.append(cas)
            
            ## there are no flexible axes --> exit loop
            if len(i_flexible)==0:
                break
            
            ## the current size of a chunk, given 
            chunk_size_kb = np.prod(chunks)*itemsize / 1024.
            # print('chunk size %0.1f [KB]'%chunk_size_kb)
            
            if ( chunk_size_kb <= size_kb ): ## if chunk size is < target, then break
                break
            else: ## otherwise, increase the divisor of the greatest 'flexible' axis
                
                ## get index of (flexible) axis with greatest size
                aa = [ i for i, j in enumerate(chunks) if (constraint[i] is None) ]
                bb = [ j for i, j in enumerate(chunks) if (constraint[i] is None) ]
                i_gt = aa[np.argmax(bb)]
                
                ## get new divisor
                if (div[i_gt]==1) and (base!=1):
                    div[i_gt] = 0
                div[i_gt] += base
            
            # print(f'div = {str(tuple(div))}')
            # print(f'div_last = {str(tuple(div_last))}')
            # print(f'chunks = {str(tuple(chunks))}')
            # print('---')
            
            ## check if in infinite loop (divisor not being updated)
            if (div_last is not None) and (div == div_last):
                raise ValueError('invalid parameters for h5_chunk_sizer() : constraint=%s, size_kb=%i, base=%i'%(str(constraint),size_kb,base))
    
    return tuple(chunks)

def h5_visititems_print_attrs(name, obj):
    '''
    callable for input to h5py.Group.visititems() to print names & attributes
    '''
    n_slashes = name.count('/')
    shift = n_slashes*2*' '
    item_name = name.split('/')[-1]
    
    if isinstance(obj,h5py._hl.dataset.Dataset):
        print(shift + item_name + ' --> shape=%s, dtype=%s'%( str(obj.shape), str(obj.dtype) ) )
    else:
        print(shift + item_name)
    
    ## print attributes
    for key, val in obj.attrs.items():
        try:
            print(shift + 2*' ' + f'{key} = {str(val)} --> dtype={str(val.dtype)}')
        except:
            print(shift + 2*' ' + f'{key} = {str(val)} --> type={str(type(val).__name__)}')

def h5_print_contents(h5filehandle):
    '''
    Print file-level attributes and recursively print names & attributes of all groups and datasets.
    '''
    
    ## file-level attributes
    for key, val in h5filehandle.attrs.items():
        try:
            print(f'{key} = {str(val)} --> dtype={str(val.dtype)}')
        except:
            print(f'{key} = {str(val)} --> type={str(type(val).__name__)}')
    
    def visitor(name, obj):
        n_slashes = name.count('/')
        shift     = n_slashes * 2 * ' '
        item_name = name.split('/')[-1]
        
        if isinstance(obj, h5py._hl.dataset.Dataset):
            print(shift + item_name + ' --> shape=%s, dtype=%s' % (str(obj.shape), str(obj.dtype)))
        else:
            print(shift + item_name)
        
        for key, val in obj.attrs.items():
            try:
                print(shift + 2 * ' ' + f'{key} = {str(val)} --> dtype={str(val.dtype)}')
            except:
                print(shift + 2 * ' ' + f'{key} = {str(val)} --> type={str(type(val).__name__)}')
    
    # Use the visitor function with visititems
    h5filehandle.visititems(visitor)

class h5_visit_container:
    '''
    callable for input to h5py.Group.visit() which stores dataset/group names
    '''
    def __init__(self):
        self.names = []
    def __call__(self, name):
        if (name not in self.names):
            self.names.append(name)

# data container interface class for EAS3 (legacy NS3D format)
# ======================================================================

class eas3:
    '''
    Interface class for EAS3 files (legacy binary NS3D output format)
    - only binary read is supported, no writing
    '''
    
    def __init__(self, fname, **kwargs):
        '''
        initialize class instance
        '''
        self.fname   = fname
        self.verbose = kwargs.get('verbose',True)
        
        if isinstance(fname, str):
            self.f = open(fname,'rb')
        elif isinstance(fname, io.BytesIO):
            self.f = fname
        else:
            raise TypeError('fname should be type str or io.BytesIO')
        
        self.udef = self.get_header()
    
    def __enter__(self):
        '''
        for use with python 'with' statement
        '''
        #print('opening from enter() --> used with statement')
        return self
    
    def __exit__(self, exception_type, exception_value, exception_traceback):
        '''
        for use with python 'with' statement
        '''
        self.f.close()
        
        if exception_type is not None:
            print('\nsafely closed EAS3 due to exception')
            print(72*'-')
            print('exception type : '+str(exception_type))
        if exception_value is not None:
            print('exception_value : '+str(exception_value))
        if exception_traceback is not None:
            print('exception_traceback : '+str(exception_traceback))
        if exception_type is not None:
            print(72*'-')
    
    def close(self):
        '''
        close() passthrough to handle from binary open()
        '''
        self.f.close()
    
    def get_header(self, **kwargs):
        
        ATTRLEN     = kwargs.get('ATTRLEN',10)
        UDEFLEN     = kwargs.get('UDEFLEN',20)
        ChangeGmode = kwargs.get('ChangeGmode',1)
        
        ## Definitions
        EAS2=1; EAS3=2
        IEEES=1; IEEED=2; IEEEQ=3
        EAS3_NO_ATTR=1; EAS3_ALL_ATTR=2
        EAS3_NO_G=1; EAS3_X0DX_G=2; EAS3_UDEF_G=3; EAS3_ALL_G=4; EAS3_FULL_G=5
        EAS3_NO_UDEF=1; EAS3_ALL_UDEF=2; EAS3_INT_UDEF=3
        
        self.IEEES = IEEES
        self.IEEED = IEEED
        self.IEEEQ = IEEEQ
        
        ## Identifier: 20 byte character
        identifier = self.f.read(20).strip()
        
        ## File type: 8 byte integer
        file_type = struct.unpack('!q',self.f.read(8))[0]
        
        ## Accuracy: 8 byte integer
        accuracy = struct.unpack('!q',self.f.read(8))[0]
        self.accuracy = accuracy
        
        if (self.accuracy == self.IEEES):
            self.dtype = np.float32
        elif (self.accuracy == self.IEEED):
            self.dtype = np.float64
        elif (self.accuracy == self.IEEEQ):
            self.dtype = np.float128
        else:
            raise ValueError('precision not identifiable')
        
        ## Array sizes: each 8 byte integer
        nzs   = struct.unpack('!q',self.f.read(8))[0]
        npar  = struct.unpack('!q',self.f.read(8))[0]
        ndim1 = struct.unpack('!q',self.f.read(8))[0]
        ndim2 = struct.unpack('!q',self.f.read(8))[0]
        ndim3 = struct.unpack('!q',self.f.read(8))[0]
        
        self.ndim1 = ndim1
        self.ndim2 = ndim2
        self.ndim3 = ndim3
        self.npar  = npar
        self.nzs   = nzs
        
        ## Attribute mode: 8 byte integer
        attribute_mode = struct.unpack('!q',self.f.read(8))[0]
        
        ## Geometry mode: each 8 byte integer
        gmode_time  = struct.unpack('!q',self.f.read(8))[0]
        gmode_param = struct.unpack('!q',self.f.read(8))[0]
        gmode_dim1  = struct.unpack('!q',self.f.read(8))[0]  
        gmode_dim2  = struct.unpack('!q',self.f.read(8))[0]
        gmode_dim3  = struct.unpack('!q',self.f.read(8))[0]
        
        ## Array sizes for geometry data: each 8 byte integer
        size_time  = struct.unpack('!q',self.f.read(8))[0]
        size_param = struct.unpack('!q',self.f.read(8))[0]
        size_dim1  = struct.unpack('!q',self.f.read(8))[0]
        size_dim2  = struct.unpack('!q',self.f.read(8))[0]
        size_dim3  = struct.unpack('!q',self.f.read(8))[0]
        
        ## Specification of user defined data: 8 byte integer
        udef = struct.unpack('!q',self.f.read(8))[0]
        
        ## Array sizes for used defined data: each 8 byte integer
        udef_char_size = struct.unpack('!q',self.f.read(8))[0]
        udef_int_size  = struct.unpack('!q',self.f.read(8))[0]
        udef_real_size = struct.unpack('!q',self.f.read(8))[0]
        
        ## Time step array: nzs x 8 byte
        time_step = np.zeros(nzs,int)
        for it in range(nzs):
            time_step[it] = struct.unpack('!q',self.f.read(8))[0]
        
        if attribute_mode==EAS3_ALL_ATTR:
            ## Time step attributes
            attr_time = [ self.f.read(ATTRLEN).decode('UTF-8').strip() ]
            for it in range(1,nzs):
                attr_time.append( self.f.read(ATTRLEN).decode('UTF-8').strip() )
            ## Parameter attributes
            attr_param = [ self.f.read(ATTRLEN).decode('UTF-8').strip() ]
            for it in range(1,npar):
                attr_param.append( self.f.read(ATTRLEN).decode('UTF-8').strip() )
            
            # Spatial attributes
            attr_dim1 = self.f.read(ATTRLEN).decode('UTF-8').strip()
            attr_dim2 = self.f.read(ATTRLEN).decode('UTF-8').strip()
            attr_dim3 = self.f.read(ATTRLEN).decode('UTF-8').strip()
        
        ## If geometry mode > EAS3_NO_G for time
        if gmode_time == EAS3_X0DX_G:
            time_data = np.zeros(2)
            for it in range(2):
                time_data[it] = struct.unpack('!d',self.f.read(8))[0]
        elif gmode_time == EAS3_ALL_G:
            time_data = np.zeros(size_time)
            for it in range(size_time):
                time_data[it] = struct.unpack('!d',self.f.read(8))[0]
        else:
            time_data = np.zeros(1)
        
        ## If geometry mode > EAS3_NO_G for parameters
        if gmode_param > EAS3_NO_G:
            param = np.zeros(size_param)
            for it in range(size_param):
                param[it] = struct.unpack('!d',self.f.read(8))[0]
        
        ## If geometry mode > EAS3_NO_G for dimensions 1 to 3
        dim1_data = np.zeros(size_dim1)
        if gmode_dim1 > EAS3_NO_G:
            for it in range(size_dim1):
                dim1_data[it] = struct.unpack('!d',self.f.read(8))[0]
            if abs(dim1_data[0]) < 1e-18: dim1_data[0] = 0.   
        dim2_data = np.zeros(size_dim2)
        if gmode_dim2 > EAS3_NO_G:
            for it in range(size_dim2):
                dim2_data[it] = struct.unpack('!d',self.f.read(8))[0]
            if abs(dim2_data[0]) < 1e-18: dim2_data[0] = 0.
        dim3_data = np.zeros(size_dim3)
        if gmode_dim3 > EAS3_NO_G:
            for it in range(size_dim3):
                dim3_data[it] = struct.unpack('!d',self.f.read(8))[0]
        else: dim3_data = 0.
        
        ## If user-defined data is chosen 
        if udef==EAS3_ALL_UDEF:
            udef_char = []
            for it in range(udef_char_size):
                udef_char.append(self.f.read(UDEFLEN).decode('UTF-8').strip())
            udef_int = np.zeros(udef_int_size,int)
            for it in range(udef_int_size):
                udef_int[it] = struct.unpack('!q',self.f.read(8))[0]
            udef_real = np.zeros(udef_real_size)
            for it in range(udef_real_size):
                udef_real[it] = struct.unpack('!d',self.f.read(8))[0]
        
        ## Option: convert gmode=EAS3_X0DX_G to gmode=EAS3_ALL_G
        if ChangeGmode==1:
            if gmode_dim1==EAS3_X0DX_G:
                dim1_data = np.linspace(dim1_data[0],dim1_data[0]+dim1_data[1]*(ndim1-1), ndim1)
                gmode_dim1=EAS3_ALL_G
            if gmode_dim2==EAS3_X0DX_G:
                dim2_data = np.linspace(dim2_data[0],dim2_data[0]+dim2_data[1]*(ndim2-1), ndim2)
                gmode_dim2=EAS3_ALL_G
            if gmode_dim3==EAS3_X0DX_G:
                dim3_data = np.linspace(dim3_data[0],dim3_data[0]+dim3_data[1]*(ndim3-1), ndim3)
                gmode_dim3=EAS3_ALL_G
            if gmode_time==EAS3_X0DX_G:
                time_data = np.linspace(time_data[0],time_data[0]+time_data[1]*(nzs  -1), nzs  )
                gmode_time=EAS3_ALL_G
        
        # ===
        
        self.attr_param = attr_param
        self.scalars    = attr_param
        self.t          = time_data
        self.nt         = self.t.size
        
        if (attr_dim1=='x'):
            self.x = dim1_data
        elif (attr_dim1=='y'):
            self.y = dim1_data
        elif (attr_dim1=='z'):
            self.z = dim1_data
        else:
            raise ValueError('attr_dim1 = %s not identifiable as any x,y,z'%attr_dim1)
        
        if (attr_dim2=='x'):
            self.x = dim2_data
        elif (attr_dim2=='y'):
            self.y = dim2_data
        elif (attr_dim2=='z'):
            self.z = dim2_data
        else:
            raise ValueError('attr_dim2 = %s not identifiable as any x,y,z'%attr_dim2)
        
        if (attr_dim3=='x'):
            self.x = dim3_data
        elif (attr_dim3=='y'):
            self.y = dim3_data
        elif (attr_dim3=='z'):
            self.z = dim3_data
        else:
            raise ValueError('attr_dim3 = %s not identifiable as any x,y,z'%attr_dim3)
        
        # === transpose order to [x,y,z]
        
        if all([(attr_dim1=='x'),(attr_dim2=='y'),(attr_dim3=='z')]):
            self.axes_transpose_xyz = (0,1,2)
        elif all([(attr_dim1=='y'),(attr_dim2=='x'),(attr_dim3=='z')]):
            self.axes_transpose_xyz = (1,0,2)
        elif all([(attr_dim1=='z'),(attr_dim2=='y'),(attr_dim3=='x')]):
            self.axes_transpose_xyz = (2,1,0)
        elif all([(attr_dim1=='x'),(attr_dim2=='z'),(attr_dim3=='y')]):
            self.axes_transpose_xyz = (0,2,1)
        elif all([(attr_dim1=='y'),(attr_dim2=='z'),(attr_dim3=='x')]):
            self.axes_transpose_xyz = (2,0,1)
        elif all([(attr_dim1=='z'),(attr_dim2=='x'),(attr_dim3=='y')]):
            self.axes_transpose_xyz = (1,2,0)
        else:
            raise ValueError('could not determine axes transpose')
        
        # ===
        
        self.nx  = self.x.size
        self.ny  = self.y.size
        self.nz  = self.z.size
        self.ngp = self.nx*self.ny*self.nz
        
        if self.verbose: print(72*'-')
        if self.verbose: even_print('nx', '%i'%self.nx )
        if self.verbose: even_print('ny', '%i'%self.ny )
        if self.verbose: even_print('nz', '%i'%self.nz )
        if self.verbose: even_print('ngp', '%i'%self.ngp )
        if self.verbose: print(72*'-')
        
        if self.verbose: even_print('x_min', '%0.2f'%self.x.min())
        if self.verbose: even_print('x_max', '%0.2f'%self.x.max())
        if self.verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (self.x[1]-self.x[0]), (self.x[-1]-self.x[-2]) ))
        if self.verbose: even_print('y_min', '%0.2f'%self.y.min())
        if self.verbose: even_print('y_max', '%0.2f'%self.y.max())
        if self.verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (self.y[1]-self.y[0]), (self.y[-1]-self.y[-2]) ))
        if self.verbose: even_print('z_min', '%0.2f'%self.z.min())
        if self.verbose: even_print('z_max', '%0.2f'%self.z.max())        
        if self.verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (self.z[1]-self.z[0]), (self.z[-1]-self.z[-2]) ))
        if self.verbose: print(72*'-'+'\n')
        
        # ===
        
        ## make a python dict from udef vectors
        udef_dict = {}
        for i in range(len(udef_char)):
            if (udef_char[i]!=''):
                if (udef_int[i]!=0):
                    udef_dict[udef_char[i]] = int(udef_int[i])
                elif (udef_real[i]!=0.):
                    udef_dict[udef_char[i]] = float(udef_real[i])
                else:
                    udef_dict[udef_char[i]] = 0.
        
        if self.verbose:
            print('udef from EAS3\n' + 72*'-')
            for key in udef_dict:
                if isinstance(udef_dict[key],float):
                    even_print(key, '%0.8f'%udef_dict[key])
                elif isinstance(udef_dict[key],int):
                    even_print(key, '%i'%udef_dict[key])
                else:
                    #print(type(udef_dict[key]))
                    raise TypeError('udef dict item not float or int')
            print(72*'-'+'\n')
        
        self.Ma    = udef_dict['Ma']
        self.Re    = udef_dict['Re']
        self.Pr    = udef_dict['Pr']
        self.kappa = udef_dict['kappa']
        self.R     = udef_dict['R']
        
        ## get T_inf
        if ('T_unend' in udef_dict):
            self.T_inf = udef_dict['T_unend']
        elif ('T_inf' in udef_dict):
            self.T_inf = udef_dict['T_inf']
        elif ('Tinf' in udef_dict):
            self.T_inf = udef_dict['Tinf']
        else:
            raise ValueError('T_inf not found in udef')
        
        ## get p_inf
        if ('p_unend' in udef_dict):
            self.p_inf = udef_dict['p_unend']
        elif ('p_inf' in udef_dict):
            self.p_inf = udef_dict['p_inf']
        elif ('pinf' in udef_dict):
            self.p_inf = udef_dict['pinf']
        else:
            raise ValueError('p_inf not found in udef')
        
        ## !!! import what is called 'C_Suth' in NS3D as 'S_Suth' !!!
        self.S_Suth = udef_dict['C_Suth'] ## [K] --> Sutherland temperature, usually 110.4 [K] for air
        
        self.mu_Suth_ref = udef_dict['mu_Suth_ref'] ## usually 1.716e-05 [kg/(m·s)] --> μ of air at T_Suth_ref = 273.15 [K]
        self.T_Suth_ref  = udef_dict['T_Suth_ref'] ## usually 273.15 [K]
        
        self.C_Suth = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## ~1.458e-6 [kg/(m·s·√K)]
        
        ## derived values
        self.rho_inf = self.p_inf/(self.R*self.T_inf) ## mass density [kg/m³]
        self.mu_inf  = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth)) ## [Pa s] | [N s m^-2]
        self.nu_inf  = self.mu_inf / self.rho_inf ## kinematic viscosity [m²/s] --> momentum diffusivity
        
        ## additional
        if ('total_avg_time' in udef_dict):
            self.total_avg_time = udef_dict['total_avg_time']
        if ('total_avg_iter_count' in udef_dict):
            self.total_avg_iter_count = udef_dict['total_avg_iter_count']
        if ('nz' in udef_dict):
            self.nz = udef_dict['nz']
        if ('cp' in udef_dict):
            self.cp = udef_dict['cp']
        if ('cv' in udef_dict):
            self.cv = udef_dict['cv']
        
        # ===
        
        if self.verbose: print(72*'-')
        if self.verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
        if self.verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
        if self.verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
        if self.verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
        if self.verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
        if self.verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
        if self.verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
        if self.verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
        if self.verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
        if self.verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
        if self.verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
        if self.verbose: print(72*'-')
        
        self.a_inf     = np.sqrt(self.kappa*self.R*self.T_inf)
        self.U_inf     = self.Ma*self.a_inf
        self.cp        = self.R*self.kappa/(self.kappa-1.)
        self.cv        = self.cp/self.kappa
        self.recov_fac = self.Pr**(1/3)
        self.Taw       = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
        self.lchar     = self.Re*self.nu_inf/self.U_inf
        
        if self.verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
        if self.verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
        if self.verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
        if self.verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
        if self.verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
        if self.verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
        if self.verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
        #if self.verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
        #if self.verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
        if self.verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
        if self.verbose: print(72*'-'+'\n')
        
        # ===
        
        udef_char = [    'Ma',     'Re',     'Pr',     'kappa',    'R',    'p_inf',    'T_inf',    'C_Suth',    'mu_Suth_ref',    'T_Suth_ref' ]
        udef_real = [self.Ma , self.Re , self.Pr , self.kappa, self.R, self.p_inf, self.T_inf, self.C_Suth, self.mu_Suth_ref, self.T_Suth_ref  ]
        udef = dict(zip(udef_char, udef_real))
        
        return udef

# 1D curve fitting
# ======================================================================

class curve_fitter(object):
    '''
    creates a curve fitter instance which is callable afterward
    - includes text for LaTeX and matplotlib
    '''
    def __init__(self, curveType, x, y):
        
        self.curveType = curveType
        
        if (curveType=='linear'):
            '''
            linear function : y=a+b·x
            a straight line on a lin-lin plot
            '''
            def curve(x, a, b):
                return a + b*x
        
        elif (curveType=='power'):
            '''
            power law y=a·x^b
            a straight line on a log-log plot
            '''
            def curve(x, a, b):
                return a*np.power(x,b)
        
        elif (curveType=='power_plus_const'):
            '''
            power law (plus a constant) y = a + b·(x^c)
            no longer a straight line on log-log but allows non-zero y-intercept
            '''
            def curve(x, a, b, c):
                return a + b*np.power(x,c)
        
        elif (curveType=='power_asymp'):
            '''
            power asymptotic
            '''
            def curve(x, a, b, c, d):
                return a/(b + c*np.power(x,d))
        
        elif (curveType=='exp'):
            '''
            exponential curve y = a + b·exp(c*x)
            '''
            def curve(x, a, b, c):
                return a + b*np.exp(c*x)
        
        elif (curveType=='log'):
            '''
            a straight line on a semi-log (lin-log) plot 
            '''
            def curve(x, a, b):
                return a + b*np.log(x)
        
        else:
            raise ValueError('curveType not recognized : %s'%str(curveType))
        
        self.__curve = curve ## private copy of curve() method
        self.popt, self.pcov = sp.optimize.curve_fit(self.__curve, x, y, maxfev=int(5e5), method='trf')
        
        # ===
        
        if (curveType=='linear'):
            a, b = self.popt
            self.txt = '%0.12e + %0.12e * x'%(a,b)
            self.latex = r'$%0.5f + %0.5f{\cdot}x$'%(a,b)
        
        elif (curveType=='power'):
            a, b = self.popt
            self.txt = '%0.12e * x**%0.12e'%(a,b)
            self.latex = r'$%0.5f{\cdot}x^{%0.5f}$'%(a,b)
        
        elif (curveType=='power_plus_const'):
            a, b, c = self.popt
            self.txt = '%0.12e + %0.12e * x**%0.12e'%(a,b,c)
            self.latex = r'$%0.5f + %0.5f{\cdot}x^{%0.5f}$'%(a,b,c)
        
        elif (curveType=='power_asymp'):
            a, b, c, d = self.popt
            self.txt = '%0.12e / (%0.12e + %0.12e * x**%0.12e)'%(a,b,c,d)
            self.latex = r'$%0.5f / (%0.5f + %0.5f {\cdot} x^{%0.5f})$'%(a,b,c,d)
        
        elif (curveType=='exp'):
            a, b, c = self.popt
            self.txt = '%0.12e + %0.12e * np.exp(%0.12e * x)'%(a,b,c)
            self.latex = r'$%0.5f + %0.5f{\cdot}\text{exp}(%0.5f{\cdot}x)$'%(a,b,c)
        
        elif (curveType=='log'):
            a, b = self.popt
            self.txt = '%0.12e + %0.12e * np.log(x)'%(a,b)
            self.latex = r'$%0.6f + %0.6f{\cdot}\text{ln}(x)$'%(a,b)
        
        else:
            raise NotImplementedError('curveType \'%s\' not recognized'%str(curveType))
    
    def __call__(self, xn):
        return self.__curve(xn, *self.popt)

# boundary layer & aerodynamics
# ======================================================================

def Blasius_solution(eta):
    '''
    f·f′′ + 2·f′′′ = 0  ==>  f′′′ = -(1/2)·f·f′′
    BCs: f(0)=0, f′(0)=0, f′(∞)=1
    -----
    for solve_ivp(): d[f′′(η)]/dη = F(f(η), f′(η), f′′(η))
    y=[f,f′,f′′], y′=[ y[1], y[2], (-1/2)·y[0]·y[2] ]
    '''
    
    #Blasius = lambda t,y: [y[1],y[2],-0.5*y[0]*y[2]]
    
    def Blasius(t,y):
        return np.array([y[1], y[2], -0.5*y[0]*y[2]])
    
    if False: ## calculate c0
        import warnings
        warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) 
        def eq_root(c0,eta):
            sol = sp.integrate.solve_ivp( fun=Blasius, t_span=[0,eta[-1]], y0=[0,0,c0], t_eval=eta, method='RK45' )
            fp  = np.copy( sol.y[1] ) ## f′
            res = 1. - fp[-1] ## residual for BC: f′(∞)=0
            return res
        eta = np.linspace(0,1e3,int(1e3))
        sol = sp.optimize.fsolve(eq_root, x0=0.332, args=(eta,), xtol=1e-9)
        c0 = sol[0]
        #print('c0 = %0.14f'%c0)
    else:
        c0 = 0.33200221890262
    
    sol = sp.integrate.solve_ivp( fun=Blasius, 
                                  t_span=[0,eta[-1]], 
                                  y0=[ 0, 0, c0 ], 
                                  t_eval=eta, 
                                  method='RK45' )
    
    f, fp, fpp = sol.y ## f′=u/U
    
    ## i_99 = np.abs(fp-0.99).argmin()
    ## eta_99 = eta[i_99]
    ## print('η(f′=%0.14f) = %0.14f'%(fp[i_99],eta_99))
    ## ## η(f′=0.0.99000000002790)=4.91131834911318
    
    return f, fp, fpp

class freestream_parameters(object):
    '''
    calculate freestream parameters & characteristic scales for dry air (21% O2 / 78% N2) and standard conditions
    '''
    
    def __init__(self, Re=None, M_inf=None, lchar=None, U_inf=None, T_inf=None, p_inf=None, rho_inf=None, Pr=None, compressible=True, HTfac=1.0):
        
        if not isinstance(compressible, bool):
            raise ValueError
        
        self.compressible = compressible
        
        ## dry air (21% O2 / 78% N2)
        M_molar = 28.9647e-3        ## molar mass [kg/mol]
        R_molar = 8.31446261815324  ## molar gas constant [J/(K·mol)]
        R       = R_molar / M_molar ## specific / individual gas constant [J/(kg·K)]
        cp      = (7/2)*R           ## isobaric specific heat (ideal gas) [J/(kg·K)]
        cv      = (5/2)*R           ## isochoric specific heat (ideal gas) [J/(kg·K)]
        kappa   = cp/cv
        
        # === get freestream static T_inf, p_inf, ρ_inf
        
        # T_inf   = 273.15 + 15        ## freestream static temperature [K]
        # p_inf   = 101325.            ## freestream static pressure [Pa]
        # rho_inf = 1.2249908312142817 ## freestream mass density [kg/m³]
        
        if all([(T_inf is None),(p_inf is None),(rho_inf is None)]): ## NONE of T,p,ρ provided --> use standard atmosphere
            T_inf   = 273.15 + 15
            p_inf   = 101325.
            rho_inf = p_inf/(R*T_inf)
        elif all([(T_inf is not None),(p_inf is not None),(rho_inf is not None)]): ## ALL T,p,ρ provided --> check
            np.testing.assert_allclose(rho_inf, p_inf/(R*T_inf), rtol=1e-14, atol=1e-14)
        elif (T_inf is not None) and (p_inf is None) and (rho_inf is None): ## only T provided
            p_inf   = 101325.
            rho_inf = p_inf/(R*T_inf)
        elif (T_inf is None) and (p_inf is not None) and (rho_inf is None): ## only p provided
            T_inf   = 273.15 + 15
            rho_inf = p_inf/(R*T_inf)
        elif (T_inf is None) and (p_inf is None) and (rho_inf is not None): ## only ρ provided
            T_inf   = 273.15 + 15
            p_inf = rho_inf*R*T_inf
        elif (T_inf is not None) and (p_inf is not None) and (rho_inf is None): ## T & p provided
            rho_inf = p_inf/(R*T_inf)
        elif (T_inf is not None) and (p_inf is None) and (rho_inf is not None): ## T & ρ provided
            p_inf = rho_inf*R*T_inf
        elif (T_inf is None) and (p_inf is not None) and (rho_inf is not None): ## p & ρ provided
            T_inf = p_inf/(rho_inf*R)
        else:
            raise ValueError('this should never happen.')
        
        np.testing.assert_allclose(rho_inf, p_inf/(R*T_inf), rtol=1e-14, atol=1e-14)
        
        # === get freestream dynamic & kinematic viscosities using Sutherland's Law
        
        ## Sutherland's Law : dynamic viscosity : μ(T)
        ## https://www.cfd-online.com/Wiki/Sutherland%27s_law
        ## White 2006 'Viscous Fluid Flow' 3rd Edition, p. 28-31
        S_Suth      = 110.4    ## [K] --> Sutherland temperature
        mu_Suth_ref = 1.716e-5 ## [kg/(m·s)] --> μ of air at T_Suth_ref = 273.15 [K]
        T_Suth_ref  = 273.15   ## [K]
        C_Suth      = mu_Suth_ref/(T_Suth_ref**(3/2))*(T_Suth_ref + S_Suth) ## [kg/(m·s·√K)]
        
        mu_inf_1 = C_Suth*T_inf**(3/2)/(T_inf+S_Suth) ## [Pa·s] | [N·s/m²]
        mu_inf_2 = mu_Suth_ref*(T_inf/T_Suth_ref)**(3/2) * ((T_Suth_ref+S_Suth)/(T_inf+S_Suth)) ## [Pa·s] | [N·s/m²]
        np.testing.assert_allclose(mu_inf_1, mu_inf_2, rtol=1e-14, atol=1e-14)
        mu_inf = mu_inf_1
        
        nu_inf = mu_inf / rho_inf ## kinematic viscosity [m²/s] --> momentum diffusivity
        
        # === get Prandtl number, thermal conductivity, thermal diffusivity
        
        if (Pr is None):
            
            ## Sutherland's Law : thermal conductivity : k(T)
            ## White 2006 'Viscous Fluid Flow' 3rd Edition, p. 28-31
            k_Suth_ref = 0.0241 ## [W/(m·K)] reference thermal conductivity
            Sk_Suth    = 194.0  ## [K]
            k_inf      = k_Suth_ref*(T_inf/T_Suth_ref)**(3/2) * ((T_Suth_ref+Sk_Suth)/(T_inf+Sk_Suth)) ## thermal conductivity [W/(m·K)]
            
            alpha     = k_inf/(rho_inf*cp) ## thermal diffusivity [m²/s]
            Pr        = nu_inf/alpha       ## [-] ==cp·mu/k_inf
        
        else:
            
            k_inf = cp*mu_inf/Pr ## thermal conductivity [W/(m·K)]
            alpha = k_inf/(rho_inf*cp) ## thermal diffusivity [m²/s]
        
        np.testing.assert_allclose(nu_inf/alpha, cp*mu_inf/k_inf, rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(Pr, nu_inf/alpha, rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(Pr, cp*mu_inf/k_inf, rtol=1e-14, atol=1e-14)
        
        # === get freestream velocity magnitude U_inf, freestream speed of sound a_inf, and Mach number M_inf
        
        a_inf = np.sqrt(kappa*R*T_inf) ## speed of sound [m/s]
        
        if (not self.compressible): ## incompressible
            
            if (U_inf is None):
                raise ValueError('if compressible=False, then providing U_inf is required')
            if (M_inf is not None) and (M_inf != 0.):
                raise ValueError('if compressible=False, then providing a non-zero M_inf makes no sense')
            
            M_inf = 0.
            a_inf = np.inf
        
        else: ## compressible
            
            if (U_inf is None) and (M_inf is None):
                raise ValueError('provide either U_inf or M_inf')
            elif (U_inf is not None) and (M_inf is not None):
                np.testing.assert_allclose(M_inf, U_inf/a_inf, rtol=1e-14, atol=1e-14)
            elif (U_inf is None) and (M_inf is not None):
                U_inf = a_inf * M_inf ## velocity freestream [m/s]
            elif (U_inf is not None) and (M_inf is None):
                M_inf = U_inf / a_inf ## Mach number
            else:
                raise ValueError('this should never happen.')
        
        np.testing.assert_allclose(M_inf, U_inf/a_inf, rtol=1e-14, atol=1e-14)
        
        # === get characteristic freestream scales: length, time, velocity
        
        if (lchar is None) and (Re is None):
            raise ValueError('provide either lchar or Re')
        elif (lchar is not None) and (Re is not None):
            np.testing.assert_allclose(Re, lchar * U_inf / nu_inf, rtol=1e-14, atol=1e-14)
        elif (lchar is None) and (Re is not None):
            lchar = Re * nu_inf / U_inf
        elif (lchar is not None) and (Re is None):
            Re = lchar * U_inf / nu_inf
        else:
            raise ValueError('this should never happen.')
        
        np.testing.assert_allclose(Re, lchar*U_inf/nu_inf, rtol=1e-14, atol=1e-14)
        
        tchar = lchar / U_inf
        uchar = lchar / tchar
        if not np.isclose(uchar, U_inf, rtol=1e-14):
            raise AssertionError('U_inf!=uchar')
        
        np.testing.assert_allclose(lchar, uchar*tchar, rtol=1e-14, atol=1e-14)
        
        # === get isentropic total state
        
        if self.compressible:
            T_tot   = T_inf   * (1 + (kappa-1)/2 * M_inf**2)
            p_tot   = p_inf   * (1 + (kappa-1)/2 * M_inf**2)**(kappa/(kappa-1))
            rho_tot = rho_inf * (1 + (kappa-1)/2 * M_inf**2)**(1/(kappa-1))
        else:
            T_tot   = T_inf + U_inf**2 / (2 * cp)
            p_tot   = p_inf + 0.5*rho_inf*U_inf**2 ## Bernoulli
            rho_tot = p_tot/(R*T_tot)
        
        np.testing.assert_allclose(T_tot , T_inf + U_inf**2/(2*cp) , rtol=1e-14, atol=1e-14)
        np.testing.assert_allclose(p_tot , rho_tot*R*T_tot         , rtol=1e-14, atol=1e-14)
        
        # === get wall temperatures for a turbulent boundary layer
        
        recov_fac = pow(Pr,1/3) ## recovery factor (turbulent flat plate boundary layer)
        Taw       = T_inf + recov_fac * U_inf**2/(2*cp) ## adiabatic wall temperature
        Tw        = T_inf + recov_fac * U_inf**2/(2*cp) * HTfac ## wall temperature (with heat transfer factor)
        
        # === attach to self
        
        self.R_molar     = R_molar
        self.M_molar     = M_molar
        self.R           = R
        self.cp          = cp
        self.cv          = cv
        self.kappa       = kappa
        
        self.T_inf       = T_inf
        self.p_inf       = p_inf
        self.rho_inf     = rho_inf
        
        self.S_Suth      = S_Suth
        self.mu_Suth_ref = mu_Suth_ref
        self.T_Suth_ref  = T_Suth_ref
        self.C_Suth      = C_Suth
        self.mu_inf      = mu_inf
        self.nu_inf      = nu_inf
        
        #self.k_Suth_ref  = k_Suth_ref
        #self.Sk_Suth     = Sk_Suth
        self.k_inf       = k_inf
        
        self.alpha       = alpha
        self.Pr          = Pr
        
        self.M_inf       = M_inf
        self.a_inf       = a_inf
        self.U_inf       = U_inf
        
        self.Re          = Re
        self.lchar       = lchar
        self.tchar       = tchar
        self.uchar       = uchar
        
        self.T_tot       = T_tot
        self.p_tot       = p_tot
        self.rho_tot     = rho_tot
        
        self.recov_fac   = recov_fac
        self.HTfac       = HTfac
        self.Taw         = Taw
        self.Tw          = Tw
    
    def set_Re(self, Re):
        '''
        set reference Reynolds Number, which determines lchar & tchar
        '''
        self.Re    = Re
        self.lchar = Re * self.nu_inf / self.U_inf
        self.tchar = self.lchar / self.U_inf
        self.uchar = self.lchar / self.tchar
        if not np.isclose(self.uchar, self.U_inf, rtol=1e-14):
            raise AssertionError('U_inf!=uchar')
        return
    
    def print(self,):
        
        even_print('R_molar'        , '%0.6f [J/(K·mol)]'   % self.R_molar    )
        even_print('M_molar'        , '%0.5e [kg/mol]'      % self.M_molar    )
        even_print('R'              , '%0.3f [J/(kg·K)]'    % self.R          )
        even_print('cp'             , '%0.3f [J/(kg·K)]'    % self.cp         )
        even_print('cv'             , '%0.3f [J/(kg·K)]'    % self.cv         )
        even_print('kappa'          , '%0.3f [-]'           % self.kappa      )
        
        print(72*'-')
        even_print('compressible'   , '%s'                  % self.compressible )
        even_print('M_inf'          , '%0.4f [-]'           % self.M_inf      )
        even_print('Re'             , '%0.4f [-]'           % self.Re         )
        
        even_print('T_inf'          , '%0.4f [K]'           % self.T_inf      )
        even_print('T_tot'          , '%0.4f [K]'           % self.T_tot      )
        even_print('Tw'             , '%0.4f [K]'           % self.Tw         )
        even_print('Taw'            , '%0.4f [K]'           % self.Taw        )
        even_print('p_inf'          , '%0.1f [Pa]'          % self.p_inf      )
        even_print('p_tot'          , '%0.1f [Pa]'          % self.p_tot      )
        even_print('rho_inf'        , '%0.4f [kg/m³]'       % self.rho_inf    )
        even_print('rho_tot'        , '%0.4f [kg/m³]'       % self.rho_tot    )
        
        even_print('T_Suth_ref'     , '%0.2f [K]'           % self.T_Suth_ref )
        even_print('C_Suth'         , '%0.3e [kg/(m·s·√K)]' % self.C_Suth     )
        even_print('S_Suth'         , '%0.2f [K]'           % self.S_Suth     )
        
        even_print('mu_inf'         , '%0.5e [kg/(m·s)]'    % self.mu_inf     )
        even_print('nu_inf'         , '%0.5e [m²/s]'        % self.nu_inf     )
        even_print('k_inf'          , '%0.5e [W/(m·K)]'     % self.k_inf      )
        even_print('alpha'          , '%0.5e [m²/s]'        % self.alpha      )
        
        even_print('Pr'             , '%0.5f [-]'           % self.Pr         )
        even_print('recovery factor', '%0.5f [-]'           % self.recov_fac  )
        
        even_print('a_inf'          , '%0.5f [m/s]'         % self.a_inf      )
        even_print('U_inf'          , '%0.5f [m/s]'         % self.U_inf      )
        even_print('uchar'          , '%0.5f [m/s]'         % self.uchar      )
        even_print('lchar'          , '%0.5e [m]'           % self.lchar      )
        even_print('tchar'          , '%0.5e [s]'           % self.tchar      )
        print(72*'-')
        
        return

def calc_profile_edge_1d(y, u=None, ddy_u=None, **kwargs):
    '''
    determine the edge location of 1D BL profile
    - [y] is a 1D coordinate vector
    - if 'ddy_u' is passed: is an externally calculated du/dy (passing it skips gradient calc in this func)
    - if 'u' is passed: 'u' still needs to be d/dy()'ed (can be less efficient)
    -----
    - 'edge' is where |du/dy|<ϵ
    - this func is units-agnostic. it is assumed that |du/dy|_y=0 is O(1)
    '''
    
    epsilon      = kwargs.get('epsilon',5e-5)
    acc          = kwargs.get('acc',6)
    edge_stencil = kwargs.get('edge_stencil','half')
    interp_kind  = kwargs.get('interp_kind','cubic')
    ongrid       = kwargs.get('ongrid',True) ## snap to grid point
    
    ## these options were deprecated
    ## check that they are not passed
    lchar = kwargs.get('lchar',None)
    if (lchar is not None):
        raise ValueError('lchar was deprecated as kwarg for turbx.calc_profile_edge_1d()')
    deriv = kwargs.get('deriv',None)
    if (deriv is not None):
        raise ValueError('deriv was deprecated as kwarg for turbx.calc_profile_edge_1d()')
    
    do_debug_plot = kwargs.get('do_debug_plot',False) ## show a debug plot for tuning ϵ
    
    ## checks
    if not isinstance(y, np.ndarray):
        raise ValueError('y should be a numpy array')
    if (y.ndim!=1):
        raise ValueError
    
    ## make sure exactly one of u | ddy_u was passed
    if (u is None) and (ddy_u is None):
        raise ValueError('pass one of either u or ddy_u')
    if (u is not None) and (ddy_u is not None):
        raise ValueError('either pass u or ddy_u (not both)')
    
    ## make sure dims of passed u | ddy_u are correct
    if (u is not None):
        if not isinstance(u, np.ndarray):
            raise ValueError
        if (u.ndim!=1):
            raise ValueError
        if (u.shape[0]!=y.shape[0]):
            raise ValueError
    if (ddy_u is not None):
        if not isinstance(ddy_u, np.ndarray):
            raise ValueError
        if (ddy_u.ndim!=1):
            raise ValueError
        if (ddy_u.shape[0]!=y.shape[0]):
            raise ValueError
    
    if not any([(interp_kind=='linear'),(interp_kind=='cubic')]):
        raise ValueError(f"'interp_kind' should be one of: 'linear','cubic'")
    
    ny = y.shape[0]
    
    ## ddy_u was NOT passed, need to calculate du/dy here
    if (ddy_u is None):
        ddy_u = gradient(u, y, axis=0, d=1, acc=acc, edge_stencil=edge_stencil)
    
    ## confirm that |d□/dy|_w is ~O(1)
    np.testing.assert_allclose( ddy_u[0], 1., rtol=0.05)
    
    ## static ϵ
    # ## get [y] of first intersection
    # jtop = ny-1
    # for j in range(ny):
    #     if ( np.abs(ddy_u[j]) < epsilon ): ## |d□/dy|<ϵ
    #         jtop = j
    #         break
    
    ## variable ϵ
    ## get [y] of first intersection
    keep_going = True
    while keep_going:
        jtop = ny-1
        for j in range(ny):
            if ( np.abs(ddy_u[j]) < epsilon ): ## |d□/dy|<ϵ
                jtop = j
                break
        if (jtop == ny-1):
            epsilon *= 1.05
            tqdm.write(f'recalibrating: ϵ={epsilon:0.3e}')
        else:
            keep_going = False
    
    if (jtop == ny-1):
        raise ValueError('jtop == ny-1')
    if (jtop<3):
        raise ValueError('jtop<3')
    
    if ongrid: ## grid-snapped, i.e. first point satisfying |d□/dy|<ϵ
        y_edge = float( y[jtop] )
    
    else: ## find interpolated |d□/dy|==ϵ
        
        #intrp_func = sp.interpolate.interp1d(y, ddy_u, kind=interp_kind, bounds_error=True) ## deprecated!
        
        if (interp_kind=='cubic'):
            
            #intrp_func = sp.interpolate.CubicSpline(y, ddy_u, axis=0, bc_type='natural', extrapolate=False)
            intrp_func = sp.interpolate.PchipInterpolator(y, ddy_u, axis=0, extrapolate=False)
            #intrp_func = sp.interpolate.Akima1DInterpolator(y, ddy_u, axis=0, method='akima', extrapolate=False)
        
        elif (interp_kind=='linear'):
            raise NotImplementedError
            ## need to use np.interp() --> no longer retuns a function! needs to be wrapped.
        else:
            raise NotImplementedError
        
        def __f_opt_edge_locator(y_test, intrp_func, epsilon):
            ddy_u_test = intrp_func(y_test)
            root = np.abs( ddy_u_test ) - epsilon
            return root
        
        sol = sp.optimize.least_squares(
                    fun=__f_opt_edge_locator,
                    args=(intrp_func,epsilon,),
                    x0 = y[0] + 0.99*(y[jtop]-y[0]),
                    #xtol=1e-15,
                    #ftol=1e-15,
                    #gtol=1e-15,
                    method='dogbox',
                    #method='trf',
                    #bounds=(y[jtop-2], y[jtop]),
                    bounds=(y[0], y[jtop]),
                    )
        if not sol.success:
            raise ValueError
        if ( sol.x.shape[0] != 1 ):
            raise ValueError
        
        y_edge = float(sol.x[0])
        
        if ( y_edge > y[jtop] ):
            raise ValueError
    
    if do_debug_plot:
        plt.close('all')
        fig1 = plt.figure(figsize=(2*(16/9),2), dpi=300)
        ax1 = plt.gca()
        ax1.set_xscale('log', base=10)
        ax1.plot(
            ddy_u,
            y,
            lw=0.8,
            )
        #ax1.set_xlim(1e-30, 1e1)
        ax1.axhline(y=y_edge, linestyle='dashed', c='gray', zorder=1, lw=0.5)
        ax1.axvline(x=epsilon, linestyle='dashed', c='gray', zorder=1, lw=0.5)
        fig1.tight_layout(pad=0.25)
        fig1.tight_layout(pad=0.25)
        plt.show()
    
    return y_edge

def calc_d99_1d(y, u, y_edge, u_edge=None, **kwargs):
    '''
    determine δ99 location of 1D profile
    - [y] is a 1D coordinate vector
    - [u] is some profile variable (streamwise velocity, pseudovelocity, etc.)
    - [y_edge] is the edge location
    - [u_edge] (optional) is the variable value at the edge location
    '''
    
    rtol        = kwargs.get('rtol',1e-3) ## rtol for asserting u[y_edge]==u_edge
    interp_kind = kwargs.get('interp_kind','cubic')
    d95         = kwargs.get('d95',False) ## δ95 instead of δ99
    
    if (y.ndim!=1):
        raise ValueError
    if (u.ndim!=1):
        raise ValueError
    if (u.shape[0]!=y.shape[0]):
        raise ValueError
    if not isinstance(y_edge, (float,np.float32,np.float64)):
        raise ValueError
    if (u_edge is not None):
        if not isinstance(u_edge, (float,np.float32,np.float64)):
            raise ValueError
    
    if (y_edge>y.max()):
        raise ValueError
    if (y_edge<y.min()):
        raise ValueError
    
    ny = y.shape[0]
    
    j_edge = np.abs(y-y_edge).argmin()
    if ( y[j_edge] < y_edge ):
        j_edge += 1
    je = min(j_edge,ny)
    je += 1 ## for numpy [:n]
    
    intrp_func = sp.interpolate.interp1d(x=y[:je], y=u[:je], kind=interp_kind, bounds_error=True)
    
    if (u_edge is not None): ## assert passed u_edge is == f(y_edge)
        np.testing.assert_allclose(u_edge, intrp_func(y_edge), rtol=rtol)
    else:
        u_edge = intrp_func(y_edge)
    
    def __f_opt_d99_locator(y_test, intrp_func, u_edge):
        root = np.abs( 0.99*u_edge - intrp_func(y_test) )
        return root
    
    def __f_opt_d95_locator(y_test, intrp_func, u_edge):
        root = np.abs( 0.95*u_edge - intrp_func(y_test) )
        return root
    
    if d95:
        __f_opt = __f_opt_d95_locator
    else:
        __f_opt = __f_opt_d99_locator
    
    sol = sp.optimize.least_squares(
                fun=__f_opt,
                args=(intrp_func,u_edge),
                x0=0.1*y_edge,
                xtol=1e-15,
                ftol=1e-15,
                gtol=1e-15,
                method='dogbox',
                bounds=(y.min(), y_edge),
                )
    if not sol.success:
        raise ValueError
    if ( sol.x.shape[0] != 1 ):
        raise ValueError
    
    delta = float(sol.x[0]) ## δ99 or δ95
    
    if ( delta > y_edge ):
        raise ValueError
    
    return delta

def calc_bl_integral_quantities_1d( y, u, rho, u_tau, d99, y_edge, rho_edge, nu_edge, u_edge, nu_wall, **kwargs):
    '''
    for 1D profile get [θ, δ*, Re_θ, Re_τ]
    '''
    
    rtol        = kwargs.get('rtol',1e-6)
    interp_kind = kwargs.get('interp_kind','cubic')
    
    if (y.ndim!=1):
        raise ValueError
    
    if (u.ndim!=1):
        raise ValueError
    if (u.shape[0]!=y.shape[0]):
        raise ValueError
    if (rho.ndim!=1):
        raise ValueError
    if (rho.shape[0]!=y.shape[0]):
        raise ValueError
    
    ## type check
    if not isinstance(u_tau, (float,np.float32,np.float64)):
        raise ValueError
    if not isinstance(d99, (float,np.float32,np.float64)):
        raise ValueError
    if not isinstance(y_edge, (float,np.float32,np.float64)):
        raise ValueError
    if not isinstance(rho_edge, (float,np.float32,np.float64)):
        print(type(rho_edge))
        raise ValueError
    if not isinstance(nu_edge, (float,np.float32,np.float64)):
        raise ValueError
    if not isinstance(u_edge, (float,np.float32,np.float64)):
        raise ValueError
    if not isinstance(nu_wall, (float,np.float32,np.float64)):
        raise ValueError
    
    if (y_edge>y.max()):
        raise ValueError
    if (y_edge<y.min()):
        raise ValueError
    
    ny = y.shape[0]
    
    rho_edge_ = sp.interpolate.interp1d(y, rho, kind=interp_kind, bounds_error=True)(y_edge)
    np.testing.assert_allclose(rho_edge, rho_edge_, rtol=rtol)
    
    u_edge_ = sp.interpolate.interp1d(y, u, kind=interp_kind, bounds_error=True)(y_edge)
    np.testing.assert_allclose(u_edge, u_edge_, rtol=rtol)
    
    # ===
    
    integrand_theta_cmp = (u*rho)/(u_edge*rho_edge)*(1-(u/u_edge))
    integrand_dstar_cmp = (1-((u*rho)/(u_edge*rho_edge)))
    
    theta_cmp_     = sp.integrate.cumulative_trapezoid(y=integrand_theta_cmp, x=y, initial=0.)
    theta_cmp_func = sp.interpolate.interp1d(y, theta_cmp_, kind=interp_kind)
    theta_cmp      = theta_cmp_func(y_edge)
    
    dstar_cmp_     = sp.integrate.cumulative_trapezoid(y=integrand_dstar_cmp, x=y, initial=0.)
    dstar_cmp_func = sp.interpolate.interp1d(y, dstar_cmp_, kind=interp_kind)
    dstar_cmp      = dstar_cmp_func(y_edge)
    
    integrand_theta_inc = (u/u_edge)*(1-(u/u_edge))
    integrand_dstar_inc = 1-(u/u_edge)
    
    theta_inc_     = sp.integrate.cumulative_trapezoid(y=integrand_theta_inc, x=y, initial=0.)
    theta_inc_func = sp.interpolate.interp1d(y, theta_inc_, kind=interp_kind)
    theta_inc      = theta_inc_func(y_edge)
    
    dstar_inc_     = sp.integrate.cumulative_trapezoid(y=integrand_dstar_inc, x=y, initial=0.)
    dstar_inc_func = sp.interpolate.interp1d(y, dstar_inc_, kind=interp_kind)
    dstar_inc      = dstar_inc_func(y_edge)
    
    # ===
    
    H12     = dstar_cmp/theta_cmp
    H12_inc = dstar_inc/theta_inc
    
    Re_tau       = d99*u_tau/nu_wall
    Re_theta     = theta_cmp*u_edge/nu_edge
    Re_theta_inc = theta_inc*u_edge/nu_edge
    Re_d99       = d99*u_edge/nu_edge
    
    dd = { 'theta_cmp':theta_cmp,
           'dstar_cmp':dstar_cmp,
           'theta_inc':theta_inc,
           'dstar_inc':dstar_inc,
           'H12':H12,
           'H12_inc':H12_inc,
           'Re_tau':Re_tau,
           'Re_theta':Re_theta,
           'Re_theta_inc':Re_theta_inc,
           'Re_d99':Re_d99,
         }
    
    return dd

# numerical & grid
# ======================================================================

def interp_2d_structured(x2d_A, y2d_A, x2d_B, y2d_B, data_A, **kwargs):
    '''
    interpolate 2D array 'data_A' from grid A onto grid B, yielding 'data_B'
    --> based on sp.interpolate.griddata()
    --> default 'cubic' interpolation, where NaNs occur, fill with 'nearest'
    --> option to re-interpolate the edges in 1D along edge path length
    '''
    
    method = kwargs.get('method','cubic')
    
    ## re-interpolate the edges of the 2D domain in 1D along path length [s]
    re_interp_edges_1d = kwargs.get('re_interp_edges_1d',False)
    
    if not isinstance(x2d_A, np.ndarray):
        raise ValueError('x2d_A should be a numpy array')
    if not isinstance(y2d_A, np.ndarray):
        raise ValueError('y2d_A should be a numpy array')
    if not isinstance(x2d_B, np.ndarray):
        raise ValueError('x2d_B should be a numpy array')
    if not isinstance(y2d_B, np.ndarray):
        raise ValueError('y2d_B should be a numpy array')
    if not isinstance(data_A, np.ndarray):
        raise ValueError('data_A should be a numpy array')
    
    # < need a lot of checks still >
    
    if not any([(method=='linear'),(method=='cubic')]):
        raise ValueError("'method' should be one of 'cubic' or 'linear'")
    
    nxA,nyA = data_A.shape
    nxB,nyB = x2d_B.shape
    
    ## interp2d() --> gets OverflowError for big meshes
    # interpolant = sp.interpolate.interp2d(x2d_A.flatten(),
    #                                       y2d_A.flatten(),
    #                                       data_A.flatten(),
    #                                       kind='linear',
    #                                       copy=True,
    #                                       bounds_error=False,
    #                                       fill_value=np.nan)
    # utang_wn = interpolant( x2d_B.flatten(), 
    #                         y2d_B.flatten() )
    
    B_nearest = sp.interpolate.griddata( points=(x2d_A.flatten(), y2d_A.flatten()),
                                         values=data_A.flatten(),
                                         xi=(x2d_B.flatten(), y2d_B.flatten()),
                                         method='nearest',
                                         fill_value=np.nan )
    B_nearest = np.reshape(B_nearest, (nxB,nyB), order='C')
    
    B_cubic = sp.interpolate.griddata( points=(x2d_A.flatten(), y2d_A.flatten()),
                                       values=data_A.flatten(),
                                       xi=(x2d_B.flatten(), y2d_B.flatten()),
                                       method=method,
                                       fill_value=np.nan )
    B_cubic = np.reshape(B_cubic, (nxB,nyB), order='C')
    
    #nan_indices = np.nonzero(np.isnan(B_cubic))
    #n_nans = np.count_nonzero(np.isnan(B_cubic))
    
    ## where the linear/cubic interpolation failed, replace with 'nearest'
    data_B = np.where( np.isnan(B_cubic), B_nearest, B_cubic)
    
    ## optionally replace the 2D edges (usually where 'nearest' is applied) with a 1D interpolation
    ## corners and 'digitally integrated' lengths must match
    if re_interp_edges_1d:
        
        ## assert that corner points are same [x]
        np.testing.assert_allclose( x2d_A[ 0, 0] , x2d_B[ 0, 0] , rtol=1e-5, atol=1e-5) ## SW
        np.testing.assert_allclose( x2d_A[-1, 0] , x2d_B[-1, 0] , rtol=1e-5, atol=1e-5) ## SE
        np.testing.assert_allclose( x2d_A[ 0,-1] , x2d_B[ 0,-1] , rtol=1e-5, atol=1e-5) ## NW
        np.testing.assert_allclose( x2d_A[-1,-1] , x2d_B[-1,-1] , rtol=1e-5, atol=1e-5) ## NE
        
        ## assert that corner points are same [y]
        np.testing.assert_allclose( y2d_A[ 0, 0] , y2d_B[ 0, 0] , rtol=1e-5, atol=1e-5) ## SW
        np.testing.assert_allclose( y2d_A[-1, 0] , y2d_B[-1, 0] , rtol=1e-5, atol=1e-5) ## SE
        np.testing.assert_allclose( y2d_A[ 0,-1] , y2d_B[ 0,-1] , rtol=1e-5, atol=1e-5) ## NW
        np.testing.assert_allclose( y2d_A[-1,-1] , y2d_B[-1,-1] , rtol=1e-5, atol=1e-5) ## NE
        
        # === [West]/[-x]/[0,:] edge
        
        dxA = np.diff( x2d_A[0,:] )
        dyA = np.diff( y2d_A[0,:] )
        dsA = np.sqrt(dxA**2 + dyA**2)
        sA  = np.cumsum(np.concatenate(([0.],dsA))) ## path length coord vector
        dxB = np.diff( x2d_B[0,:] )
        dyB = np.diff( y2d_B[0,:] )
        dsB = np.sqrt(dxB**2 + dyB**2)
        sB  = np.cumsum(np.concatenate(([0.],dsB))) ## path length coord vector
        np.testing.assert_allclose( sA.max() , sB.max() , rtol=1e-3 ) ## path length comparison
        
        dA_ = np.copy(data_A[0,:])
        dB_ = np.copy(data_B[0,:])
        dB_[ 0] = dA_[ 0]
        dB_[-1] = dA_[-1]
        d_func = sp.interpolate.interp1d(sA, dA_, kind=method, bounds_error=True)
        dB_[1:-1] = d_func(sB[1:-1])
        
        data_B[0,:] = dB_
        
        
        # === [East]/[+x]/[-1,:] edge
        
        dxA = np.diff( x2d_A[-1,:] )
        dyA = np.diff( y2d_A[-1,:] )
        dsA = np.sqrt(dxA**2 + dyA**2)
        sA  = np.cumsum(np.concatenate(([0.],dsA))) ## path length coord vector
        dxB = np.diff( x2d_B[-1,:] )
        dyB = np.diff( y2d_B[-1,:] )
        dsB = np.sqrt(dxB**2 + dyB**2)
        sB  = np.cumsum(np.concatenate(([0.],dsB))) ## path length coord vector
        np.testing.assert_allclose( sA.max() , sB.max() , rtol=1e-3 ) ## path length comparison
        
        dA_ = np.copy(data_A[-1,:])
        dB_ = np.copy(data_B[-1,:])
        dB_[ 0] = dA_[ 0]
        dB_[-1] = dA_[-1]
        d_func = sp.interpolate.interp1d(sA, dA_, kind=method, bounds_error=True)
        dB_[1:-1] = d_func(sB[1:-1])
        
        data_B[-1,:] = dB_
        
        
        # === [South]/[-y]/[:,0] edge
        
        dxA = np.diff( x2d_A[:,0] )
        dyA = np.diff( y2d_A[:,0] )
        dsA = np.sqrt(dxA**2 + dyA**2)
        sA  = np.cumsum(np.concatenate(([0.],dsA))) ## path length coord vector
        dxB = np.diff( x2d_B[:,0] )
        dyB = np.diff( y2d_B[:,0] )
        dsB = np.sqrt(dxB**2 + dyB**2)
        sB  = np.cumsum(np.concatenate(([0.],dsB))) ## path length coord vector
        np.testing.assert_allclose( sA.max() , sB.max() , rtol=1e-3 ) ## path length comparison
        
        dA_ = np.copy(data_A[:,0])
        dB_ = np.copy(data_B[:,0])
        dB_[ 0] = dA_[ 0]
        dB_[-1] = dA_[-1]
        d_func = sp.interpolate.interp1d(sA, dA_, kind=method, bounds_error=True)
        dB_[1:-1] = d_func(sB[1:-1])
        
        data_B[:,0] = dB_
        
        
        # === [North]/[+y]/[:,-1] edge
        
        dxA = np.diff( x2d_A[:,-1] )
        dyA = np.diff( y2d_A[:,-1] )
        dsA = np.sqrt(dxA**2 + dyA**2)
        sA  = np.cumsum(np.concatenate(([0.],dsA))) ## path length coord vector
        dxB = np.diff( x2d_B[:,-1] )
        dyB = np.diff( y2d_B[:,-1] )
        dsB = np.sqrt(dxB**2 + dyB**2)
        sB  = np.cumsum(np.concatenate(([0.],dsB))) ## path length coord vector
        np.testing.assert_allclose( sA.max() , sB.max() , rtol=1e-3 ) ## path length comparison
        
        dA_ = np.copy(data_A[:,-1])
        dB_ = np.copy(data_B[:,-1])
        dB_[ 0] = dA_[ 0]
        dB_[-1] = dA_[-1]
        d_func = sp.interpolate.interp1d(sA, dA_, kind=method, bounds_error=True)
        dB_[1:-1] = d_func(sB[1:-1])
        
        data_B[:,-1] = dB_
    
    if np.isnan(data_B).any():
        raise AssertionError('interpolated scalar field has NaNs')
    
    return data_B

def interp_1d(x1d_A, x1d_B, data_A, axis=0, **kwargs):
    '''
    interpolate 1D array 'data_A' from grid A onto grid B, yielding 'data_B'
    - data_A can be any shape, as long as size of 'axis' dim is == size of x1d_A
    - based on sp.interpolate.interp1d()
    - default 'cubic' interpolation, where NaNs occur, fill with 'nearest'
    '''
    
    method = kwargs.get('method','cubic') ## 'linear','cubic'
    
    if not isinstance(x1d_A, np.ndarray):
        raise ValueError('x1d_A should be a numpy array')
    if not isinstance(x1d_B, np.ndarray):
        raise ValueError('x1d_B should be a numpy array')
    if not isinstance(data_A, np.ndarray):
        raise ValueError('data_A should be a numpy array')
    if (x1d_A.ndim!=1):
        raise ValueError('x1d_A.ndim!=1')
    if (x1d_B.ndim!=1):
        raise ValueError('x1d_B.ndim!=1')
    #if (data_A.ndim!=1):
    #    raise ValueError('data_A.ndim!=1')
    #if ( x1d_A.shape[0] != data_A.shape[0] ):
    #    raise ValueError('x1d_A.shape[0] != data_A.shape[0]')
    
    ## < could check monotonicity >
    
    if not any([(method=='linear'),(method=='cubic')]):
        raise ValueError("'method' should be one of 'cubic' or 'linear'")
    
    intrp_func_nearest = sp.interpolate.interp1d(x1d_A, data_A, axis=axis, kind='nearest', bounds_error=False, fill_value='extrapolate')
    intrp_func         = sp.interpolate.interp1d(x1d_A, data_A, axis=axis, kind=method,    bounds_error=False, fill_value=np.nan)
    
    B_nearest = intrp_func_nearest(x1d_B)
    B         = intrp_func(x1d_B)
    
    #n_nans         = np.count_nonzero(np.isnan(B))
    #n_nans_nearest = np.count_nonzero(np.isnan(B_nearest))
    
    data_B = np.where( np.isnan(B), B_nearest, B)
    
    if np.isnan(data_B).any():
        raise AssertionError('interpolated scalar field has NaNs')
    
    return data_B

def fd_coeff_calculator(stencil, d=1, x=None, dx=None):
    '''
    Calculate Finite Difference Coefficients for Arbitrary Stencil
    -----
    stencil : indices of stencil pts e.g. np.array([-2,-1,0,1,2])
    d       : derivative order
    x       : locations of grid points corresponding to stencil indices
    dx      : spacing of grid points in the case of uniform grid
    -----
    https://en.wikipedia.org/wiki/Finite_difference_coefficient
    https://web.media.mit.edu/~crtaylor/calculator.html
    -----
    Fornberg B. (1988) Generation of Finite Difference Formulas on
    Arbitrarily Spaced Grids, Mathematics of Computation 51, no. 184 : 699-706.
    http://www.ams.org/journals/mcom/1988-51-184/S0025-5718-1988-0935077-0/S0025-5718-1988-0935077-0.pdf
    '''
    
    stencil = np.asanyarray(stencil)
    
    if not isinstance(stencil, np.ndarray):
        raise ValueError('stencil should be of type np.ndarray')
    if (stencil.ndim!=1):
        raise ValueError('stencil should be 1D')
    if (stencil.shape[0]<2):
        raise ValueError('stencil size should be >=2')
    if (0 not in stencil):
        raise ValueError('stencil does not contain 0')
    if not np.issubdtype(stencil.dtype, np.integer):
        raise ValueError('stencil.dtype not a subdtype of np.integer')
    
    if not isinstance(d, int):
        raise ValueError('d (derivative order) should be of type int')
    if not (d>0):
        raise ValueError('d (derivative order) should be >0')
    
    if (dx is None) and (x is None):
        raise ValueError('one of args \'dx\' or \'x\' should be defined')
    if (dx is not None) and (x is not None):
        raise ValueError('only one of args \'dx\' or \'x\' should be defined')
    if (dx is not None):
        if not isinstance(dx, float):
            raise ValueError('dx should be of type float')
    
    if (x is not None):
        if not isinstance(x, np.ndarray):
            raise ValueError('x should be of type np.ndarray')
        if (x.shape[0] != stencil.shape[0]):
            raise ValueError('x, stencil should have same shape')
        if (not np.all(np.diff(x) > 0.)) and (not np.all(np.diff(x) < 0.)):
            raise AssertionError('x is not monotonically increasing/decreasing')
    
    ## overwrite stencil (int index) to be coordinate array (delta from 0 position)
    
    i0 = np.where(stencil==0)[0][0]
    
    if (x is not None):
        stencil = x - x[i0] 
    
    if (dx is not None):
        stencil = dx * stencil.astype(np.float64)
    
    nn = stencil.shape[0]
    
    dvec = np.zeros( (nn,) , dtype=np.float64 )
    #dvec = np.zeros( (nn,) , dtype=np.longdouble )
    dfac=1
    for i in range(d):
        dfac *= (i+1)
    dvec[d] = dfac
    
    ## increase precision
    #stencil = np.copy(stencil).astype(np.longdouble)
    
    stencil_abs_max         = np.abs(stencil).max()
    stencil_abs_min_nonzero = np.abs(stencil[[ i for i in range(stencil.size) if i!=i0 ]]).min()
    
    '''
    scale/normalize the coordinate stencil (to avoid ill-conditioning)
    - if coordinates are already small/large, the Vandermonde matrix becomes
       HIGHLY ill-conditioned due to row exponents
    - coordinates are normalized here so that smallest absolute non-zero delta coord. is =1
    - RHS vector (dvec) gets normalized too
    - FD coefficients are (theoretically) unaffected
    '''
    normalize_stencil = True
    
    if normalize_stencil:
        stencil /= stencil_abs_min_nonzero
    
    mat = np.zeros( (nn,nn) , dtype=np.float64)
    #mat = np.zeros( (nn,nn) , dtype=np.longdouble)
    for i in range(nn):
        mat[i,:] = np.power( stencil , i )
    
    ## condition_number = np.linalg.cond(mat, p=-2)
    
    # mat_inv = np.linalg.inv( mat )
    # coeffv  = np.dot( mat_inv , dvec )
    
    if normalize_stencil:
        for i in range(nn):
            dvec[i] /= np.power( stencil_abs_min_nonzero , i )
    
    #coeffv = np.linalg.solve(mat, dvec)
    coeffv = sp.linalg.solve(mat, dvec)
    
    return coeffv

def assemble_1d_fd_coeff_vector_central(x=None, dx=None, edge_stencil='full', acc=4, d=1):
    '''
    assemble FD coefficients for 1D coordinate vector using central discretization
    -----
    d : derivative order
    x : 1D coordinate array
    ...
    '''
    
    ## checks
    
    uniform_grid = False ## tmp
    nx = x.shape[0]
    
    fdc_vec = [] ## vector of finite difference coefficient information to be returned
    
    ## for the d'th derivative with accuracy=acc, the following formula gives the n pts of the (central) stencil
    stencil_npts = 2*int(np.floor((d+1)/2)) - 1 + acc
    
    if not isinstance(stencil_npts, int):
        raise ValueError('stencil_npts must be of type \'int\'')
    if (stencil_npts<3):
        raise ValueError('stencil_npts should be >=3')
    if ((stencil_npts-1)%2 != 0):
        raise ValueError('(stencil_npts-1) should be divisible by 2 (for central stencil)')
    if (stencil_npts > nx):
        raise ValueError('stencil_npts > nx')
    
    if all([ (edge_stencil!='half') , (edge_stencil!='full') ]):
        raise ValueError('edge_stencil=%s not valid. options are: \'full\', \'half\''%str(edge_stencil))
    
    # ===
    
    n_full_central_stencils = nx - stencil_npts + 1
    
    # if ( n_full_central_stencils < 5 ) and not no_warn:
    #     print('\nWARNING\n'+72*'-')
    #     print('n pts with full central stencils = %i (<5)'%n_full_central_stencils)
    #     #print('nx//3=%i'%(nx//3))
    #     print('--> consider reducing acc arg (accuracy order)')
    #     print(72*'-'+'\n')
    
    stencil_width = stencil_npts-1
    sw2           = stencil_width//2
    
    ## left side
    for i in range(0,sw2):
        
        if (edge_stencil=='half'):
            stencil_L = np.arange(-i,sw2+1)
        elif (edge_stencil=='full'):
            stencil_L = np.arange(-i,stencil_width+1-i)
        else:
            raise ValueError('edge_stencil options are: \'full\', \'half\'')
        
        i_range = np.arange( 0 , stencil_L.shape[0] )
        
        if uniform_grid:
            fdc = fd_coeff_calculator( stencil_L , d=d , dx=x )
        else:
            fdc = fd_coeff_calculator( stencil_L , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil_L ] )
    
    ## inner pts
    stencil = np.arange(stencil_npts) - sw2
    if uniform_grid:
        fdc_inner = fd_coeff_calculator( stencil , d=d , dx=x )
    for i in range(sw2,nx-sw2):
        
        i_range  = np.arange(i-sw2,i+sw2+1)
        
        if uniform_grid:
            fdc = fdc_inner
        else:
            fdc = fd_coeff_calculator( stencil , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil ] )
    
    ## right side
    for i in range(nx-sw2,nx):
        
        if (edge_stencil=='half'):
            stencil_R = np.arange(-sw2,nx-i)
        elif (edge_stencil=='full'):
            stencil_R = np.arange(-stencil_width+(nx-i-1),nx-i)
        else:
            raise ValueError('edge_stencil options are: \'full\', \'half\'')
        
        i_range  = np.arange( nx-stencil_R.shape[0] , nx )
        
        if uniform_grid:
            fdc = fd_coeff_calculator( stencil_R , d=d , dx=x )
        else:
            fdc = fd_coeff_calculator( stencil_R , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil_R ] )
    
    
    return fdc_vec

def assemble_1d_fd_coeff_vector_custom(x, stencil_base, d=1):
    '''
    assemble FD coefficients for 1D coordinate vector using a custom stencil index
    -----
    stencil_base : stencil index array, e.g. np.array([-1,0,1,2,3],dtype=np.int32)
    d            : derivative order
    x            : 1D coordinate array
    '''
    
    ## checks
    if not isinstance(x, np.ndarray):
        raise ValueError("x must be type np.ndarray")
    if not (x.ndim==1):
        raise ValueError("x must be 1D")
    if not isinstance(stencil_base, np.ndarray):
        raise ValueError("stencil_base must be type np.ndarray")
    if not (stencil_base.ndim==1):
        raise ValueError("stencil_base must be 1D")
    if not np.issubdtype(stencil_base.dtype, np.integer):
        raise ValueError('stencil_base.dtype not a subdtype of np.integer')
    if (0 not in stencil_base):
        raise ValueError('stencil_base does not contain 0')
    if not np.all(np.diff(stencil_base)==1):
        raise ValueError("stencil_base must have np.all(np.diff(stencil_base)==1")
    
    nx = x.shape[0]            ## size of coord vector
    ss = stencil_base.shape[0] ## size of stencil
    
    if (ss>nx):
        raise ValueError('stencil is larger than the coordinate vector')
    
    i0  = np.where(stencil_base==0)[0][0] ## location of 0 in stencil_base
    niL = i0                              ## n stencil indices to the left (L) of 0
    niR = ss-i0-1                         ## n stencil indices to the right (R) of 0
    
    fdc_vec = [] ## vector of finite difference coefficient information to be returned
    for i in range(nx):
        
        iR = nx-i-1 ## decrementer --> [nx-1...0] --> 0 at last loop index
        
        shift=0
        isL=False
        isR=False
        if (niL>i): ## left edge
            shift = niL-i
            isL=True
        if (niR>iR): ## right edge
            shift = iR-niR
            isR=True
        
        if (isL and isR):
            raise ValueError('something went wrong in assemble_1d_fd_coeff_vector_custom()')
        
        stencil_ = np.copy(stencil_base + shift)
        
        ## the indices corresponding to coordinate vector [x]
        i_range = np.arange(i+stencil_.min(),i+stencil_.max()+1)
        
        ## get actual coefficients
        fdc = fd_coeff_calculator( stencil_, d=d, x=x[i_range] )
        
        ## add all info to output list
        fdc_vec.append( [ fdc , i_range , stencil_ ] )
    
    return fdc_vec

def gradient(u, x=None, d=1, axis=0, acc=6, edge_stencil='full', return_coeffs=False, no_warn=True):
    '''
    Numerical Gradient Approximation Using Finite Differences
    -----
    - calculates stencil given arbitrary accuracy & derivative order
    - handles non-uniform grids
    - accuracy order is only mathematically valid for:
       - uniform coordinate array
       - inner points which have full central stencil
    - handles N-D numpy arrays (gradient performed over axis denoted by axis arg)
    -----
    u    : input array to perform differentiation upon
    x    : coordinate vector (np.ndarray) OR dx (float) in the case of a uniform grid
    d    : derivative order
    axis : axis along which to perform gradient
    acc  : accuracy order (only fully valid for inner points with central stencil on uniform grid)
    -----
    edge_stencil  : type of edge stencil to use ('half','full')
    return_coeffs : if True, then return stencil & coefficient information
    -----
    # stencil_npts : number of index pts in (central) stencil
    #     --> no longer an input
    #     --> using 'acc' (accuracy order) instead and calculating npts from formula
    #     - stencil_npts=3 : stencil=[      -1,0,+1      ]
    #     - stencil_npts=5 : stencil=[   -2,-1,0,+1,+2   ]
    #     - stencil_npts=7 : stencil=[-3,-2,-1,0,+1,+2,+3]
    #     - edges are filled out with appropriate clipping of central stencil
    -----
    turbx.gradient( u , x , d=1 , acc=2 , edge_stencil='half' , axis=0 )
    ...reproduces...
    np.gradient(u, x, edge_order=1, axis=0)
    
    turbx.gradient( u , x , d=1 , acc=2 , edge_stencil='full' , axis=0 )
    ...reproduces...
    np.gradient(u, x, edge_order=2, axis=0)
    -----
    https://en.wikipedia.org/wiki/Finite_difference_coefficient
    https://web.media.mit.edu/~crtaylor/calculator.html
    -----
    Fornberg B. (1988) Generation of Finite Difference Formulas on
    Arbitrarily Spaced Grids, Mathematics of Computation 51, no. 184 : 699-706.
    http://www.ams.org/journals/mcom/1988-51-184/S0025-5718-1988-0935077-0/S0025-5718-1988-0935077-0.pdf
    '''
    
    u = np.asanyarray(u)
    nd = u.ndim
    
    # print('contiguous   : %s'%str( u.data.contiguous   ) )
    # print('C-contiguous : %s'%str( u.data.c_contiguous ) )
    # print('F-contiguous : %s'%str( u.data.f_contiguous ) )
    
    if (nd==0):
        raise ValueError('turbx.gradient() requires input that is at least 1D')
    
    axes = tuple(range(nd))
    
    if not isinstance(axis, int):
        raise ValueError('axis should be of type int')
    if (axis not in axes):
        raise ValueError('axis=%i is not valid for array with u.ndim=%s'%(axis,str(u.ndim)))
    
    nx = u.shape[axis] ## size of axis over which gradient will be performed
    
    if (nx<3):
        raise ValueError('nx<3')
    
    if (x is not None):
        if isinstance(x, float):
            if (x<=0.):
                raise ValueError('if x is a float it should be >0.')
        elif isinstance(x, int):
            x = float(x)
        elif isinstance(x, np.ndarray):
            if (x.ndim!=1):
                raise ValueError('x should be 1D if it is of type np.ndarray')
            if (x.shape[0]!=nx):
                raise ValueError('size of x does not match data axis specified')
            if (not np.all(np.diff(x) > 0.)) and (not np.all(np.diff(x) < 0.)):
                    raise AssertionError('x is not monotonically increasing/decreasing')
            
            ## optimization: check if x is actually uniformly spaced, in which case x=Δx
            dx0 = x[1]-x[0]
            if np.all(np.isclose(np.diff(x), dx0, rtol=1e-12)): 
                #print('turbx.gradient() : x arr with x.shape=%s seems like it is actually uniformly spaced. applying x=%0.8e'%(str(x.shape),dx0))
                x = dx0
        
        else:
            raise ValueError('x should be a 1D np.ndarray or float')
    else:
        x = 1. ## if x not provided, assume uniform unit coordinate vector
    
    if isinstance(x, float):
        uniform_grid = True
    elif isinstance(x, np.ndarray):
        uniform_grid = False
    else:
        raise ValueError('turbx.gradient() : this should never happen... check!')
    
    if not isinstance(d, int):
        raise ValueError('d (derivative order) should be of type int')
    if not (d>0):
        raise ValueError('d (derivative order) should be >0')
    
    if not isinstance(acc, int):
        raise ValueError('acc (accuracy order) should be of type int')
    if not (acc>=2):
        raise ValueError('acc (accuracy order) should be >=2')
    if (acc%2!=0):
        raise ValueError('acc (accuracy order) should be an integer multiple of 2')
    
    ## for the d'th derivative with accuracy=acc, the following formula gives the n pts of the (central) stencil
    stencil_npts = 2*int(np.floor((d+1)/2)) - 1 + acc
    
    if not isinstance(stencil_npts, int):
        raise ValueError('stencil_npts must be of type \'int\'')
    if (stencil_npts<3):
        raise ValueError('stencil_npts should be >=3')
    if ((stencil_npts-1)%2 != 0):
        raise ValueError('(stencil_npts-1) should be divisible by 2 (for central stencil)')
    if (stencil_npts > nx):
        raise ValueError('stencil_npts > nx')
    
    if all([ (edge_stencil!='half') , (edge_stencil!='full') ]):
        raise ValueError('edge_stencil=%s not valid. options are: \'full\', \'half\''%str(edge_stencil))
    
    # ===
    
    n_full_central_stencils = nx - stencil_npts + 1
    
    if ( n_full_central_stencils < 5 ) and not no_warn:
        print('\nWARNING\n'+72*'-')
        print('n pts with full central stencils = %i (<5)'%n_full_central_stencils)
        #print('nx//3=%i'%(nx//3))
        print('--> consider reducing acc arg (accuracy order)')
        print(72*'-'+'\n')
    
    stencil_width = stencil_npts-1
    sw2           = stencil_width//2
    
    # === build up stencil & coefficients vector
    
    fdc_vec = [] ## vector of finite difference coefficient information
    
    ## left side
    for i in range(0,sw2):
        
        ## full
        stencil_L = np.arange(-i,stencil_width+1-i)
        
        ## half
        if (edge_stencil=='half') and not all([(acc==2),(d>=2)]):
            stencil_L = np.arange(-i,sw2+1)
        
        i_range = np.arange( 0 , stencil_L.shape[0] )
        
        if uniform_grid:
            fdc = fd_coeff_calculator( stencil_L , d=d , dx=x )
        else:
            fdc = fd_coeff_calculator( stencil_L , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil_L ] )
    
    ## inner pts
    stencil = np.arange(stencil_npts) - sw2
    if uniform_grid:
        fdc_inner = fd_coeff_calculator( stencil , d=d , dx=x )
    for i in range(sw2,nx-sw2):
        
        i_range  = np.arange(i-sw2,i+sw2+1)
        
        if uniform_grid:
            fdc = fdc_inner
        else:
            fdc = fd_coeff_calculator( stencil , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil ] )
    
    ## right side
    for i in range(nx-sw2,nx):
        
        ## full
        stencil_R = np.arange(-stencil_width+(nx-i-1),nx-i)
        
        ## half
        if (edge_stencil=='half') and not all([(acc==2),(d>=2)]):
            stencil_R = np.arange(-sw2,nx-i)
        
        i_range  = np.arange( nx-stencil_R.shape[0] , nx )
        
        if uniform_grid:
            fdc = fd_coeff_calculator( stencil_R , d=d , dx=x )
        else:
            fdc = fd_coeff_calculator( stencil_R , d=d , x=x[i_range] )
        
        fdc_vec.append( [ fdc , i_range , stencil_R ] )
    
    # === debug
    
    # print('i_range')
    # for fdc_vec_ in fdc_vec:
    #     print(fdc_vec_[1])
    # print('')
    # print('stencil')
    # for fdc_vec_ in fdc_vec:
    #     print(fdc_vec_[2])
    
    # === evaluate gradient
    
    if (nd==1): ## 1D
        
        u_ddx = np.zeros_like(u)
        for i in range(len(fdc_vec)):
            fdc, i_range, stencil = fdc_vec[i]
            u_ddx[i] = np.dot( fdc , u[i_range] )
    
    else: ## N-D
        
        ## the order in memory of the incoming data
        if u.data.contiguous:
            if u.data.c_contiguous:
                order='C'
            elif u.data.f_contiguous:
                order='F'
            else:
                raise ValueError
        else:
            order='C'
        
        ## if the array is C-ordered, use axis=0 as the axis to shift to
        ## if the array is F-ordered, use axis=-1 as the axis to shift to
        if order=='C':
            shift_pos=0 ## 0th axis
        elif order=='F':
            shift_pos=nd-1 ## last axis
        
        ## shift gradient axis
        u = np.swapaxes(u, axis, shift_pos)
        shape_new = u.shape
        if (shift_pos==0):
            size_all_but_ax = np.prod(np.array(shape_new)[1:])
        elif (shift_pos==nd-1):
            size_all_but_ax = np.prod(np.array(shape_new)[:-1])
        else:
            raise ValueError
        
        ## reshape N-D to 2D
        ## --> shift_pos=0    : gradient axis is 0, all other axes are flattened on axis=1)
        ## --> shift_pos=last : ...
        if (shift_pos==0):
            u = np.reshape(u, (nx, size_all_but_ax), order=order)
        elif (shift_pos==nd-1):
            u = np.reshape(u, (size_all_but_ax, nx), order=order)
        else:
            raise ValueError
        
        u_ddx = np.zeros_like(u,order=order)
        for i in range(nx):
            fdc, i_range, stencil = fdc_vec[i]
            ia=min(i_range)
            ib=max(i_range)+1
            if (shift_pos==0):
                u_ddx[i,:] = np.einsum('ij,i->j', u[ia:ib,:], fdc)
            else:
                u_ddx[:,i] = np.einsum('ji,i->j', u[:,ia:ib], fdc)
        
        ## reshape 2D back to original N-D
        u_ddx = np.reshape(u_ddx, shape_new, order=order)
        
        ## shift gradient axis back to original position
        u_ddx = np.swapaxes(u_ddx, shift_pos, axis)
    
    ## the original data array should have been de-referenced during this func
    u = None; del u
    
    if return_coeffs:
        return u_ddx, fdc_vec
    else:
        return u_ddx

def get_metric_tensor_3d(x3d, y3d, z3d, acc=2, edge_stencil='full', **kwargs):
    '''
    compute the grid metric tensor (inverse of grid Jacobian) for a 3D grid
    -----
    Computational Fluid Mechanics and Heat Transfer (2012) Pletcher, Tannehill, Anderson
    p.266-270, 335-337, 652
    '''
    
    verbose = kwargs.get('verbose',False)
    no_warn = kwargs.get('no_warn',False)
    
    if not isinstance(x3d, np.ndarray):
        raise ValueError('x3d should be of type np.ndarray')
    if not isinstance(y3d, np.ndarray):
        raise ValueError('y3d should be of type np.ndarray')
    if not isinstance(z3d, np.ndarray):
        raise ValueError('z3d should be of type np.ndarray')
    
    if (x3d.ndim!=3):
        raise ValueError('x3d should have ndim=3 (xyz)')
    if (y3d.ndim!=3):
        raise ValueError('y3d should have ndim=3 (xyz)')
    if (z3d.ndim!=3):
        raise ValueError('z3d should have ndim=3 (xyz)')
    
    if not (x3d.shape==y3d.shape):
        raise ValueError('x3d.shape!=y3d.shape')
    if not (y3d.shape==z3d.shape):
        raise ValueError('y3d.shape!=z3d.shape')
    
    nx,ny,nz = x3d.shape
    
    ## the 'computational' grid (unit Cartesian)
    ## --> [x_comp,y_comp,z_comp ]= [ξ,η,ζ] = [q1,q2,q3]
    #x_comp = np.arange(nx, dtype=np.float64)
    #y_comp = np.arange(ny, dtype=np.float64)
    #z_comp = np.arange(nz, dtype=np.float64)
    x_comp = 1.
    y_comp = 1.
    z_comp = 1.
    
    # === get Jacobian :: ∂(x,y,z)/∂(q1,q2,q3)
    
    t_start = timeit.default_timer()
    
    dxdx = gradient(x3d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydx = gradient(y3d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dzdx = gradient(z3d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    ##
    dxdy = gradient(x3d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydy = gradient(y3d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dzdy = gradient(z3d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    ##
    dxdz = gradient(x3d, z_comp, axis=2, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydz = gradient(y3d, z_comp, axis=2, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dzdz = gradient(z3d, z_comp, axis=2, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    
    J = np.stack((np.stack((dxdx, dydx, dzdx), axis=3),
                  np.stack((dxdy, dydy, dzdy), axis=3),
                  np.stack((dxdz, dydz, dzdz), axis=3)), axis=4)
    
    t_delta = timeit.default_timer() - t_start
    if verbose: tqdm.write( even_print('get J','%0.3f [s]'%(t_delta,), s=True) )
    
    # === get metric tensor M = J^-1 = ∂(q1,q2,q3)/∂(x,y,z) = ∂(ξ,η,ζ)/∂(x,y,z)
    
    if False: ## method 1
        
        t_start = timeit.default_timer()
        
        M = np.linalg.inv(J)
        
        # M_bak = np.copy(M)
        # for i in range(nx):
        #     for j in range(ny):
        #         for k in range(nz):
        #             M[i,j,k,:,:] = sp.linalg.inv( J[i,j,k,:,:] )
        # np.testing.assert_allclose(M_bak, M, atol=1e-12, rtol=1e-12)
        # print('check passed')
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get M','%0.3f [s]'%(t_delta,), s=True) )
    
    if True: ## method 2
        
        if ('M' in locals()):
            M_bak = np.copy(M)
            M = None; del M
        
        t_start = timeit.default_timer()
        
        a = J[:,:,:,0,0]
        b = J[:,:,:,0,1]
        c = J[:,:,:,0,2]
        d = J[:,:,:,1,0]
        e = J[:,:,:,1,1]
        f = J[:,:,:,1,2]
        g = J[:,:,:,2,0]
        h = J[:,:,:,2,1]
        i = J[:,:,:,2,2]
        
        # a = J[:,:,:,0,0]
        # b = J[:,:,:,1,0]
        # c = J[:,:,:,2,0]
        # d = J[:,:,:,0,1]
        # e = J[:,:,:,1,1]
        # f = J[:,:,:,2,1]
        # g = J[:,:,:,0,2]
        # h = J[:,:,:,1,2]
        # i = J[:,:,:,2,2]
        
        I = ( + a*e*i
              + b*f*g
              + c*d*h
              - c*e*g
              - b*d*i
              - a*f*h )
        
        M = np.zeros((nx,ny,nz,3,3), dtype=np.float64)
        M[:,:,:,0,0] = +( dydy * dzdz - dydz * dzdy ) / I ## ξ_x
        M[:,:,:,0,1] = -( dxdy * dzdz - dxdz * dzdy ) / I ## ξ_y
        M[:,:,:,0,2] = +( dxdy * dydz - dxdz * dydy ) / I ## ξ_z
        M[:,:,:,1,0] = -( dydx * dzdz - dydz * dzdx ) / I ## η_x
        M[:,:,:,1,1] = +( dxdx * dzdz - dxdz * dzdx ) / I ## η_y
        M[:,:,:,1,2] = -( dxdx * dydz - dxdz * dydx ) / I ## η_z
        M[:,:,:,2,0] = +( dydx * dzdy - dydy * dzdx ) / I ## ζ_x
        M[:,:,:,2,1] = -( dxdx * dzdy - dxdy * dzdx ) / I ## ζ_y
        M[:,:,:,2,2] = +( dxdx * dydy - dxdy * dydx ) / I ## ζ_z
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get M','%0.3f [s]'%(t_delta,), s=True) )
        
        if ('M_bak' in locals()):
            np.testing.assert_allclose(M[:,:,:,0,0], M_bak[:,:,:,0,0], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_x')
            np.testing.assert_allclose(M[:,:,:,0,1], M_bak[:,:,:,0,1], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_y')
            np.testing.assert_allclose(M[:,:,:,0,2], M_bak[:,:,:,0,2], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_z')
            np.testing.assert_allclose(M[:,:,:,1,0], M_bak[:,:,:,1,0], atol=1e-14, rtol=1e-14)
            print('check passed: η_x')
            np.testing.assert_allclose(M[:,:,:,1,1], M_bak[:,:,:,1,1], atol=1e-14, rtol=1e-14)
            print('check passed: η_y')
            np.testing.assert_allclose(M[:,:,:,1,2], M_bak[:,:,:,1,2], atol=1e-14, rtol=1e-14)
            print('check passed: η_z')
            np.testing.assert_allclose(M[:,:,:,2,0], M_bak[:,:,:,2,0], atol=1e-14, rtol=1e-14)
            print('check passed: ζ_x')
            np.testing.assert_allclose(M[:,:,:,2,1], M_bak[:,:,:,2,1], atol=1e-14, rtol=1e-14)
            print('check passed: ζ_y')
            np.testing.assert_allclose(M[:,:,:,2,2], M_bak[:,:,:,2,2], atol=1e-14, rtol=1e-14)
            print('check passed: ζ_z')
            np.testing.assert_allclose(M, M_bak, atol=1e-14, rtol=1e-14)
            print('check passed: M')
    
    return M

def get_metric_tensor_2d(x2d, y2d, acc=2, edge_stencil='full', **kwargs):
    '''
    compute the grid metric tensor (inverse of grid Jacobian) for a 2D grid
    -----
    Computational Fluid Mechanics and Heat Transfer (2012) Pletcher, Tannehill, Anderson
    p.266-270, 335-337, 652
    '''
    
    verbose = kwargs.get('verbose',False)
    no_warn = kwargs.get('no_warn',False)
    
    if not isinstance(x2d, np.ndarray):
        raise ValueError('x2d should be of type np.ndarray')
    if not isinstance(y2d, np.ndarray):
        raise ValueError('y2d should be of type np.ndarray')
    
    if (x2d.ndim!=2):
        raise ValueError('x2d should have ndim=2 (xy)')
    if (y2d.ndim!=2):
        raise ValueError('y2d should have ndim=2 (xy)')
    
    if not (x2d.shape==y2d.shape):
        raise ValueError('x2d.shape!=y2d.shape')
    
    nx,ny = x2d.shape
    
    ## the 'computational' grid (unit Cartesian)
    ## --> [x_comp,y_comp]= [ξ,η] = [q1,q2]
    #x_comp = np.arange(nx, dtype=np.float64)
    #y_comp = np.arange(ny, dtype=np.float64)
    x_comp = 1.
    y_comp = 1.
    
    # === get Jacobian :: ∂(x,y)/∂(q1,q2)
    
    t_start = timeit.default_timer()
    
    dxdx = gradient(x2d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydx = gradient(y2d, x_comp, axis=0, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dxdy = gradient(x2d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    dydy = gradient(y2d, y_comp, axis=1, d=1, acc=acc, edge_stencil=edge_stencil, no_warn=no_warn)
    
    J = np.stack((np.stack((dxdx, dydx), axis=2),
                  np.stack((dxdy, dydy), axis=2)), axis=3)
    
    t_delta = timeit.default_timer() - t_start
    if verbose: tqdm.write( even_print('get J','%0.3f [s]'%(t_delta,), s=True) )
    
    # === get metric tensor M = J^-1 = ∂(q1,q2)/∂(x,y) = ∂(ξ,η)/∂(x,y)
    
    if False: ## method 1
        
        t_start = timeit.default_timer()
        
        M = np.linalg.inv(J)
        
        # M_bak = np.copy(M)
        # M = np.zeros((nx,ny,2,2),dtype=np.float64)
        # for i in range(nx):
        #     for j in range(ny):
        #         M[i,j,:,:] = sp.linalg.inv( J[i,j,:,:] )
        # np.testing.assert_allclose(M_bak, M, atol=1e-12, rtol=1e-12)
        # print('check passed')
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get M','%0.3f [s]'%(t_delta,), s=True) )
    
    if True: ## method 2
        
        if ('M' in locals()):
            M_bak = np.copy(M)
            M = None; del M
        
        t_start = timeit.default_timer()
        
        ## Jacobian determinant
        I = dxdx*dydy - dydx*dxdy
        
        # I_bak = np.copy(I)
        # I = None; del I
        # I = np.linalg.det(J)
        # np.testing.assert_allclose(I, I_bak, atol=1e-14, rtol=1e-14)
        # print('check passed')
        
        M = np.zeros((nx,ny,2,2), dtype=np.float64)
        M[:,:,0,0] = +dydy / I ## ξ_x
        M[:,:,0,1] = -dxdy / I ## ξ_y
        M[:,:,1,0] = -dydx / I ## η_x
        M[:,:,1,1] = +dxdx / I ## η_y
        
        t_delta = timeit.default_timer() - t_start
        if verbose: tqdm.write( even_print('get M','%0.3f [s]'%(t_delta,), s=True) )
        
        if ('M_bak' in locals()):
            np.testing.assert_allclose(M[:,:,0,0], M_bak[:,:,0,0], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_x')
            np.testing.assert_allclose(M[:,:,0,1], M_bak[:,:,0,1], atol=1e-14, rtol=1e-14)
            print('check passed: ξ_y')
            np.testing.assert_allclose(M[:,:,1,0], M_bak[:,:,1,0], atol=1e-14, rtol=1e-14)
            print('check passed: η_x')
            np.testing.assert_allclose(M[:,:,1,1], M_bak[:,:,1,1], atol=1e-14, rtol=1e-14)
            print('check passed: η_y')
            np.testing.assert_allclose(M, M_bak, atol=1e-14, rtol=1e-14)
            print('check passed: M')
    
    return M

def get_grid_quality_metrics_2d(x2d, y2d, **kwargs):
    '''
    get 2d grid quality metrics
    -----
    - skew
    - avg diagonal length (not yet implemented)
    -----
    ## https://coreform.com/cubit_help/mesh_generation/mesh_quality_assessment/quadrilateral_metrics.htm
    '''
    
    verbose = kwargs.get('verbose',True)
    
    if not isinstance(x2d, np.ndarray):
        raise ValueError('x2d should be of type np.ndarray')
    if not isinstance(y2d, np.ndarray):
        raise ValueError('y2d should be of type np.ndarray')
    
    if (x2d.ndim!=2):
        raise ValueError('x2d should have ndim=2 (xy)')
    if (y2d.ndim!=2):
        raise ValueError('y2d should have ndim=2 (xy)')
    
    if not (x2d.shape==y2d.shape):
        raise ValueError('x2d.shape!=y2d.shape')
    
    nx,ny = x2d.shape
    xy2d = np.stack((x2d,y2d), axis=-1)
    
    '''
    there are very likely ways to vectorize the functions below
    '''
    
    ds1avg = np.zeros((nx,ny), dtype=np.float64)
    ds2avg = np.zeros((nx,ny), dtype=np.float64)
    
    if verbose: progress_bar = tqdm(total=(nx-1)*(ny-1), ncols=100, desc='grid ds', leave=False, file=sys.stdout)
    for i in range(nx):
        for j in range(ny):
            
            ## W/E ds
            if (i==0):
                dsW = None
                dsE = sp.linalg.norm( xy2d[i+1,j,:] - xy2d[i,  j,:], ord=2)
                ds1avg[i,j] = dsE
            elif (i==nx-1):
                dsW = sp.linalg.norm( xy2d[i,  j,:] - xy2d[i-1,j,:], ord=2)
                dsE = None
                ds1avg[i,j] = dsW
            else:
                dsW = sp.linalg.norm( xy2d[i,  j,:] - xy2d[i-1,j,:], ord=2)
                dsE = sp.linalg.norm( xy2d[i+1,j,:] - xy2d[i,  j,:], ord=2)
                ds1avg[i,j] = 0.5*(dsW+dsE)
            
            ## S/N ds
            if (j==0):
                dsS = None
                dsN = sp.linalg.norm( xy2d[i,j+1,:] - xy2d[i,  j,:], ord=2)
                ds2avg[i,j] = dsN
            elif (j==ny-1):
                dsS = sp.linalg.norm( xy2d[i,  j,:] - xy2d[i,j-1,:], ord=2)
                dsN = None
                ds2avg[i,j] = dsS
            else:
                dsS = sp.linalg.norm( xy2d[i,  j,:] - xy2d[i,j-1,:], ord=2)
                dsN = sp.linalg.norm( xy2d[i,j+1,:] - xy2d[i,  j,:], ord=2)
                ds2avg[i,j] = 0.5*(dsS+dsN)
            
            if verbose: progress_bar.update()
    if verbose: progress_bar.close()
    
    grid_inner_angle_cosines = np.zeros((nx-1,ny-1,4), dtype=np.float64)
    if verbose: progress_bar = tqdm(total=(nx-1)*(ny-1), ncols=100, desc='grid skew angle', leave=False, file=sys.stdout)
    for i in range(nx-1):
        for j in range(ny-1):
            ## (SW-->NW)::(SW-->SE)
            v1 = xy2d[i,j+1,:] - xy2d[i,j,:]
            v2 = xy2d[i+1,j,:] - xy2d[i,j,:]
            v1mag = sp.linalg.norm(v1, ord=2)
            v2mag = sp.linalg.norm(v2, ord=2)
            grid_inner_angle_cosines[i,j,0] = np.abs(np.dot(v1,v2)/(v1mag*v2mag))
            ## (SE-->SW)::(SE-->NE)
            v1 = xy2d[i,j,:]     - xy2d[i+1,j,:]
            v2 = xy2d[i+1,j+1,:] - xy2d[i+1,j,:]
            v1mag = sp.linalg.norm(v1, ord=2)
            v2mag = sp.linalg.norm(v2, ord=2)
            grid_inner_angle_cosines[i,j,1] = np.abs(np.dot(v1,v2)/(v1mag*v2mag))
            ## (NE-->NW)::(NE-->SE)
            v1 = xy2d[i,j+1,:] - xy2d[i+1,j+1,:]
            v2 = xy2d[i+1,j,:] - xy2d[i+1,j+1,:]
            v1mag = sp.linalg.norm(v1, ord=2)
            v2mag = sp.linalg.norm(v2, ord=2)
            grid_inner_angle_cosines[i,j,2] = np.abs(np.dot(v1,v2)/(v1mag*v2mag))
            ## (NW-->NE)::(NW-->SW)
            v1 = xy2d[i+1,j+1,:] - xy2d[i,j+1,:]
            v2 = xy2d[i,j,:]     - xy2d[i,j+1,:]
            v1mag = sp.linalg.norm(v1, ord=2)
            v2mag = sp.linalg.norm(v2, ord=2)
            grid_inner_angle_cosines[i,j,3] = np.abs(np.dot(v1,v2)/(v1mag*v2mag))
            ##
            if verbose: progress_bar.update()
    if verbose: progress_bar.close()
    
    skew = np.max(grid_inner_angle_cosines, axis=-1)
    
    grid_quality_dict = { 'skew':skew, 'ds1avg':ds1avg, 'ds2avg':ds2avg }
    return grid_quality_dict

def smoothstep(N, a=None, b=None, order=3, mode='index'):
    '''
    a smoothed step function that is differentiable (order) times
    - actually starts/ends at identically 0/1 --> non-asymptotic (unlike e.g. sigmoid)
    - based on Hermite polynomials
    -----
    index : N should be an int... returns funcs on array of size N (independent of grid)
    coord_vec : N should be a numpy array describing coordinates (a,b) are bounds of step in coord vec
    '''
    if (mode=='index'):
        if (b is None):
            b=N-1
        if (a is None):
            a=0
        if not all([isinstance(N,int),isinstance(a,int),isinstance(b,int)]):
            raise ValueError('N,a,b must all be type int for mode=\'index\'')
        x = np.arange(0,N,1,dtype=np.float64)
        a = x[a]
        b = x[b]
    elif (mode=='coord_vec'):
        if not (isinstance(N,np.ndarray)):
            raise ValueError('N should be a numpy array.')
        x = np.copy(N)
        if (b is None):
            b=x[-1]
        if (a is None):
            a=x[0]
    else:
        raise ValueError('mode=\'%s\' not a valid input. options are: \'index\' \'coord_vec\' '%str(mode))
    
    x = np.clip((x-a)/(b-a),0,1)
    f = np.zeros(x.size, dtype=np.float64)
    for n in range(0,order+1):
        f += sp.special.comb(order+n,n) * sp.special.comb(2*order+1, order-n) * (-x) ** n
    f *= x**(order+1)
    f_inv = 1. - f
    return f, f_inv

def stretch_1d_cluster_ends(x, max_growth_rate=1.02, max_ratio=3, inverse=False, n_uniform=2):
    '''
    stretch 1D grid, cluster at ends
    '''
    
    x = np.copy(x)
    
    ## backup for check later
    xmin = x.min()
    xmax = x.max()
    
    nx = x.shape[0]
    
    if (nx<10):
        raise ValueError
    
    ## scaling func range [0,1]
    f_scale = np.zeros(nx-1,dtype=np.float64)
    f_scale[n_uniform:-n_uniform]  = signal.windows.parzen(nx-1-2*n_uniform)
    #f_scale[n_uniform:-n_uniform]  = signal.windows.hamming(nx-1-2*n_uniform)
    #f_scale[n_uniform:-n_uniform]  = signal.windows.hann(nx-1-2*n_uniform)
    f_scale -= f_scale.min()
    f_scale /= f_scale.max()
    
    ## coarser at ends (rather than finer at ends)
    if inverse:
        f_scale = 1. - f_scale
    
    # === solve for max dx ratio based on max allowable growth rate
    
    def __f_opt_max_growth_rate(dx_ratio_max, x,max_growth_rate_tgt):
        '''
        given a max growth rate (e.g. 1.02), find the max dx_ratio
        '''
        x = np.copy(x)
        nx = x.shape[0]
        
        dx_fac = 1 + f_scale*(dx_ratio_max-1) ## scaling func range [1,dx_ratio_max]
        
        dx_  = np.diff(x,n=1)
        dx_ *= dx_fac
        x_   = np.cumsum(np.concatenate(([0.],dx_)))
        
        growth_rate = np.zeros((nx-3,), dtype=np.float64)
        for i in range(1,nx-2):
            dxL=dx_[i]
            dxR=dx_[i+1]
            growth_rate[i-1] = max( dxL/dxR , dxR/dxL )
        max_growth_rate = growth_rate.max()
        
        root = np.abs( max_growth_rate - max_growth_rate_tgt )
        return root
    
    sol = sp.optimize.least_squares(fun=__f_opt_max_growth_rate,
                                    x0=2,
                                    xtol=1e-14,
                                    ftol=1e-14,
                                    method='trf',
                                    max_nfev=int(1e4),
                                    args=(x,max_growth_rate,),
                                    bounds=(1.,999999.))
    if not sol.success:
        raise ValueError
    
    dx_ratio_max = sol.x
    
    ## if the ratio is > the input 'max_ratio', take 'max_ratio' instead
    if (dx_ratio_max>max_ratio):
        dx_ratio_max = max_ratio
        assert_exact_growth_rate = False
        assert_max_growth_rate = True
    else:
        assert_exact_growth_rate = True
        assert_max_growth_rate = False
    
    # ===
    
    dx_fac = 1 + f_scale*(dx_ratio_max-1) ## scaling func range [1,dx_ratio_max]
    
    dx_  = np.diff(x,n=1)
    dx_ *= dx_fac
    x_   = np.cumsum(np.concatenate(([0.],dx_)))
    scf  = xmax / x_.max() ## scale factor
    
    dx = np.diff(x,n=1)
    x  = np.cumsum(np.concatenate(([0.],dx*dx_fac*scf)))
    dx = np.diff(x,n=1)
    
    ## recalculate growth rate, do checks
    aa = dx[1:]/dx[:-1]
    bb = dx[:-1]/dx[1:]
    cc = np.maximum(aa,bb)
    max_growth_rate_ = cc.max()
    if assert_exact_growth_rate:
        np.testing.assert_allclose( max_growth_rate_, max_growth_rate, rtol=1e-6, atol=1e-6 )
    if assert_max_growth_rate:
        if ( max_growth_rate > max_growth_rate ):
            raise AssertionError
    
    ## assert min/max stayed the same
    np.testing.assert_allclose(x.max(), xmax, rtol=1e-14, atol=1e-14)
    np.testing.assert_allclose(x.min(), xmin, rtol=1e-14, atol=1e-14)
    
    # ===
    
    if False: ## plot : debug : grid dx_fac
        plt.close('all')
        fig1 = plt.figure(figsize=(6,6/2), dpi=200)
        ax1 = plt.gca()
        ax1.tick_params(axis='x', which='both', direction='out')
        ax1.tick_params(axis='y', which='both', direction='out')
        ##
        #ln1, = ax1.plot(np.arange(dx.shape[0]), dx, c=red, linestyle='None', marker='o', markersize=1.0, zorder=20)
        ln1, = ax1.plot(np.arange(dx.shape[0]), dx_fac*scf, c='blue', linestyle='None', marker='o', markersize=1.0, zorder=20)
        ##
        fig1.tight_layout(pad=0.25)
        fig1.tight_layout(pad=0.25)
        dpi_out = 2160/plt.gcf().get_size_inches()[1]
        #fig1.savefig('dx_fac.png', dpi=dpi_out)
        plt.show()
    
    if False: ## plot : debug : grid dx_fac
        plt.close('all')
        fig1 = plt.figure(figsize=(6,6/2), dpi=200)
        ax1 = plt.gca()
        ax1.tick_params(axis='x', which='both', direction='out')
        ax1.tick_params(axis='y', which='both', direction='out')
        ##
        ln1, = ax1.plot(np.arange(dx.shape[0]), dx, c='blue', linestyle='None', marker='o', markersize=1.0, zorder=20)
        ##
        fig1.tight_layout(pad=0.25)
        fig1.tight_layout(pad=0.25)
        dpi_out = 2160/plt.gcf().get_size_inches()[1]
        #fig1.savefig('dx.png', dpi=dpi_out)
        plt.show()
    
    return x

# time integration
# ======================================================================

class time_integrate_2d_seeder():
    '''
    for use with turbx.time_integrate_2d()
    - yields (N,2) arrays of pts as func of timestep
    - 'fixed position' : return original initialized points
    '''
    def __init__(self, xy_pts, dti=10, method='fixed position', **kwargs):
        self.xy_pts = np.copy(xy_pts)
        self.dti = dti
        self.method = method
    
    def __call__(self,ti):
        if (self.method=='fixed position'):
            if (ti%self.dti==0):
                return np.copy( self.xy_pts )
            else:
                return None
        elif (self.method=='random in poly'):
            raise NotImplementedError
        else:
            raise ValueError

def time_integrate_2d(u,v, x2d,y2d, dt,nt, xy_pts=None, xy_pt_seeder=None, **kwargs):
    '''
    integrate [x,y] paths for a single [u,v] field
    - xy_pts is a (N,2) numpy array describing the start locations of the particles
    - xy_pt_seeder() callable provides particle coordinates as a function of time
    '''
    
    verbose      = kwargs.get('verbose',False)
    method       = kwargs.get('method','cubic') ## interpolation method 'linear','cubic'
    direction    = kwargs.get('direction','fwdbwd') ## 'fwd','bwd','fwdbwd'
    bounds_check = kwargs.get('bounds_check',True)
    
    method_integration = kwargs.get('method_integration','euler1D')
    
    if verbose: print('\n'+'turbx.time_integrate_2d()'+'\n'+72*'-')
    t_start_func = timeit.default_timer()
    
    if verbose: even_print(f'bounds_check',str(bounds_check))
    
    ## checks
    if not any([(method=='linear'),(method=='cubic')]):
        raise ValueError("'method' should be one of 'cubic' or 'linear'")
    if not any([(direction=='fwd'),(direction=='bwd'),(direction=='fwdbwd')]):
        raise ValueError("'direction' should be one of 'fwd','bwd','fwdbwd'")
    if not isinstance(u, np.ndarray):
        raise ValueError('u should be a numpy array')
    if not isinstance(v, np.ndarray):
        raise ValueError('v should be a numpy array')
    if not isinstance(x2d, np.ndarray):
        raise ValueError('x2d should be a numpy array')
    if not isinstance(y2d, np.ndarray):
        raise ValueError('y2d should be a numpy array')
    
    if (xy_pts is not None): ## there are initial points
        if not isinstance(xy_pts, np.ndarray):
            raise ValueError('xy_pts should be a numpy array')
        if (xy_pts.ndim!=2):
            raise ValueError('xy_pts.ndim!=2')
        if (xy_pts.shape[1]!=2):
            raise ValueError('xy_pts should have shape (N,2)')
    else: ## there are no initial points
        
        ## make sure that there is a callable 'xy_pt_seeder' to provide points
        if (xy_pt_seeder is None):
            raise ValueError
        else:
            if not callable(xy_pt_seeder):
                raise ValueError
    
    if (x2d.ndim!=2):
        raise ValueError('x2d.ndim!=2')
    if (y2d.ndim!=2):
        raise ValueError('y2d.ndim!=2')
    if (u.shape!=v.shape):
        raise ValueError('u.shape!=v.shape')
    if (x2d.shape!=y2d.shape):
        raise ValueError('x2d.shape!=y2d.shape')
    if (u.shape!=x2d.shape):
        raise ValueError('u.shape!=x2d.shape')
    
    ## are we using point seeding?
    if (xy_pt_seeder is not None):
        pt_seeding_active = True
    else:
        pt_seeding_active = False
    
    ## number of initial points
    if (xy_pts is not None):
        n_pts = xy_pts.shape[0]
    else:
        n_pts = 0
    
    ## if there are no initial points and none will be seeded
    if (n_pts==0) and not pt_seeding_active:
        raise ValueError
    
    if verbose: even_print(f'n pts (initial)',f'{n_pts:d}')
    
    ## naive time array, could be updated for 'fwdbwd'
    t = dt*np.arange(nt,dtype=np.float64)
    
    if (direction=='fwd'):
        ti_fwd = np.arange(nt)
    elif (direction=='bwd'):
        ti_bwd = np.flip(np.arange(nt))
    elif (direction=='fwdbwd'):
        nt_nom = nt
        nt = 1 + 2*(nt-1)
        tL = np.copy( np.flip(-t[1:]) )
        tR = np.copy( t[1:] )
        t  = np.concatenate( [tL,np.array([0.],dtype=np.float64),tR] , casting='no' )
        ti_fwd = np.arange(nt_nom-1,nt)
        ti_bwd = np.flip(np.arange(0,nt_nom))
    else:
        raise ValueError
    
    if verbose: even_print(f'nt',f'{nt:d}')
    
    ## do a 'test run' of the seeding function
    ## get the total number of pts that will be seeded
    n_pts_seeded=0
    if pt_seeding_active:
        for ti in range(nt):
            xy_pts_seeded = xy_pt_seeder(ti=ti)
            if (xy_pts_seeded is not None):
                n_pts_seeded += xy_pts_seeded.shape[0]
    if verbose: even_print(f'n pts (seeded)',f'{n_pts_seeded:d}')
    
    ## total points = initial points + seeded points
    n_pts += n_pts_seeded
    if verbose: even_print(f'n pts (total)',f'{n_pts:d}')
    
    ## shape of the [x,y] grid and the [u,v] fields for forming interpolant
    nx,ny = x2d.shape
    if verbose: even_print(f'nx',f'{nx:d}')
    if verbose: even_print(f'ny',f'{ny:d}')
    
    ## if checking bounds, then
    ## construct polygon object using matplotlib
    if bounds_check:
        n_edge_pts = 2*nx + 2*(ny-2)
        n_edge_faces = n_edge_pts
        poly = np.zeros((n_edge_faces,2),dtype=np.float64)
        ii=-1
        for j in range(ny): ## W
            ii+=1
            poly[ii,0] = x2d[0,j]
            poly[ii,1] = y2d[0,j]
        for i in range(1,nx-1): ## N
            ii+=1
            poly[ii,0] = x2d[i,-1]
            poly[ii,1] = y2d[i,-1]
        for j in range(ny): ## E
            ii+=1
            poly[ii,0] = x2d[-1,-(j+1)]
            poly[ii,1] = y2d[-1,-(j+1)]
        for i in range(1,nx-1): ## S
            ii+=1
            poly[ii,0] = x2d[-(i+1),0]
            poly[ii,1] = y2d[-(i+1),0]
        
        poly_obj = mpl.path.Path(poly,closed=False)
    
    ## get interpolant callables
    t_start = timeit.default_timer()
    
    if (method=='cubic'):
        
        f_u = sp.interpolate.CloughTocher2DInterpolator(points=(x2d.ravel(), y2d.ravel()),
                                                                values=u.ravel(),
                                                                fill_value=np.nan )
        
        f_v = sp.interpolate.CloughTocher2DInterpolator(points=(x2d.ravel(), y2d.ravel()),
                                                                values=v.ravel(),
                                                                fill_value=np.nan )
    
    elif (method=='linear'):
        
        f_u = sp.interpolate.LinearNDInterpolator(points=(x2d.ravel(), y2d.ravel()),
                                                          values=u.ravel(),
                                                          fill_value=np.nan )
        
        f_v = sp.interpolate.LinearNDInterpolator(points=(x2d.ravel(), y2d.ravel()),
                                                          values=v.ravel(),
                                                          fill_value=np.nan )
    
    else:
        raise NotImplementedError
    
    t_delta = timeit.default_timer() - t_start
    if verbose: even_print(f'time construct interpolant',format_time_string(t_delta))
    
    scalars = [ 'x','y',
                'u','v',
                't',
                'id',
              ]
    
    scalars_dtype = [ np.float64, np.float64,
                      np.float64, np.float64,
                      np.float64,
                      np.int64,
                    ]
    
    data = np.zeros(shape=(n_pts,nt), dtype={'names':scalars, 'formats':scalars_dtype})
    if verbose: even_print(f'size data','%0.1f %s'%format_nbytes(data.nbytes))
    
    ## fill in IDs
    for ti in range(nt):
        data['id'][:,ti] = np.arange(n_pts, dtype=np.int64)
    
    ## populate [x,y,u,v] in data storage array
    data['x'][:,:] = np.nan
    data['y'][:,:] = np.nan
    data['u'][:,:] = np.nan
    data['v'][:,:] = np.nan
    
    ## make copy of original locations
    if (xy_pts is not None):
        xy_pts_orig = np.copy(xy_pts)
    else:
        xy_pts_orig = None
    
    # ==================================================================
    
    t_start = timeit.default_timer()
    
    if verbose:
        progress_bar = tqdm(total=nt, ncols=100, desc='convect', leave=False, file=sys.stdout)
    
    ## convect (forwards)
    if any([(direction=='fwd'),(direction=='fwdbwd')]):
        for ti in ti_fwd:
            tt = t[ti]
            
            ## store time
            data['t'][:,ti] = tt
            
            ## add pts
            if pt_seeding_active:
                xy_pts_seeded = xy_pt_seeder(ti=ti)
                if (xy_pts_seeded is not None):
                    try:
                        xy_pts = np.concatenate((xy_pts,xy_pts_seeded),axis=0)
                    except ValueError: ## e.g. is None
                        xy_pts = np.copy(xy_pts_seeded)
            
            if (xy_pts is None):
                raise ValueError(f'xy_pts is None at ti={ti:d}')
            
            n_pts = xy_pts.shape[0]
            
            ## store position
            data['x'][:n_pts,ti] = np.copy( xy_pts[:,0] )
            data['y'][:n_pts,ti] = np.copy( xy_pts[:,1] )
            
            ## get mask for NaN / non-NaN pts
            ii_nan = np.where(  np.isnan(xy_pts[:,0]) |  np.isnan(xy_pts[:,1]) )
            ii     = np.where( ~np.isnan(xy_pts[:,0]) & ~np.isnan(xy_pts[:,1]) )
            if (len(ii[0])+len(ii_nan[0]) != n_pts):
                raise ValueError
            
            ## interpolate velocity
            u_pts = np.squeeze( f_u( xy_pts[ii,0], xy_pts[ii,1] ) )
            v_pts = np.squeeze( f_v( xy_pts[ii,0], xy_pts[ii,1] ) )
            
            ## write to data array
            data['u'][ii,ti]     = np.copy( u_pts )
            data['v'][ii,ti]     = np.copy( v_pts )
            data['u'][ii_nan,ti] = np.nan
            data['v'][ii_nan,ti] = np.nan
            
            ## convect (forwards, Euler 1D)
            xy_pts[ii,0] += u_pts*dt
            xy_pts[ii,1] += v_pts*dt
            
            ## test if contained after convect
            if bounds_check:
                in_poly = poly_obj.contains_points(xy_pts)
                ii_not_in_poly = np.invert(in_poly)
                xy_pts[ii_not_in_poly,:] = np.nan
            
            if verbose: progress_bar.update()
    
    ## convect (backwards)
    if any([(direction=='bwd'),(direction=='fwdbwd')]):
        
        ## reset position before bwd interpolation
        if (xy_pts_orig is None):
            xy_pts = None
        else:
            xy_pts = np.copy(xy_pts_orig)
        
        for ti in ti_bwd:
            tt = t[ti]
            
            ## store time
            data['t'][:,ti] = tt
            
            ## add pts
            if pt_seeding_active:
                #xy_pts_seeded = xy_pt_seeder(ti=ti)
                xy_pts_seeded = xy_pt_seeder(ti=nt-ti-1) ## if seeding backwards
                if (xy_pts_seeded is not None):
                    try:
                        xy_pts = np.concatenate((xy_pts,xy_pts_seeded),axis=0)
                    except ValueError: ## e.g. is None
                        xy_pts = np.copy(xy_pts_seeded)
            
            if (xy_pts is None):
                raise ValueError(f'xy_pts is None at ti={ti:d}')
            
            n_pts = xy_pts.shape[0]
            
            ## store position
            data['x'][:n_pts,ti] = np.copy( xy_pts[:,0] )
            data['y'][:n_pts,ti] = np.copy( xy_pts[:,1] )
            
            ## get mask for NaN / non-NaN pts
            ii_nan = np.where(  np.isnan(xy_pts[:,0]) |  np.isnan(xy_pts[:,1]) )
            ii     = np.where( ~np.isnan(xy_pts[:,0]) & ~np.isnan(xy_pts[:,1]) )
            if (len(ii[0])+len(ii_nan[0]) != n_pts):
                raise ValueError
            
            ## interpolate velocity
            u_pts = np.squeeze( f_u( xy_pts[ii,0], xy_pts[ii,1] ) )
            v_pts = np.squeeze( f_v( xy_pts[ii,0], xy_pts[ii,1] ) )
            
            data['u'][ii,ti]     = np.copy( u_pts )
            data['v'][ii,ti]     = np.copy( v_pts )
            data['u'][ii_nan,ti] = np.nan
            data['v'][ii_nan,ti] = np.nan
            
            ## convect (backwards, Euler 1D)
            xy_pts[ii,0] -= u_pts*dt
            xy_pts[ii,1] -= v_pts*dt
            
            ## test if contained after convect
            if bounds_check:
                in_poly = poly_obj.contains_points(xy_pts)
                ii_not_in_poly = np.invert(in_poly)
                xy_pts[ii_not_in_poly,:] = np.nan
            
            if verbose: progress_bar.update()
    
    if verbose: progress_bar.close()
    
    t_delta = timeit.default_timer() - t_start
    if verbose: even_print(f'time convect',format_time_string(t_delta))
    #if verbose: even_print(f'time total',format_time_string((timeit.default_timer() - t_start_func)))
    
    if verbose: print(72*'-')
    if verbose: print('total time : turbx.time_integrate_2d() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
    if verbose: print(72*'-')
    
    return data

# csys
# ======================================================================

def rect_to_cyl(xyz,**kwargs):
    '''
    convert [x,y,<z>] to [θ,r,<z>]
    '''
    
    cx = kwargs.get('cx',0.)
    cy = kwargs.get('cy',0.)
    
    trz = np.zeros_like(xyz)
    
    if (xyz.ndim==1) and (xyz.shape[-1]==2): ## a single point, shape=(2,)
        xx = xyz[0]-cx
        yy = xyz[1]-cy
        trz[0] = np.arctan2(yy,xx)
        trz[1] = np.sqrt(xx**2 + yy**2)
    
    elif (xyz.ndim==1) and (xyz.shape[-1]==3): ## a single point, shape=(3,)
        xx = xyz[0]-cx
        yy = xyz[1]-cy
        trz[0] = np.arctan2(yy,xx)
        trz[1] = np.sqrt(xx**2 + yy**2)
        trz[2] = xyz[2]
    
    elif (xyz.ndim==2) and (xyz.shape[-1]==2): ## a 1D vector of 2D points, shape=(N,2)
        xx = xyz[:,0]-cx
        yy = xyz[:,1]-cy
        trz[:,0] = np.arctan2(yy,xx)
        trz[:,1] = np.sqrt(xx**2 + yy**2)
    
    elif (xyz.ndim==2) and (xyz.shape[-1]==3): ## a 1D vector of 3D points, shape=(N,3)
        xx = xyz[:,0]-cx
        yy = xyz[:,1]-cy
        trz[:,0] = np.arctan2(yy,xx)
        trz[:,1] = np.sqrt(xx**2 + yy**2)
        trz[:,2] = xyz[:,2]
    
    elif (xyz.ndim==3) and (xyz.shape[-1]==2): ## 2D, shape=(nx,ny,2)
        xx    = xyz[:,:,0] - cx
        yy    = xyz[:,:,1] - cy
        trz[:,:,0] = np.arctan2(yy,xx)
        trz[:,:,1] = np.sqrt(xx**2 + yy**2)
    
    elif (xyz.ndim==4) and (xyz.shape[-1]==3): ## 3D, shape=(nx,ny,nz,3)
        xx    = xyz[:,:,:,0] - cx
        yy    = xyz[:,:,:,1] - cy
        trz[:,:,:,0] = np.arctan2(yy,xx)
        trz[:,:,:,1] = np.sqrt(xx**2 + yy**2)
        trz[:,:,:,2] = xyz[:,:,:,2]
    
    else:
        raise ValueError('this input is not supported')
    
    return trz

def cyl_to_rect(trz,**kwargs):
    '''
    convert [θ,r,<z>] to [x,y,<z>]
    '''
    
    cx = kwargs.get('cx',0.)
    cy = kwargs.get('cy',0.)
    
    xyz = np.zeros_like(trz)
    
    if (trz.ndim==1) and (trz.shape[-1]==2): ## a single point, shape=(2,)
        tt = trz[0]
        rr = trz[1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[0] = xx + cx
        xyz[1] = yy + cy
    
    elif (trz.ndim==1) and (trz.shape[-1]==3): ## a single point, shape=(3,)
        tt = trz[0]
        rr = trz[1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[0] = xx + cx
        xyz[1] = yy + cy
        xyz[2] = trz[2]
    
    elif (trz.ndim==2) and (trz.shape[-1]==2): ## a 1D vector of 2D points, shape=(N,2)
        tt = trz[:,0]
        rr = trz[:,1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[:,0] = xx + cx
        xyz[:,1] = yy + cy
    
    elif (trz.ndim==2) and (trz.shape[-1]==3): ## a 1D vector of 3D points, shape=(N,3)
        tt = trz[:,0]
        rr = trz[:,1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[:,0] = xx + cx
        xyz[:,1] = yy + cy
        xyz[:,2] = trz[:,2]
    
    elif (trz.ndim==3) and (trz.shape[-1]==2): ## 2D, shape=(nx,ny,2)
        tt = trz[:,:,0]
        rr = trz[:,:,1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[:,:,0] = xx + cx
        xyz[:,:,1] = yy + cy
    
    elif (trz.ndim==4) and (trz.shape[-1]==3): ## 3D, shape=(nx,ny,nz,3)
        tt = trz[:,:,:,0]
        rr = trz[:,:,:,1]
        xx = rr*np.cos(tt)
        yy = rr*np.sin(tt)
        xyz[:,:,:,0] = xx + cx
        xyz[:,:,:,1] = yy + cy
        xyz[:,:,:,2] = trz[:,:,:,2]
    
    else:
        raise ValueError('this input is not supported')
    
    return xyz

def rotate_2d(xy,theta,**kwargs):
    '''
    rotate a cartesian [x,y] point coordinate array around a center point [cx,cy]
    '''
    
    cx = kwargs.get('cx',0.)
    cy = kwargs.get('cy',0.)
    
    xy_rotated = np.zeros_like(xy)
    
    xy_ = np.copy(xy) ## prevent any pointer disasters
    
    translation_mat_1 = np.array([[ 1, 0, -cx ],
                                  [ 0, 1, -cy ],
                                  [ 0, 0,  1  ]], dtype=np.float64)
    
    rotation_mat = np.array([[ np.cos(theta), -np.sin(theta), 0 ],
                             [ np.sin(theta),  np.cos(theta), 0 ],
                             [ 0,              0,             1 ]], dtype=np.float64)
    
    translation_mat_2 = np.array([[ 1, 0, +cx ],
                                  [ 0, 1, +cy ],
                                  [ 0, 0,  1  ]], dtype=np.float64)
    
    transform_mat = np.einsum( 'ij,jk,kl->il' , translation_mat_2 , rotation_mat , translation_mat_1 )
    
    if (xy_.ndim==1) and (xy_.shape[-1]==2): ## a single point (2,)
        xy_ = np.concatenate((xy_,[1])) ## pad a 1
        xy_rotated = np.einsum( 'ij,j->i' , transform_mat, xy_ )
        xy_rotated = xy_rotated[:2]
    
    elif (xy_.ndim==2) and (xy_.shape[-1]==2): ## a 1D vector of points (N,2)
        raise NotImplementedError
    elif (xy_.ndim==3) and (xy_.shape[-1]==2): ## a 2D field of points (nx,ny,2)
        raise NotImplementedError
    else:
        raise ValueError('this input is not supported')
    
    return xy_rotated

# post-processing : vector & tensor ops
# ======================================================================

def get_grad(a,b,c, x,y,z, **kwargs):
    '''
    get the 3D gradient tensor (∂Ai/∂xj) from vector A=[a,b,c]
    -----
    - a,b,c are 3D arrays
    - x,y,z are 1D arrays (coord vectors)
    '''
    
    do_stack = kwargs.get('do_stack',True)
    hiOrder  = kwargs.get('hiOrder',True)
    verbose  = kwargs.get('verbose',False)
    
    print('get_grad() has been deprecated --> needs to be updated with turbx.gradient()')
    sys.exit(1)
    
    nx = x.size; ny = y.size; nz = z.size
    
    dtype = a.dtype
    
    # === gradients with O3 Spline + natural BCs
    if hiOrder:
        '''
        this could be parallelized with multiprocessing + async threads
        '''
        dadx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dady = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dadz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdy = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdy = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        
        if verbose: progress_bar = tqdm(total=((nx*ny)+(ny*nz)+(nz*nx)), ncols=100, desc='get_grad()', leave=False, file=sys.stdout)
        
        for i in range(nx):
            for j in range(ny):
                dadz[i,j,:] = sp.interpolate.CubicSpline(z,a[i,j,:],bc_type='natural')(z,1)
                dbdz[i,j,:] = sp.interpolate.CubicSpline(z,b[i,j,:],bc_type='natural')(z,1)
                dcdz[i,j,:] = sp.interpolate.CubicSpline(z,c[i,j,:],bc_type='natural')(z,1)
                if verbose: progress_bar.update()
        for j in range(ny):
            for k in range(nz):
                dadx[:,j,k] = sp.interpolate.CubicSpline(x,a[:,j,k],bc_type='natural')(x,1)
                dbdx[:,j,k] = sp.interpolate.CubicSpline(x,b[:,j,k],bc_type='natural')(x,1)
                dcdx[:,j,k] = sp.interpolate.CubicSpline(x,c[:,j,k],bc_type='natural')(x,1)
                if verbose: progress_bar.update()
        for k in range(nz):
            for i in range(nx):
                dady[i,:,k] = sp.interpolate.CubicSpline(y,a[i,:,k],bc_type='natural')(y,1)
                dbdy[i,:,k] = sp.interpolate.CubicSpline(y,b[i,:,k],bc_type='natural')(y,1)
                dcdy[i,:,k] = sp.interpolate.CubicSpline(y,c[i,:,k],bc_type='natural')(y,1)
                if verbose: progress_bar.update()
        
        if verbose: progress_bar.close()
    
    else: ## numpy.gradient() --> 1st (or 2nd?) order
        if verbose: progress_bar = tqdm(total=9, ncols=100, desc='get_grad()', leave=False, file=sys.stdout)
        
        dadx = np.gradient(a, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        dady = np.gradient(a, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        dadz = np.gradient(a, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        dbdx = np.gradient(b, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        dbdy = np.gradient(b, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        dbdz = np.gradient(b, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        dcdx = np.gradient(c, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        dcdy = np.gradient(c, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        dcdz = np.gradient(c, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        
        if verbose: progress_bar.close()
    
    if do_stack:
        dAdx_ij = np.stack((np.stack((dadx, dady, dadz), axis=3),
                            np.stack((dbdx, dbdy, dbdz), axis=3),
                            np.stack((dcdx, dcdy, dcdz), axis=3)), axis=4)
        return dAdx_ij
    else:
        dAdx_ij = dict(zip(['dadx', 'dady', 'dadz', 'dbdx', 'dbdy', 'dbdz', 'dcdx', 'dcdy', 'dcdz'], 
                           [ dadx,   dady,   dadz,   dbdx,   dbdy,   dbdz,   dcdx,   dcdy,   dcdz]))
        return dAdx_ij

def get_curl(a,b,c, x,y,z, **kwargs):
    '''
    get 3D curl vector ∇⨯A , A=[a,b,c]
    '''
    
    do_stack = kwargs.get('do_stack',True)
    hiOrder  = kwargs.get('hiOrder',True)
    verbose  = kwargs.get('verbose',False)
    
    print('get_curl() has been deprecated --> needs to be updated with turbx.gradient()')
    sys.exit(1)
    
    nx = x.size; ny = y.size; nz = z.size
    
    dtype = u.dtype
    
    ## gradients with O3 Spline + natural BCs
    if hiOrder:
        '''
        this could be easily parallelized
        '''
        
        #dadx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dady = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dadz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        #dbdy = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dbdz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdx = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        dcdy = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        #dcdz = np.zeros(shape=(nx,ny,nz), dtype=dtype)
        
        if verbose: progress_bar = tqdm(total=((nx*ny)+(ny*nz)+(nz*nx)), ncols=100, desc='get_curl()', leave=False, file=sys.stdout)
        
        for i in range(nx):
            for j in range(ny):
                dadz[i,j,:] = sp.interpolate.CubicSpline(z,a[i,j,:],bc_type='natural')(z,1)
                dbdz[i,j,:] = sp.interpolate.CubicSpline(z,b[i,j,:],bc_type='natural')(z,1)
                #dcdz[i,j,:] = sp.interpolate.CubicSpline(z,c[i,j,:],bc_type='natural')(z,1)
                if verbose: progress_bar.update()
        for j in range(ny):
            for k in range(nz):
                #dadx[:,j,k] = sp.interpolate.CubicSpline(x,a[:,j,k],bc_type='natural')(x,1)
                dbdx[:,j,k] = sp.interpolate.CubicSpline(x,b[:,j,k],bc_type='natural')(x,1)
                dcdx[:,j,k] = sp.interpolate.CubicSpline(x,c[:,j,k],bc_type='natural')(x,1)
                if verbose: progress_bar.update()
        for k in range(nz):
            for i in range(nx):
                dady[i,:,k] = sp.interpolate.CubicSpline(y,a[i,:,k],bc_type='natural')(y,1)
                #dbdy[i,:,k] = sp.interpolate.CubicSpline(y,b[i,:,k],bc_type='natural')(y,1)
                dcdy[i,:,k] = sp.interpolate.CubicSpline(y,c[i,:,k],bc_type='natural')(y,1)
                if verbose: progress_bar.update()
        
        if verbose: progress_bar.close()
    
    else: ## numpy.gradient() --> 1st (or 2nd?) order
        if verbose: progress_bar = tqdm(total=9, ncols=100, desc='get_curl()', leave=False, file=sys.stdout)
        
        #dadx = np.gradient(a, x, edge_order=2, axis=0)
        #if verbose: progress_bar.update()
        dady = np.gradient(a, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        dadz = np.gradient(a, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        dbdx = np.gradient(b, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        #dbdy = np.gradient(b, y, edge_order=2, axis=1)
        #if verbose: progress_bar.update()
        dbdz = np.gradient(b, z, edge_order=2, axis=2)
        if verbose: progress_bar.update()
        dcdx = np.gradient(c, x, edge_order=2, axis=0)
        if verbose: progress_bar.update()
        dcdy = np.gradient(c, y, edge_order=2, axis=1)
        if verbose: progress_bar.update()
        #dcdz = np.gradient(c, z, edge_order=2, axis=2)
        #if verbose: progress_bar.update()
        
        if verbose: progress_bar.close()
    
    # ===
    curl_x = dcdy - dbdz
    curl_y = dadz - dcdx
    curl_z = dbdx - dady
    
    if do_stack:
        curl = np.stack((curl_x, curl_y, curl_z), axis=3)
        return curl
    else:
        curl = dict(zip(['curl_x', 'curl_y', 'curl_z'], 
                        [ curl_x,   curl_y,   curl_z]))
        return curl

# post-processing : spectral, statistical
# ======================================================================

def get_overlapping_window_size(asz, n_win, overlap_fac):
    '''
    get window length and overlap given a
    desired number of windows and a nominal overlap factor
    -----
    --> the output should be passed to get_overlapping_windows()
        to do the actual padding & windowing
    '''
    if not isinstance(asz, (int,np.int32,np.int64)):
        raise TypeError('arg asz must be type int')
    if not isinstance(n_win, (int,np.int32,np.int64)):
        raise TypeError('arg n_win must be type int')
    if (overlap_fac >= 1.):
        raise ValueError('arg overlap_fac must be <1')
    if (overlap_fac < 0.):
        raise ValueError('arg overlap_fac must be >0')
    n_ends = n_win+1
    n_mids = n_win
    
    # === solve for float-valued window 'mid' size & 'end' size
    def eqn(soltup, asz=asz, overlap_fac=overlap_fac):
        (endsz,midsz) = soltup
        eq1 = asz - n_ends*endsz - n_mids*midsz
        eq2 = overlap_fac*(midsz+2*endsz) - endsz
        return [eq1, eq2]
    
    guess = asz*0.5
    endsz,midsz = sp.optimize.fsolve(eqn, (guess,guess), (asz,overlap_fac))
    win_len     = midsz + 2*endsz
    overlap     = endsz
    
    win_len = max(math.ceil(win_len),1)
    overlap = max(math.floor(overlap),0)
    
    return win_len, overlap

def get_overlapping_windows(a, win_len, overlap):
    '''
    subdivide 1D array into overlapping windows
    '''
    #pad_mode = kwargs.get('pad_mode','append')
    ##
    if not isinstance(a, np.ndarray):
        raise TypeError('arg a must be type np.ndarray')
    if not isinstance(win_len, int):
        raise TypeError('arg win_len must be type int')
    if not isinstance(overlap, int):
        raise TypeError('arg overlap must be type int')
    ##
    asz   = a.size
    skip  = win_len - overlap
    n_pad = (win_len - asz%skip)%skip
    #a_pad = np.concatenate(( np.zeros(n_pad,dtype=a.dtype) , np.copy(a) )) ## prepend
    a_pad = np.concatenate(( np.copy(a) , np.zeros(n_pad,dtype=a.dtype) )) ## append
    ##
    b = np.lib.stride_tricks.sliding_window_view(a_pad, win_len, axis=0)
    b = np.copy(b[::skip,:])
    n_win = b.shape[0]
    ##
    if (n_pad > 0.5*win_len):
        print('WARNING: n_pad > overlap')
    ##
    return b, n_win, n_pad

def ccor(ui,uj,**kwargs):
    '''
    1D normalized cross-correlation
    '''
    if (ui.ndim!=1):
        raise AssertionError('ui.ndim!=1')
    if (uj.ndim!=1):
        raise AssertionError('uj.ndim!=1')
    
    mode     = kwargs.get('mode','full')
    get_lags = kwargs.get('get_lags',False)
    
    lags = sp.signal.correlation_lags(ui.shape[0], uj.shape[0], mode=mode)
    #R    = sp.signal.correlate(ui, uj, mode=mode, method='direct')
    R    = sp.signal.correlate(ui, uj, mode=mode, method='fft') ## 'fft' is O(100)x faster for size O(10000) arrays
    norm = np.sqrt( np.sum( ui**2 ) ) * np.sqrt( np.sum( uj**2 ) )
    
    if (norm==0.):
        #R = np.ones((lags.shape[0],), dtype=ui.dtype)
        R = np.zeros((lags.shape[0],), dtype=ui.dtype)
    else:
        R /= norm
    
    if get_lags:
        return lags, R
    else:
        return R

def ccor_naive(u,v,**kwargs):
    '''
    1D normalized cross-correlation (naive version)
    - this kernel is designed as a check for ccor()
    '''
    if (u.ndim!=1):
        raise AssertionError('u.ndim!=1')
    if (v.ndim!=1):
        raise AssertionError('v.ndim!=1')
    
    ii = np.arange(u.shape[0],dtype=np.int32)
    jj = np.arange(v.shape[0],dtype=np.int32)
    
    ## lags (2D)
    ll     = np.stack(np.meshgrid(ii,jj,indexing='ij'), axis=-1)
    ll     = ll[:,:,0] - ll[:,:,1]
    
    ## lags (1D)
    lmin   = ll.min()
    lmax   = ll.max()
    n_lags = lmax-lmin+1
    lags   = np.arange(lmin,lmax+1)
    
    uu, vv = np.meshgrid(u,v,indexing='ij')
    uv     = np.stack((uu,vv), axis=-1)
    uvp    = np.prod(uv,axis=-1)
    
    c=-1
    R = np.zeros(n_lags, dtype=np.float64)
    for lag in lags:
        c+=1
        X = np.where(ll==lag)
        N = X[0].shape[0]
        R_ = np.sum(uvp[X]) / ( np.sqrt(np.sum(u**2)) * np.sqrt(np.sum(v**2)) )
        R[c] = R_
    return lags, R

def ccor_vec(ui,uj,axis=0):
    '''
    normalized cross-correlation, vectorized wrapper
    ---
    Parameters:
        ui, uj : ndarray
            Input arrays with the same shape.
        axis : int
            Axis along which to compute the cross-correlation.
    Returns:
        R : ndarray
            Cross-correlation results normalized where norms are non-zero.
    '''
    
    if ui.shape != uj.shape:
        raise ValueError('ui and uj must have the same shape.')
    
    nd = ui.ndim
    if (nd<2):
        raise ValueError('nd<2')
    
    shape_orig = ui.shape
    axes = tuple(range(ui.ndim))
    
    if not isinstance(axis, int):
        raise ValueError('axis should be of type int')
    if (axis not in axes):
        raise ValueError(f'axis={axis:d} is not valid for array with ui.ndim={str(ui.ndim)}')
    
    ## normalization for output array
    norm = np.sqrt( np.sum( ui**2, axis=axis, keepdims=True ) ) * np.sqrt( np.sum( uj**2, axis=axis, keepdims=True ) )
    
    nx = ui.shape[axis] ## size of axis over which gradient will be performed
    shift_pos=nd-1 ## last axis
    
    ## shift cross-correlation axis to last axis
    nx = ui.shape[axis] ## size of axis over which gradient will be performed
    ui = np.swapaxes(ui, axis, shift_pos)
    uj = np.swapaxes(uj, axis, shift_pos)
    shape_new = ui.shape
    size_all_but_ax = np.prod(np.array(shape_new)[:-1])
    
    ## shape of output
    n_lags = 2*nx - 1
    shape_out = list(shape_orig)
    shape_out[axis] = n_lags
    shape_out = tuple(shape_out)
    
    ## reshape N-D to 2D
    ## cross-correlation axis is 1, all other axes are flattened on axis=0)
    ui  = np.reshape(ui, (size_all_but_ax,nx), order='C')
    uj  = np.reshape(uj, (size_all_but_ax,nx), order='C')
    
    ## vectorize kernel
    __ccor_kernel = np.vectorize(
        lambda u,v: sp.signal.correlate(u,v, mode='full', method='direct'),
        signature="(n),(n)->(m)",
    )
    
    ## run kernel
    R = __ccor_kernel(ui, uj)
    
    ## 2D to N-D
    R = np.reshape(R, (*shape_new[:-1],n_lags), order='C')
    
    ## shift cross-correlation axis back to original position
    R = np.swapaxes(R, axis, shift_pos)
    if (R.shape != shape_out):
        raise ValueError
    
    ## normalize
    mask = norm != 0
    mask = np.broadcast_to(mask, shape_out)
    norm = np.broadcast_to(norm, shape_out)
    R[mask] /= norm[mask]
    R[~mask] = 0.
    return R

# binary I/O
# ======================================================================

def gulp(fname, **kwargs):
    '''
    read a complete binary file into memory, return 'virtual file'
    -----
    - returned handle can be opened via h5py as if it were on disk (with very high performance)
    - of course this only works if the entire file fits into memory
    - best use case: medium size file, large number of high-frequency read ops
    '''
    verbose = kwargs.get('verbose',True)
    f_size_gb = os.path.getsize(fname)/1024**3
    if verbose: tqdm.write('>>> gulp() : %s : %0.2f [GB]'%(os.path.basename(fname),f_size_gb))
    t_start = timeit.default_timer()
    with open(fname, 'rb') as fnb:
        bytes_in_mem = io.BytesIO(fnb.read())
    t_delta = timeit.default_timer() - t_start
    if verbose: tqdm.write('>>> gulp() : %s : %0.2f [GB/s]'%(format_time_string(t_delta), (f_size_gb/t_delta)))
    return bytes_in_mem

# utilities
# ======================================================================

def format_time_string(tsec):
    '''
    format seconds as dd:hh:mm:ss
    '''
    m, s = divmod(abs(int(round(tsec))),60)
    h, m = divmod(m,60)
    d, h = divmod(h,24)
    d    = int(round(d))
    h    = int(round(h))
    m    = int(round(m))
    s    = int(round(s))
    if (d==0) and (h==0) and (m==0):
        s_out = f'{s:d}s'
    elif (d==0) and (h==0):
        s_out = f'{m:d}m:{s:02d}s'
    elif (d==0):
        s_out = f'{h:d}h:{m:02d}m:{s:02d}s'
    else:
        s_out = f'{d:d}d:{h:02d}h:{m:02d}m:{s:02d}s'
    if (tsec >= 0):
        return s_out
    else:
        return f'-{s_out}'

def format_nbytes(size):
    '''
    format a number of bytes to [B],[KB],[MB],[GB],[TB]
    '''
    if not isinstance(size,(int,float)):
        raise ValueError('arg should be of type int or float')
    if (size<1024):
        size_fmt, size_unit = size, '[B]'
    elif (size>1024) and (size<=1024**2):
        size_fmt, size_unit = size/1024, '[KB]'
    elif (size>1024**2) and (size<=1024**3):
        size_fmt, size_unit = size/1024**2, '[MB]'
    elif (size>1024**3) and (size<=1024**4):
        size_fmt, size_unit = size/1024**3, '[GB]'
    else:
        size_fmt, size_unit = size/1024**4, '[TB]'
    return size_fmt, size_unit

def even_print(label, output, **kwargs):
    '''
    print/return a fixed width message
    '''
    terminal_width = kwargs.get('terminal_width',72)
    s              = kwargs.get('s',False) ## return string
    
    ndots = (terminal_width-2) - len(label) - len(output)
    text = label+' '+ndots*'.'+' '+output
    if s:
        return text
    else:
        #sys.stdout.write(text)
        print(text)
        return

# plotting & matplotlib
# ======================================================================

def set_mpl_env(**kwargs):
    r'''
    Setup the matplotlib environment
    --------------------------------
    
    - styles   : https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html
    - rcParams : https://matplotlib.org/stable/tutorials/introductory/customizing.html
    
    TrueType / OpenType Fonts
    -------------------------
    
    - IBM Plex
        - https://github.com/IBM/plex/archive/refs/heads/master.zip
        - https://www.fontsquirrel.com/fonts/download/ibm-plex --> doesnt contain 'Condensed'
        - test at : https://www.ibm.com/plex/plexness/
        
        ----- Linux / Ubuntu / WSL2
        $> sudo apt install fonts-ibm-plex
        ----- Linux / Ubuntu / WSL2 (manual)
        $> wget https://github.com/IBM/plex/archive/refs/heads/master.zip
        $> unzip master.zip
        $> cd plex-master
        $> sudo mkdir -p /usr/share/fonts/opentype/ibm-plex
        $> sudo mkdir -p /usr/share/fonts/truetype/ibm-plex
        $> sudo find . -name '*.otf' -exec cp -v {} /usr/share/fonts/opentype/ibm-plex/ \;
        $> sudo find . -name '*.ttf' -exec cp -v {} /usr/share/fonts/truetype/ibm-plex/ \;
        $> fc-cache -f -v
        $> fc-list | grep 'IBM'
        ----- regenerate matplotlib font cache (just delete, gets regenerated)
        $> rm ~/.cache/matplotlib/fontlist-v390.json
        $> rm -rf ~/.cache/matplotlib/tex.cache
    
    - Latin Modern Math
        - http://www.gust.org.pl/projects/e-foundry/lm-math/download/latinmodern-math-1959.zip
    
    - Latin Modern (lmodern in LaTeX)
        - https://www.fontsquirrel.com/fonts/download/Latin-Modern-Roman.zip
        - http://www.gust.org.pl/projects/e-foundry/latin-modern/download/lm2.004otf.zip
        
        --> usually already installed at : /usr/share/texmf/fonts/opentype/public/lm/
        ----- Linux / Ubuntu / WSL2
        $> wget https://www.fontsquirrel.com/fonts/download/Latin-Modern-Roman.zip
        $> unzip Latin-Modern-Roman.zip -d Latin-Modern-Roman
        $> cd Latin-Modern-Roman
        $> sudo mkdir -p /usr/share/fonts/opentype/lmodern
        $> sudo find . -name '*.otf' -exec cp -v {} /usr/share/fonts/opentype/lmodern/ \;
        $> fc-cache -f -v
        $> fc-list | grep 'Latin'
        ----- regenerate matplotlib font cache (just delete, gets regenerated)
        $> rm ~/.cache/matplotlib/fontlist-v390.json
        $> rm -rf ~/.cache/matplotlib/tex.cache
    
    - Computer Modern (default in LaTeX)
        - https://www.fontsquirrel.com/fonts/download/computer-modern.zip
        - http://mirrors.ctan.org/fonts/cm/ps-type1/bakoma.zip
        ----- Linux / Ubuntu / WSL2
        $> sudo apt install fonts-cmu
        ----- Linux / Ubuntu / WSL2 (manual install)
        ## wget https://sourceforge.net/projects/cm-unicode/files/latest/download -O cm-unicode-0.7.0-ttf.tar.xz
        ## tar -xvf cm-unicode-0.7.0-ttf.tar.xz
        ## wget https://www.fontsquirrel.com/fonts/download/computer-modern -O computer-modern.zip
        $> wget https://mirrors.ctan.org/fonts/cm-unicode.zip
        $> unzip computer-modern.zip -d computer-modern
        $> cd computer-modern
        $> sudo mkdir -p /usr/share/fonts/truetype/cmu
        $> sudo find . -name '*.ttf' -exec cp -v {} /usr/share/fonts/truetype/cmu/ \;
        $> fc-cache -f -v
        $> fc-list | grep 'CMU'
        ----- regenerate matplotlib font cache (just delete, gets regenerated)
        $> rm ~/.cache/matplotlib/fontlist-v390.json
        $> rm -rf ~/.cache/matplotlib/tex.cache
    
    Windows
    -------
    --> download, install, then delete : C:/Users/%USERNAME%/.matplotlib/fontlist-v390.json
    --> this JSON file will get regenerated with newly installed fonts
    '''
    
    useTex   = kwargs.get('useTex',False) ## use LaTeX text rendering
    darkMode = kwargs.get('darkMode',False)
    font     = kwargs.get('font',None)
    fontsize = kwargs.get('fontsize',10)
    
    ## mpl.rcParams.update(mpl.rcParamsDefault) ## reset rcparams to defaults
    
    if darkMode:
        mpl.style.use('dark_background') ## dark mode
    else:
        mpl.style.use('default')
    
    if useTex:
        
        mpl.rcParams['text.usetex'] = True
        
        ## 'Text rendering with LaTeX'
        ## https://matplotlib.org/stable/tutorials/text/usetex.html
        
        ## https://github.com/matplotlib/matplotlib/issues/5076/
        ## https://github.com/matplotlib/matplotlib/issues/17673/
        
        preamble_opts = [ r'', ]
        
        preamble_opts += [
                        #r'\usepackage[utf8]{inputenc}',
                        #r'\usepackage[T1]{fontenc}',
                        r'\usepackage{amsmath}', 
                        #r'\usepackage{amsfonts}',
                        r'\usepackage{amssymb}',
                        #r'\usepackage{textcomp}'
                        r'\usepackage{gensymb}', ## generic symbols 
                        #r'\usepackage{xfrac}',
                        #r'\usepackage{nicefrac}',
                        ]
        
        if (font==None): ## default
            mpl.rcParams['font.family']= 'serif'
            mpl.rcParams['font.serif'] = 'Computer Modern Roman'
        
        elif (font=='Aptos') or (font=='aptos'):
            
            #mpl.use('pgf') ## set pgf backend, !! .svg will not be supported for export !!
            #mpl.rcParams['pgf.rcfonts'] = True
            #mpl.rcParams['pgf.texsystem'] = 'xelatex'
            #mpl.rcParams['font.sans-serif'] = ['Aptos'] # Fallback fonts for sans-serif
            
            mpl.rcParams['font.family'] = 'sans-serif'
            mpl.rcParams['font.sans-serif']  = 'Aptos'
            
            preamble_opts +=    [   #r'\usepackage{amsmath}',
                                    #r'\usepackage{fontspec}', ## no if using mathspec
                                    r'\usepackage{mathspec}',
                                    r'\setmainfont{Aptos}',
                                    r'\setmathrm{Aptos}',
                                    r'\setmathfont(Digits,Latin){Aptos}',
                                ]
        
        elif (font=='IBM Plex Sans') or (font=='IBM Plex') or (font=='IBM') or (font=='ibm'):
            preamble_opts +=  [ r'\usepackage{plex-sans}', ## IBM Plex Sans
                                r'\renewcommand{\familydefault}{\sfdefault}', ## sans as default family
                                r'\renewcommand{\seriesdefault}{c}', ## condensed {*} as default series
                                r'\usepackage[italic]{mathastext}', ## use default font in math mode
                              ]
        
        elif (font=='times') or (font=='Times') or (font=='Times New Roman'):
            mpl.rcParams['font.family'] = 'serif'
            mpl.rcParams['font.serif']  = 'Times'
            #preamble_opts +=  [ r'\usepackage{txfonts}' ] ## Times-like fonts mathtext symbols
            preamble_opts +=  [ r'\usepackage{newtxtext}',
                                r'\usepackage{newtxmath}',
                                #r'\usepackage[italic]{mathastext}', ## use default font in math mode
                              ]
        
        elif (font=='lmodern') or (font=='Latin Modern') or (font=='Latin Modern Roman') or (font=='lmr'):
            preamble_opts +=  [ r'\usepackage{lmodern}' ]
        
        elif (font=='Palatino') or (font=='palatino'):
            mpl.rcParams['font.family'] = 'serif'
            mpl.rcParams['font.serif']  = 'Palatino'
        
        elif (font=='Helvetica') or (font=='helvetica'):
            mpl.rcParams['font.family']      = 'sans-serif'
            mpl.rcParams['font.sans-serif']  = 'Helvetica'
            preamble_opts +=  [ r'\renewcommand{\familydefault}{\sfdefault}', ## sans as default family
                                r'\usepackage[italic]{mathastext}', ## use default font in math mode
                              ]
        
        elif (font=='Avant Garde'):
            mpl.rcParams['font.family']     = 'sans-serif'
            mpl.rcParams['font.sans-serif'] = 'Avant Garde'
            preamble_opts +=  [ r'\renewcommand{\familydefault}{\sfdefault}', ## sans as default family
                                r'\usepackage[italic]{mathastext}', ## use default font in math mode
                              ]
        
        elif (font=='Computer Modern Roman') or (font=='Computer Modern') or (font=='CMR') or (font=='cmr'):
            mpl.rcParams['font.family'] = 'serif'
            mpl.rcParams['font.serif']  = 'Computer Modern Roman'
        
        else:
            raise ValueError('font=%s not a valid option'%str(font))
        
        ## make preamble string
        mpl.rcParams['text.latex.preamble'] = '\n'.join(preamble_opts)
        
        ## if using pgf, unset it
        if ( mpl.get_backend() == 'pgf' ):
            mpl.rcParams['pgf.preamble'] = '\n'.join(preamble_opts)
            mpl.rcParams['text.latex.preamble'] = r''
    
    else: ## use OpenType (OTF) / TrueType (TTF) fonts (and no TeX rendering)
        
        mpl.rcParams['text.usetex'] = False
        
        ## Register OTF/TTF Fonts (only necessary once)
        if False:
            # === register (new) fonts : Windows --> done automatically if you delete ~/.cache/matplotlib/fontlist-v390.json
            ##mpl.font_manager.findSystemFonts(fontpaths='C:/Windows/Fonts', fontext='ttf')
            #mpl.font_manager.findSystemFonts(fontpaths='C:/Users/'+os.path.expandvars('%USERNAME%')+'/AppData/Local/Microsoft/Windows/Fonts', fontext='ttf')
            mpl.font_manager.findSystemFonts(fontpaths=mpl.font_manager.win32FontDirectory(), fontext='ttf')
            
            # === register (new) fonts : Linux / WSL2 --> done automatically if you delete C:/Users/%USERNAME%/.matplotlib/fontlist-v390.json
            mpl.font_manager.findSystemFonts(fontpaths=None, fontext='ttf')
            mpl.font_manager.findSystemFonts(fontpaths=None, fontext='otf')
        
        ## example: list all TTF font properties
        if False:
            fonts = mpl.font_manager.fontManager.ttflist
            #fonts = [f for f in fonts if all([('IBM' in f.name),('Condensed' in f.name)])] ## filter list
            for f in fonts:
                print(f.name)
                print(Path(f.fname).stem)
                print('weight  : %s'%str(f.weight))
                print('style   : %s'%str(f.style))
                print('stretch : %s'%str(f.stretch))
                print('variant : %s'%str(f.variant))
                print('-----'+'\n')
        
        ## get list of names of all registered fonts
        if (font is not None):
            try:
                if hasattr(mpl.font_manager,'get_font_names'):
                    ## Matplotlib >3.6.X
                    fontnames = mpl.font_manager.get_font_names()
                elif hasattr(mpl.font_manager,'get_fontconfig_fonts'):
                    ## Matplotlib <=3.5.X
                    fontlist = mpl.font_manager.get_fontconfig_fonts()
                    fontnames = sorted(list(set([mpl.font_manager.FontProperties(fname=fname).get_name() for fname in fontlist])))
                else:
                    fontnames = None
            except:
                fontnames = None
        
        # === OTF/TTF fonts (when NOT using LaTeX rendering)
        
        if (font==None):
            pass ## do nothing, use system / matplotlib default font
        
        ## Microsoft Aptos
        elif (font=='Aptos') or (font=='aptos'):
            
            if (fontnames is not None) and ('Aptos' in fontnames):
                
                ## condensed
                mpl.rcParams['font.family'] = 'Aptos'
                mpl.rcParams['font.weight'] = 'regular' ## 200, 300/'light', 400, 450
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'Aptos'
                mpl.rcParams['mathtext.it'] = 'Aptos:italic:regular'
                mpl.rcParams['mathtext.bf'] = 'Aptos:bold'
                mpl.rcParams['mathtext.cal'] = 'Aptos:italic'
        
        ## IBM Plex Sans
        elif (font=='IBM Plex Sans') or (font=='IBM Plex') or (font=='IBM') or (font=='ibm'):
            
            if (fontnames is not None) and ('IBM Plex Sans Condensed' in fontnames):
                
                ## condensed
                mpl.rcParams['font.family'] = 'IBM Plex Sans Condensed'
                mpl.rcParams['font.weight'] = 'regular' ## 200, 300/'light', 400, 450
                #mpl.rcParams['font.style'] = 'normal' ## 'normal', 'italic', 'oblique'
                #mpl.rcParams['font.variant'] = 'normal' ## 'normal', 'small-caps'
                #mpl.rcParams['font.stretch'] = 'condensed' ## always 'condensed' for family
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'IBM Plex Sans Condensed:regular'
                mpl.rcParams['mathtext.it'] = 'IBM Plex Sans Condensed:italic:regular'
                mpl.rcParams['mathtext.bf'] = 'IBM Plex Sans Condensed:bold'
                mpl.rcParams['mathtext.cal'] = 'IBM Plex Sans Condensed:italic'
                
                ## regular --> too wide
                # mpl.rcParams['font.family'] = 'IBM Plex Sans'
                # mpl.rcParams['font.weight'] = '400' ## 'light'
                # #mpl.rcParams['font.stretch'] = 'normal' ## always 'normal' for family
                # mpl.rcParams['mathtext.fontset'] = 'custom'
                # mpl.rcParams['mathtext.default'] = 'it'
                # mpl.rcParams['mathtext.rm'] = 'IBM Plex Sans:regular'
                # mpl.rcParams['mathtext.it'] = 'IBM Plex Sans:italic:regular'
                # mpl.rcParams['mathtext.bf'] = 'IBM Plex Sans:bold'
        
        ## Latin Modern Roman (lmodern in LaTeX, often used)
        elif (font=='lmodern') or (font=='Latin Modern') or (font=='Latin Modern Roman') or (font=='lmr'):
            if (fontnames is not None) and ('Latin Modern Roman' in fontnames):
                mpl.rcParams['font.family'] = 'Latin Modern Roman'
                mpl.rcParams['font.weight'] = '400'
                mpl.rcParams['font.style'] = 'normal'
                mpl.rcParams['font.variant'] = 'normal'
                #mpl.rcParams['font.stretch'] = 'condensed'
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'Latin Modern Roman:normal'
                mpl.rcParams['mathtext.it'] = 'Latin Modern Roman:italic'
                mpl.rcParams['mathtext.bf'] = 'Latin Modern Roman:bold'
        
        ## Times New Roman
        elif (font=='times') or (font=='Times') or (font=='Times New Roman'):
            if (fontnames is not None) and ('Times New Roman' in fontnames):
                mpl.rcParams['font.family'] = 'Times New Roman'
                mpl.rcParams['font.weight'] = 'normal'
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'Times New Roman:normal'
                mpl.rcParams['mathtext.it'] = 'Times New Roman:italic'
                mpl.rcParams['mathtext.bf'] = 'Times New Roman:bold'
        
        ## Computer Modern (LaTeX default)
        elif (font=='Computer Modern Roman') or (font=='CMU Serif') or (font=='Computer Modern') or (font=='CMR') or (font=='cmr') or (font=='cmu'):
            if (fontnames is not None) and ('CMU Serif' in fontnames):
                mpl.rcParams['font.family'] = 'CMU Serif'
                mpl.rcParams['font.weight'] = 'regular'
                mpl.rcParams['font.style'] = 'normal'
                mpl.rcParams['mathtext.fontset'] = 'custom'
                mpl.rcParams['mathtext.default'] = 'it'
                mpl.rcParams['mathtext.rm'] = 'CMU Serif:regular'
                mpl.rcParams['mathtext.it'] = 'CMU Serif:italic:regular'
                mpl.rcParams['mathtext.bf'] = 'CMU Serif:bold'
        
        else:
            raise ValueError('font=%s not a valid option'%str(font))
        
        ## Manually point to a TTF/OTF file
        if False:
            #fe = mpl.font_manager.FontEntry(fname='/usr/share/texmf/fonts/opentype/public/lm/lmroman10-regular.otf', name='10erLatin')
            fe = mpl.font_manager.FontEntry(fname='C:/Users/'+os.path.expandvars('%USERNAME%')+'/AppData/Local/Microsoft/Windows/Fonts/lmroman10-regular.otf', name='10erLatin')
            mpl.font_manager.fontManager.ttflist.insert(0, fe)
            mpl.rcParams['font.family'] = fe.name
            mpl.rcParams['font.weight'] = '400'
            mpl.rcParams['font.style'] = 'normal'
            mpl.rcParams['font.variant'] = 'normal'
            mpl.rcParams['mathtext.fontset'] = 'custom'
            mpl.rcParams['mathtext.default'] = 'it'
            mpl.rcParams['mathtext.rm'] = fe.name+':normal'
            mpl.rcParams['mathtext.it'] = fe.name+':italic'
            mpl.rcParams['mathtext.bf'] = fe.name+':bold'
    
    # ===
    
    ## ## list all options
    ## print(mpl.rcParams.keys())
    
    axesAndTickWidth = 0.5
    
    mpl.rcParams['figure.figsize'] = 3, 3/(16/9)
    mpl.rcParams['figure.dpi']     = 200
    #mpl.rcParams['figure.facecolor'] = 'k'
    #mpl.rcParams['figure.autolayout'] = True ### tight_layout() --> just use instead : fig1.tight_layout(pad=0.20)
    
    #mpl.rcParams['figure.constrained_layout.use'] = True
    #mpl.rcParams['figure.constrained_layout.h_pad']  = 0.0 ## Padding around axes objects. Float representing
    #mpl.rcParams['figure.constrained_layout.w_pad']  = 0.0 ## inches. Default is 3/72 inches (3 points)
    #mpl.rcParams['figure.constrained_layout.hspace'] = 0.2 ## Space between subplot groups. Float representing
    #mpl.rcParams['figure.constrained_layout.wspace'] = 0.2 ## a fraction of the subplot widths being separated.
    
    #mpl.rcParams['figure.subplot.bottom'] = 0.02
    #mpl.rcParams['figure.subplot.top'] = 0.98
    #mpl.rcParams['figure.subplot.left'] = 0.02
    #mpl.rcParams['figure.subplot.right'] = 0.98
    #mpl.rcParams['figure.subplot.hspace'] = 0.02
    #mpl.rcParams['figure.subplot.wspace'] = 0.2
    
    #mpl.rcParams['pdf.compression'] = 2 ## 0-9
    #mpl.rcParams['pdf.fonttype'] = 42  # Output Type 3 (Type3) or Type 42 (TrueType)
    #mpl.rcParams['pdf.use14corefonts'] = False
    
    mpl.rcParams['savefig.pad_inches'] = 0.20
    mpl.rcParams['savefig.dpi']        = 1920/3
    
    mpl.rcParams['xtick.major.size']  = 2.5
    mpl.rcParams['xtick.major.width'] = axesAndTickWidth
    mpl.rcParams['xtick.minor.size']  = 1.4
    mpl.rcParams['xtick.minor.width'] = axesAndTickWidth*1.0
    #mpl.rcParams['xtick.color'] = 'k' ## set with mpl.style.use()
    mpl.rcParams['xtick.direction']   = 'in'
    
    mpl.rcParams['ytick.major.size']  = 2.5
    mpl.rcParams['ytick.major.width'] = axesAndTickWidth
    mpl.rcParams['ytick.minor.size']  = 1.4
    mpl.rcParams['ytick.minor.width'] = axesAndTickWidth*1.0
    #mpl.rcParams['ytick.color'] = 'k' ## set with mpl.style.use()
    mpl.rcParams['ytick.direction'] = 'in'
    
    mpl.rcParams['xtick.labelsize'] = fontsize
    mpl.rcParams['ytick.labelsize'] = fontsize
    
    mpl.rcParams['xtick.major.pad'] = 3.0
    mpl.rcParams['xtick.minor.pad'] = 3.0
    mpl.rcParams['ytick.major.pad'] = 3.0
    mpl.rcParams['ytick.minor.pad'] = 3.0
    
    mpl.rcParams['lines.linewidth']  = 1.0
    mpl.rcParams['lines.linestyle']  = 'solid'
    mpl.rcParams['lines.marker']     = 'None' #'o'
    mpl.rcParams['lines.markersize'] = 1.2
    mpl.rcParams['lines.markeredgewidth'] = 0.
    
    #mpl.rcParams['axes.facecolor'] = 'k' ## set with mpl.style.use()
    mpl.rcParams['axes.linewidth'] = axesAndTickWidth
    mpl.rcParams['axes.labelpad']  = 2.0
    mpl.rcParams['axes.titlesize'] = fontsize
    mpl.rcParams['axes.labelsize'] = fontsize
    #mpl.rcParams['axes.formatter.use_mathtext'] = True
    mpl.rcParams['axes.axisbelow'] = False ## dont allow axes, ticks to be under lines --> doesn't work for artist objects with zorder >2.5
    
    mpl.rcParams['legend.fontsize'] = fontsize
    mpl.rcParams['legend.shadow']   = False
    mpl.rcParams['legend.borderpad'] = 0.3
    mpl.rcParams['legend.framealpha'] = 1.0
    mpl.rcParams['legend.edgecolor']  = 'inherit'
    mpl.rcParams['legend.handlelength'] = 1.5 ## linestyle='-.' will get cropped for <1.5
    mpl.rcParams['legend.handletextpad'] = 0.3
    mpl.rcParams['legend.borderaxespad'] = 0.7
    mpl.rcParams['legend.columnspacing'] = 0.5
    mpl.rcParams['legend.fancybox'] = False
    
    return

class mpl_typeset_templates():
    '''
    figure & axis sizes for various typesetting templates
    https://tex.stackexchange.com/questions/8260/what-are-the-various-units-ex-em-in-pt-bp-dd-pc-expressed-in-mm
    ---
    - jfm : Journal of Fluid Mechanics
        - \textwidth is 384 [pt] / ~5.3134 [in]
        - figures have 9pt fontsize
    '''
    def __init__(self, template='jfm', aspect=1.6, nx=1):
        if (template=='jfm'):
            '''
            see jfm.cls
            '''
            self.fontsize   = 9 ## fontsize in FIGURE env
            self.textwidth  = 32*0.16605 ## [pc]*[in/pc] = 5.3136 [in]
            self.textheight = 600/72.26999 ## [pt]/[pt/in] = 8.3022012318 [in]
            self.textwidth  = round(self.textwidth,4) ## = 5.3136 [in]
            self.textheight = round(self.textheight,4) ## = 8.3022 [in]
        else:
            raise ValueError
        
        self.nx = nx ## number of panels in [x]
        self.aspect = aspect
        
        ## [in]
        self.width   = self.textwidth / self.nx
        self.height  = self.width / self.aspect
        self.figsize = (self.width,self.height)
        
        if (template=='jfm'):
            if (self.height > self.textheight):
                raise ValueError(f'height {self.height:0.8f} [in] is >textheight = {self.textheight:0.8f} [in]')

def colors_table():
    '''
    a table of color hues
    -----
    clrs = colors_table()
    red = clrs['red'][9]
    -----
    https://yeun.github.io/open-color/
    https://twitter.com/nprougier/status/1323575342204936192
    https://github.com/rougier/scientific-visualization-book/blob/master/code/colors/open-colors.py
    '''
    
    colors = {  "gray"   :  { 0: '#f8f9fa', 1: '#f1f3f5', 2: '#e9ecef', 3: '#dee2e6', 4: '#ced4da', 
                              5: '#adb5bd', 6: '#868e96', 7: '#495057', 8: '#343a40', 9: '#212529', },
                "red"    :  { 0: '#fff5f5', 1: '#ffe3e3', 2: '#ffc9c9', 3: '#ffa8a8', 4: '#ff8787', 
                              5: '#ff6b6b', 6: '#fa5252', 7: '#f03e3e', 8: '#e03131', 9: '#c92a2a', },
                "pink"   :  { 0: '#fff0f6', 1: '#ffdeeb', 2: '#fcc2d7', 3: '#faa2c1', 4: '#f783ac',
                              5: '#f06595', 6: '#e64980', 7: '#d6336c', 8: '#c2255c', 9: '#a61e4d', },
                "grape"  :  { 0: '#f8f0fc', 1: '#f3d9fa', 2: '#eebefa', 3: '#e599f7', 4: '#da77f2',
                              5: '#cc5de8', 6: '#be4bdb', 7: '#ae3ec9', 8: '#9c36b5', 9: '#862e9c', },
                "violet" :  { 0: '#f3f0ff', 1: '#e5dbff', 2: '#d0bfff', 3: '#b197fc', 4: '#9775fa',
                              5: '#845ef7', 6: '#7950f2', 7: '#7048e8', 8: '#6741d9', 9: '#5f3dc4', },
                "indigo" :  { 0: '#edf2ff', 1: '#dbe4ff', 2: '#bac8ff', 3: '#91a7ff', 4: '#748ffc', 
                              5: '#5c7cfa', 6: '#4c6ef5', 7: '#4263eb', 8: '#3b5bdb', 9: '#364fc7', },
                "blue"   :  { 0: '#e7f5ff', 1: '#d0ebff', 2: '#a5d8ff', 3: '#74c0fc', 4: '#4dabf7',
                              5: '#339af0', 6: '#228be6', 7: '#1c7ed6', 8: '#1971c2', 9: '#1864ab', },
                "cyan"   :  { 0: '#e3fafc', 1: '#c5f6fa', 2: '#99e9f2', 3: '#66d9e8', 4: '#3bc9db',
                              5: '#22b8cf', 6: '#15aabf', 7: '#1098ad', 8: '#0c8599', 9: '#0b7285', },
                "teal"   :  { 0: '#e6fcf5', 1: '#c3fae8', 2: '#96f2d7', 3: '#63e6be', 4: '#38d9a9',
                              5: '#20c997', 6: '#12b886', 7: '#0ca678', 8: '#099268', 9: '#087f5b', },
                "green"  :  { 0: '#ebfbee', 1: '#d3f9d8', 2: '#b2f2bb', 3: '#8ce99a', 4: '#69db7c',
                              5: '#51cf66', 6: '#40c057', 7: '#37b24d', 8: '#2f9e44', 9: '#2b8a3e', },
                "lime"   :  { 0: '#f4fce3', 1: '#e9fac8', 2: '#d8f5a2', 3: '#c0eb75', 4: '#a9e34b',
                              5: '#94d82d', 6: '#82c91e', 7: '#74b816', 8: '#66a80f', 9: '#5c940d', },
                "yellow" :  { 0: '#fff9db', 1: '#fff3bf', 2: '#ffec99', 3: '#ffe066', 4: '#ffd43b',
                              5: '#fcc419', 6: '#fab005', 7: '#f59f00', 8: '#f08c00', 9: '#e67700', },
                "orange" :  { 0: '#fff4e6', 1: '#ffe8cc', 2: '#ffd8a8', 3: '#ffc078', 4: '#ffa94d',
                              5: '#ff922b', 6: '#fd7e14', 7: '#f76707', 8: '#e8590c', 9: '#d9480f', }, }
    
    return colors

def get_standard_colors():
    '''
    a convenience func to return some colors from colors_table()
    '''
    
    cc = colors_table()
    
    red,pink,grape,violet,indigo,blue,cyan,teal,green,lime,yellow,orange = \
    cc['red'][7],\
    cc['pink'][3],\
    cc['grape'][6],\
    cc['violet'][6],\
    cc['indigo'][8],\
    cc['blue'][6],\
    cc['cyan'][3],\
    cc['teal'][3],\
    cc['green'][7],\
    cc['lime'][4],\
    cc['yellow'][4],\
    cc['orange'][6]
    
    colors_str = [ 'red','pink','grape','violet','indigo','blue','cyan','teal','green','lime','yellow','orange' ]
    colors = [ red,pink,grape,violet,indigo,blue,cyan,teal,green,lime,yellow,orange ]
    
    red_dk,pink_dk,grape_dk,violet_dk,indigo_dk,blue_dk,cyan_dk,teal_dk,green_dk,lime_dk,yellow_dk,orange_dk = \
    cc['red'][6+2],\
    cc['pink'][3+2],\
    cc['grape'][6+2],\
    cc['violet'][6+2],\
    cc['indigo'][8+2-1],\
    cc['blue'][6+2],\
    cc['cyan'][3+2],\
    cc['teal'][3+2],\
    cc['green'][7+2],\
    cc['lime'][4+2],\
    cc['yellow'][4+2],\
    cc['orange'][6+2]
    
    colors_dark_str = [ 'red_dk','pink_dk','grape_dk','violet_dk','indigo_dk','blue_dk','cyan_dk','teal_dk','green_dk','lime_dk','yellow_dk','orange_dk' ]
    colors_dark = [red_dk,pink_dk,grape_dk,violet_dk,indigo_dk,blue_dk,cyan_dk,teal_dk,green_dk,lime_dk,yellow_dk,orange_dk]
    
    ## to dict
    colors    = dict(zip(colors_str, colors))
    colors_dk = dict(zip(colors_dark_str, colors_dark))
    
    return colors, colors_dk

def cmap_Lch_edit(cmap,n=256,**kwargs):
    '''
    scale the L*c*h channels of a colormap
    '''
    
    L1 = kwargs.get('L1',0.4)
    L2 = kwargs.get('L2',95)
    
    c1 = kwargs.get('c1',0.6)
    c2 = kwargs.get('c2',100)
    
    type_valid=False
    if isinstance(cmap, mpl.colors.ListedColormap):
        type_valid=True
    if isinstance(cmap, mpl.colors.LinearSegmentedColormap):
        type_valid=True
    if not type_valid:
        raise ValueError
    
    x    = np.linspace(0,1,n,dtype=np.float64)
    rgba = cmap(x)
    rgb  = np.copy(rgba[:,:3])
    Lch  = cspace_converter('sRGB1', 'CIELCh')(rgb)
    
    Lmin = Lch[:,0].min()
    Lmax = Lch[:,0].max()
    Lrng = Lmax - Lmin
    
    cmin = Lch[:,1].min()
    cmax = Lch[:,1].max()
    crng = cmax - cmin
    
    hmin = Lch[:,2].min()
    hmax = Lch[:,2].max()
    hrng = hmax - hmin
    
    for i in range(n):
        L,c,h = Lch[i,:]
        L = ( L1 + (L-Lmin)/Lrng*(1-L1)) * L2
        c = ( c1 + (c-cmin)/crng*(1-c1)) * c2
        Lch[i,:] = np.array([L,c,h])
    
    ## test plot : Lch
    if False:
        plt.close('all')
        fig1 = plt.figure(figsize=(5,5/(16/9)), dpi=300)
        ax1 = plt.gca()
        ax1.tick_params(axis='x', which='both', direction='in')
        ax1.tick_params(axis='y', which='both', direction='in')
        ax1.xaxis.set_ticks_position('both')
        ax1.yaxis.set_ticks_position('both')
        ln1, = ax1.plot( x ,Lch[:,0], c='red'   , lw=1.0 , )
        ln1, = ax1.plot( x, Lch[:,1], c='blue'  , lw=1.0 , )
        ln1, = ax1.plot( x, Lch[:,2]*100/360, c='green' , lw=1.0 , )
        ax1.set_xlim(0,1)
        ax1.set_ylim(0,100)
        fig1.tight_layout(pad=0.25)
        fig1.tight_layout(pad=0.25)
        #dpi_out = 2160/plt.gcf().get_size_inches()[1]
        #fig1.savefig('Lch.png', dpi=dpi_out)
        plt.show()
    
    ## convert LCH --> RGB
    rgbn = cspace_converter('CIELCh', 'sRGB1')(Lch)
    
    # print('R min max %0.2f %0.2f'%(rgbn[:,0].min(), rgbn[:,0].max()))
    # print('G min max %0.2f %0.2f'%(rgbn[:,1].min(), rgbn[:,1].max()))
    # print('B min max %0.2f %0.2f'%(rgbn[:,2].min(), rgbn[:,2].max()))
    
    rgbn = np.clip(rgbn,0,1)
    
    return rgbn

def colors_test_plot(color_dict):
    '''
    plot test image for color palette
    '''
    
    plt.close('all')
    fig1 = plt.figure(figsize=(2.5*(16/9),2.5), dpi=200)
    ax1 = plt.gca()
    ax1.tick_params(axis='x', which='both', direction='in')
    ax1.tick_params(axis='y', which='both', direction='in')
    ax1.xaxis.set_ticks_position('both')
    ax1.yaxis.set_ticks_position('both')
    x_ = np.linspace(0,np.pi,500)
    n_colors = len(color_dict)
    i=-1
    for color_name, color_val in color_dict.items():
        i+=1
        phase_ = - np.pi*(i/(n_colors-1))
        ln1, = ax1.plot( x_,
                         np.cos(2*x_ + phase_ ),
                         c=color_val,
                         lw=1.0, ls='solid', zorder=1 )
        print(f'{color_name} {str(color_val)}')
    fig1.tight_layout(pad=0.25)
    fig1.tight_layout(pad=0.25)
    plt.show()
    
    return

def color_dict_to_tex(color_dict):
    '''
    print colors as Tex definitions
    '''
    for color_name, color_val in color_dict.items():
        print(f'\\definecolor{{{color_name}}}{{HTML}}{{{color_val.replace(r"#",r"")}}}')
    return

def get_Lch_colors(hues,**kwargs):
    '''
    given a list of hues [0-360], chroma & luminance, return
        colors in hex (html) or rgb format
    -----
    Lch color picker : https://css.land/lch/
    '''
    
    c = kwargs.get('c',110) ## chroma
    L = kwargs.get('L',65) ## luminance
    fmt = kwargs.get('fmt','rgb') ## output format : hex or rgb tuples
    test_plot = kwargs.get('test_plot',False) ## plot to test colors
    
    colors_rgb=[]
    colors_Lab=[]
    for h in hues:
        hX = h*np.pi/180
        LX,a,b = skimage.color.lch2lab([L,c,hX])
        colors_Lab.append([LX,a,b])
        r,g,b = skimage.color.lab2rgb([LX,a,b])
        colors_rgb.append([r,g,b])
    
    if test_plot:
        nc = len(colors_rgb)
        x = np.linspace(0,2*np.pi,1000)
        plt.close('all')
        fig1 = plt.figure(figsize=(4, 4*(9/16)), dpi=300)
        ax1 = plt.gca()
        for i in range(nc):
            ax1.plot(x, np.cos(x-i*np.pi/nc), c=colors_rgb[i])
        fig1.tight_layout(pad=0.15)
        plt.show()
    
    if (fmt=='rgb'):
        colors=colors_rgb
    elif (fmt=='hex'):
        colors=[mpl.colors.to_hex(c) for c in colors_rgb]
    elif (fmt=='Lab'):
        colors=colors_Lab
    else:
        raise NameError('fmt=%s not a valid option'%str(fmt))
    
    return colors

def hex2rgb(hexstr,**kwargs):
    '''
    return (r,g,b) [0-1] from html/hexadecimal
    '''
    base = kwargs.get('base',1)
    hexstr = hexstr.lstrip('#')
    c = tuple(int(hexstr[i:i+2], 16) for i in (0, 2, 4))
    if (base==1):
        c = tuple(i/255. for i in c)
    return c

def hsv_adjust_hex(hex_list,h_fac,s_fac,v_fac,**kwargs):
    '''
    adjust the (h,s,v) values of a list of html color codes (#XXXXXX)
    --> if single #XXXXXX is passed, returns single
    --> margin : adjust proportional to available margin
    '''
    margin = kwargs.get('margin',False)
    
    if isinstance(hex_list, str):
        single=True
        hex_list = [hex_list]
    else:
        single=False
    
    colors_rgb = [ hex2rgb(c) for c in hex_list ]
    colors_hsv = mpl.colors.rgb_to_hsv(colors_rgb)
    for ci in range(len(colors_hsv)):
        c = colors_hsv[ci]
        h, s, v = c
        if margin:
            h_margin = 1. - h
            s_margin = 1. - s
            v_margin = 1. - v
        else:
            h_margin = 1.
            s_margin = 1.
            v_margin = 1.
        h = max(0.,min(1.,h + h_margin*h_fac))
        s = max(0.,min(1.,s + s_margin*s_fac))
        v = max(0.,min(1.,v + v_margin*v_fac))
        colors_hsv[ci] = (h,s,v)
    colors_rgb = mpl.colors.hsv_to_rgb(colors_hsv)
    hex_list_out = [mpl.colors.to_hex(c).upper() for c in colors_rgb]
    if single:
        hex_list_out=hex_list_out[0] ## just the one tuple
    return hex_list_out

def axs_grid_initializer(**kwargs):
    '''
    initialize several axes in a grid
    '''
    
    w_prop    = kwargs.get('w_prop',True) ## peg to width vals
    figsize_x = kwargs.get('figsize_x',6)
    figsize_y = kwargs.get('figsize_y',4) ## figsizes in 
    mrg_l     = kwargs.get('mrg_l',0.05)
    mrg_r     = kwargs.get('mrg_r',0.05)
    mrg_w     = kwargs.get('mrg_w',0.05)
    mrg_t     = kwargs.get('mrg_t',0.05)
    mrg_b     = kwargs.get('mrg_b',0.05)
    mrg_h     = kwargs.get('mrg_h',0.05)
    
    cols = kwargs.get('cols',2)
    rows = kwargs.get('rows',1)
    
    dpi_fig = kwargs.get('dpi_fig',110)
    
    
    fig = plt.figure(figsize=(figsize_x,figsize_y), dpi=dpi_fig)
    aspect = figsize_x/figsize_y
    if w_prop:
        mrg_t *= aspect
        mrg_b *= aspect
        mrg_h *= aspect
    
    ax_width  = (1.0 - (cols-1)*mrg_w - mrg_l - mrg_r)/cols
    ax_height = (1.0 - (rows-1)*mrg_h - mrg_t - mrg_b)/rows
    axs          = np.zeros(shape=(cols,rows),   dtype=object)
    ax_grid_dims = np.zeros(shape=(cols,rows,4), dtype=np.float64) ## the original axis positions
    for j in range(rows):
        for i in range(cols):
            ii = i ## left to right
            jj = (rows-1) - j ## top to bottom
            x0 = mrg_l + ii*mrg_w + ii*ax_width
            dx = ax_width
            y0 = mrg_b + jj*mrg_h + jj*ax_height
            dy = ax_height
            ax = fig.add_axes([x0,y0,dx,dy])
            
            ax_grid_dims[i,j,:] = x0,y0,dx,dy
            axs[i,j] = ax
    
    ## axs          = np.asarray(axs)
    ## ax_grid_dims = np.asarray(ax_grid_dims, dtype=object)
    ## axs          = np.reshape(axs,          (cols, rows),    order='F')
    ## ax_grid_dims = np.reshape(ax_grid_dims, (cols, rows, 4), order='F') ; print(ax_grid_dims.shape)
    
    return fig, axs, ax_grid_dims

def axs_grid_compress(fig,axs,**kwargs):
    '''
    compress ax grid
    '''
    dim = kwargs.get('dim',1)
    offset_px = kwargs.get('offset_px',5)
    transFigInv = fig.transFigure.inverted()
    
    ## on screen pixel size
    fig_px_x, fig_px_y = fig.get_size_inches()*fig.dpi
    #print('fig size px : %0.3f %0.3f'%(fig_px_x, fig_px_y))
    
    cols, rows = axs.shape
    for j in range(rows-1):
        
        ### determine the min x0 in each row
        top_row_y0s = []
        low_row_y1s = []
        for i in range(cols):
            
            x0,  y0,  dx,  dy  = axs[i,j+1].get_position().bounds
            
            ### pixel values of the axis tightbox
            x0A, y0A, dxA, dyA = axs[i,j+0].get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds
            x0B, y0B, dxB, dyB = axs[i,j+1].get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds
            #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
            #print('x0B, y0B, dxB, dyB : %0.2f %0.2f %0.2f %0.2f'%(x0B, y0B, dxB, dyB))
            
            ### convert pixel vals to dimless
            x0A, y0A = transFigInv.transform_point([x0A, y0A])
            dxA, dyA = transFigInv.transform_point([dxA, dyA])
            x0B, y0B = transFigInv.transform_point([x0B, y0B])
            dxB, dyB = transFigInv.transform_point([dxB, dyB])
            #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
            #print('x0B, y0B, dxB, dyB : %0.2f %0.2f %0.2f %0.2f'%(x0B, y0B, dxB, dyB))
            #print('\n')
            
            top_row_y0s.append(y0A)
            low_row_y1s.append(y0B+dyB)
        
        y_shift = min(top_row_y0s) - max(low_row_y1s)
        
        # =====
        
        for i in range(cols):
            
            x0,  y0,  dx,  dy  = axs[i,j+1].get_position().bounds
            
            # x0A, y0A, dxA, dyA = axs[i,j+0].get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds
            # x0B, y0B, dxB, dyB = axs[i,j+1].get_tightbbox(fig.canvas.get_renderer(), call_axes_locator=False).bounds
            # #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
            # #print('x0B, y0B, dxB, dyB : %0.2f %0.2f %0.2f %0.2f'%(x0B, y0B, dxB, dyB))
            # 
            # ### convert pixel vals to dimless
            # x0A, y0A = transFigInv.transform_point([x0A, y0A])
            # dxA, dyA = transFigInv.transform_point([dxA, dyA])
            # x0B, y0B = transFigInv.transform_point([x0B, y0B])
            # dxB, dyB = transFigInv.transform_point([dxB, dyB])
            # #print('x0A, y0A, dxA, dyA : %0.2f %0.2f %0.2f %0.2f'%(x0A, y0A, dxA, dyA))
            # #print('x0B, y0B, dxB, dyB : %0.2f %0.2f %0.2f %0.2f'%(x0B, y0B, dxB, dyB))
            # #print('\n')
            
            x0n = x0
            #y0n = y0+(y0A-(y0B+dyB))-(offset_px/fig_px_y)
            y0n = y0+y_shift-(offset_px/fig_px_y)
            dxn = dx
            dyn = dy
            
            axs[i,j+1].set_position([x0n,y0n,dxn,dyn])
    
    return

def cmap_convert_mpl_to_pview(cmap,fname,cmap_name,**kwargs):
    '''
    convert python/matplotlib cmap object to JSON for Paraview
    '''
    
    # === we dont want to mess up the current cmap/norm when we set_array() and autoscale()
    cmapX = copy.deepcopy(cmap)
    #normX = copy.deepcopy(norm)
    
    N = kwargs.get('N',256)
    #lo = kwargs.get('lo',0.)
    #hi = kwargs.get('hi',1.)
    #norm = kwargs.get('norm',mpl.colors.Normalize(vmin=0.0, vmax=1.0))
    
    sclMap = mpl.cm.ScalarMappable(cmap=cmapX)
    x      = np.linspace(0,1,N+1)
    sclMap.set_array(x)
    sclMap.autoscale()
    colors = sclMap.to_rgba(x)
    
    ## output .json formatted ascii file
    #f = open(fname,'w')
    f = io.open(fname,'w',newline='\n')
    
    out_str='''[
    {
        "ColorSpace" : "RGB",
        "Name" : "%s",
        "RGBPoints" : 
        ['''%(cmap_name, )
    
    f.write(out_str)
    
    for i in range(len(x)):
        c = x[i]
        if (i==len(x)-1):
            maybeComma=''
        else:
            maybeComma=','
        color=colors[i]
        out_str='''
            %0.6f,
            %0.17f,
            %0.17f,
            %0.17f%s'''%(c,color[0],color[1],color[2],maybeComma)
        f.write(out_str)
    
    out_str = '''\n%s]\n%s}\n]'''%(8*' ',4*' ')
    f.write(out_str)
    f.close()
    print('--w-> %s'%fname)
    return

# main()
# ======================================================================

if __name__ == '__main__':
    pass
