import io
import os
import re
import shutil
import subprocess
import sys
import textwrap
import time
import timeit
import types
from pathlib import Path

import h5py
import numpy as np
from mpi4py import MPI
from tqdm import tqdm

from .eas4 import eas4
from .h5 import h5_chunk_sizer, h5_ds_force_allocate_chunks
from .utils import even_print, format_time_string

# ======================================================================

class cgd(h5py.File):
    '''
    Curvilinear Grid Data (CGD)
    ---------------------------
    - super()'ed h5py.File class
    '''
    
    def __init__(self, *args, **kwargs):
        
        self.fname, self.open_mode = args
        
        self.fname_path = os.path.dirname(self.fname)
        self.fname_base = os.path.basename(self.fname)
        self.fname_root, self.fname_ext = os.path.splitext(self.fname_base)
        
        ## default to libver='latest' if none provided
        if ('libver' not in kwargs):
            kwargs['libver'] = 'latest'
        
        ## catch possible user error --> could prevent accidental EAS overwrites
        if (self.fname_ext=='.eas'):
            raise ValueError('EAS4 files should not be opened with turbx.cgd()')
        
        ## check if none-None communicator, but no driver='mpio'
        if ('comm' in kwargs) and (kwargs['comm'] is not None) and ('driver' not in kwargs):
            raise ValueError("comm is provided as not None, but driver='mpio' not provided")
        
        ## determine if using mpi
        if ('driver' in kwargs) and (kwargs['driver']=='mpio'):
            if ('comm' not in kwargs):
                raise ValueError("if driver='mpio', then comm should be provided")
            self.usingmpi = True
        else:
            self.usingmpi = False
        
        ## determine communicator & rank info
        if self.usingmpi:
            self.comm    = kwargs['comm']
            self.n_ranks = self.comm.Get_size()
            self.rank    = self.comm.Get_rank()
        else:
            self.comm    = None
            self.n_ranks = 1
            self.rank    = 0
            if ('comm' in kwargs):
                del kwargs['comm']
        
        ## cgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        stripe_count   = kwargs.pop('stripe_count'   , 16    )
        stripe_size_mb = kwargs.pop('stripe_size_mb' , 2     )
        perms          = kwargs.pop('perms'          , '640' )
        no_indep_rw    = kwargs.pop('no_indep_rw'    , False )
        
        ## passthrough arg to get_header() to automatically read the full 3D grid on every MPI rank
        ## upon opening. This is in general a bad idea for CGD, which has a 3D grid. If every rank reads the
        ## full grid, RAM can very quickly fill up.
        read_grid = kwargs.pop('read_grid', False)
        
        if not isinstance(stripe_count, int):
            raise ValueError('stripe_count must be int')
        if not isinstance(stripe_size_mb, int):
            raise ValueError('stripe_size_mb must be int')
        if not isinstance(perms, str) or len(perms)!=3 or not re.fullmatch(r'\d{3}',perms):
            raise ValueError("perms must be 3-digit string like '660'")
        
        ## if not using MPI, remove 'driver' and 'comm' from kwargs
        if ( not self.usingmpi ) and ('driver' in kwargs):
            kwargs.pop('driver')
        if ( not self.usingmpi ) and ('comm' in kwargs):
            kwargs.pop('comm')
        
        ## | mpiexec --mca io romio321 -n $NP python3 ...
        ## | mpiexec --mca io ompio -n $NP python3 ...
        ## | ompi_info --> print ompi settings (grep 'MCA io' for I/O opts)
        ## | export ROMIO_FSTYPE_FORCE="lustre:" --> force Lustre driver over UFS when using ROMIO
        ## | export ROMIO_FSTYPE_FORCE="ufs:"
        ## | export ROMIO_PRINT_HINTS=1 --> show available hints
        ##
        ## https://doku.lrz.de/best-practices-hints-and-optimizations-for-io-10747318.html
        ##
        ## ## Using OMPIO
        ## export OMPI_MCA_sharedfp=^lockedfile,individual
        ## mpiexec --mca io ompio -n $NP python3 script.py
        ##
        ## ## Using Cray MPICH
        ## to print ROMIO hints : export MPICH_MPIIO_HINTS_DISPLAY=1
        
        ## set MPI hints, passed through 'mpi_info' dict
        if self.usingmpi:
            if ('info' in kwargs):
                self.mpi_info = kwargs['info']
            else:
                mpi_info = MPI.Info.Create()
                
                ## ROMIO -- data sieving & collective buffering
                mpi_info.Set('romio_ds_write' , 'disable'   ) ## ds = data sieving
                mpi_info.Set('romio_ds_read'  , 'disable'   )
                #mpi_info.Set('romio_cb_read'  , 'automatic' ) ## cb = collective buffering
                #mpi_info.Set('romio_cb_write' , 'automatic' )
                mpi_info.Set('romio_cb_read'  , 'enable' ) ## cb = collective buffering
                mpi_info.Set('romio_cb_write' , 'enable' )
                
                ## ROMIO -- collective buffer size
                mpi_info.Set('cb_buffer_size' , str(int(round(1*1024**3))) ) ## 1 [GB]
                
                ## ROMIO -- force collective I/O
                if no_indep_rw:
                    mpi_info.Set('romio_no_indep_rw' , 'true' )
                
                ## ROMIO -- N Aggregators
                #mpi_info.Set('cb_nodes' , str(min(16,self.n_ranks//2)) )
                mpi_info.Set('cb_nodes' , str(min(16,self.n_ranks)) )
                
                ## add to kwargs to be passed to h5py.File() at super() call
                kwargs['info'] = mpi_info
                self.mpi_info = mpi_info
        
        # === HDF5 tuning factors (independent of MPI I/O driver)
        
        ## rdcc_w0 : preemption policy (weight) for HDF5's raw data chunk cache
        ## - influences how HDF5 evicts chunks from the per-process chunk cache
        ## - 1.0 favors retaining fully-read chunks (good for read-heavy access)
        ## - 0.0 favors recently-used chunks (better for partial writes)
        if ('rdcc_w0' not in kwargs):
            kwargs['rdcc_w0'] = 0.75
        
        ## rdcc_nbytes : maximum total size of the HDF5 raw chunk cache per dataset per process
        if ('rdcc_nbytes' not in kwargs):
            kwargs['rdcc_nbytes'] = int(1*1024**3) ## 1 [GB]
        
        ## rdcc_nslots : number of hash table slots in the raw data chunk cache
        ## - should be ~= ( rdcc_nbytes / chunk size )
        if ('rdcc_nslots' not in kwargs):
            #kwargs['rdcc_nslots'] = 16381 ## prime
            kwargs['rdcc_nslots'] = kwargs['rdcc_nbytes'] // (2*1024**2) ## assume 2 [MB] chunks
            #kwargs['rdcc_nslots'] = kwargs['rdcc_nbytes'] // (128*1024**2) ## assume 128 [MB] chunks
        
        ## cgd() unique kwargs (not h5py.File kwargs) --> pop() rather than get()
        verbose = kwargs.pop( 'verbose' , False )
        force   = kwargs.pop( 'force'   , False )
        
        if not isinstance(verbose, bool):
            raise ValueError
        if not isinstance(force, bool):
            raise ValueError
        
        # === initialize file on FS
        
        ## if file open mode is 'w', the file exists, and force is False
        ## --> raise error
        if (self.open_mode == 'w') and (force is False) and os.path.isfile(self.fname):
            if (self.rank==0):
                print('\n'+72*'-')
                print(self.fname+' already exists! opening with \'w\' would overwrite.\n')
                openModeInfoStr = '''
                                  r       : Read only, file must exist
                                  r+      : Read/write, file must exist
                                  w       : Create file, truncate if exists
                                  w- or x : Create file, fail if exists
                                  a       : Read/write if exists, create otherwise
                                  
                                  or use force=True arg:
                                  
                                  >>> with cgd(<<fname>>,'w',force=True) as f:
                                  >>>     ...
                                  '''
                print(textwrap.indent(textwrap.dedent(openModeInfoStr), 2*' ').strip('\n'))
                print(72*'-'+'\n')
                sys.stdout.flush()
            
            if (self.comm is not None):
                self.comm.Barrier()
            raise FileExistsError
        
        ## if file open mode is 'w'
        ## --> <delete>, touch, chmod, stripe
        if (self.open_mode == 'w'):
            if (self.rank==0):
                if os.path.isfile(self.fname): ## if the file exists, delete it
                    os.remove(self.fname)
                    time.sleep(1.)
                Path(self.fname).touch() ## touch a new file
                time.sleep(1.)
                os.chmod(self.fname, int(perms, base=8)) ## change permissions
                if shutil.which('lfs') is not None: ## set stripe if on Lustre
                    cmd_str_lfs_migrate = f'lfs migrate --stripe-count {stripe_count:d} --stripe-size {stripe_size_mb:d}M {self.fname} > /dev/null 2>&1'
                    return_code = subprocess.call(cmd_str_lfs_migrate, shell=True)
                    if (return_code != 0):
                        raise ValueError('lfs migrate failed')
                    time.sleep(1.)
        
        if (self.comm is not None):
            self.comm.Barrier()
        
        ## call actual h5py.File.__init__()
        super(cgd, self).__init__(*args, **kwargs)
        self.get_header(verbose=verbose, read_grid=read_grid)
        return
    
    def get_header(self,**kwargs):
        '''
        Helper for __init__
        Read attributes of CGD class instance & attach
        '''
        
        verbose = kwargs.get('verbose',True)
        
        ## by default, do NOT read CGD grid upon opening
        ## leave it up to the individual functions to read grid into RAM
        read_grid = kwargs.get('read_grid',False) 
        
        if (self.rank!=0):
            verbose=False
        
        # === attrs
        
        if ('duration_avg' in self.attrs.keys()):
            self.duration_avg = self.attrs['duration_avg']
        if ('rectilinear' in self.attrs.keys()):
            self.rectilinear = self.attrs['rectilinear']
        if ('curvilinear' in self.attrs.keys()):
            self.curvilinear = self.attrs['curvilinear']
        
        ## these should be set in the (init_from_() funcs)
        if ('fclass' in self.attrs.keys()):
            self.fclass = self.attrs['fclass'] ## 'rgd','cgd',...
        if ('fsubtype' in self.attrs.keys()):
            self.fsubtype = self.attrs['fsubtype'] ## 'unsteady','mean','prime',...
        
        # === udef
        
        ## Base
        header_attr_keys = [
            'Ma','Re','Pr',
            'kappa','R',
            'p_inf','T_inf',
            'S_Suth','mu_Suth_ref','T_Suth_ref',
            ]
        
        ## Derived
        header_attr_keys_derived = [
            'C_Suth','mu_inf','rho_inf','nu_inf',
            'a_inf','U_inf',
            'cp','cv',
            'recov_fac','Taw',
            'lchar','tchar',
            'uchar','M_inf',
            'p_tot_inf', 'T_tot_inf', 'rho_tot_inf',
            ]
        
        ## Read,attach -- base attributes
        for key in header_attr_keys:
            if key in self.attrs:
                setattr(self, key, self.attrs[key]) ## Attach to cgd() instance
        
        ## Read,attach -- derived attributes
        for key in header_attr_keys_derived:
            if key in self.attrs:
                setattr(self, key, self.attrs[key]) ## Attach to cgd() instance
        
        ## Check derived
        if all([ hasattr(self,key) for key in header_attr_keys ]): ## Has all base attrs
            
            cc = types.SimpleNamespace() ## Temporary obj
            
            ## Re-calculate derived attrs for assertion
            cc.C_Suth      = self.mu_Suth_ref/(self.T_Suth_ref**(3/2))*(self.T_Suth_ref + self.S_Suth) ## [kg/(m·s·√K)]
            cc.mu_inf      = self.mu_Suth_ref*(self.T_inf/self.T_Suth_ref)**(3/2) * ((self.T_Suth_ref+self.S_Suth)/(self.T_inf+self.S_Suth))
            cc.rho_inf     = self.p_inf/(self.R*self.T_inf)
            #cc.nu_inf      = self.mu_inf/self.rho_inf ## no!
            cc.nu_inf      = cc.mu_inf/cc.rho_inf
            cc.a_inf       = np.sqrt(self.kappa*self.R*self.T_inf)
            cc.U_inf       = self.Ma*self.a_inf
            cc.cp          = self.R*self.kappa/(self.kappa-1.)
            cc.cv          = self.cp/self.kappa
            cc.recov_fac   = self.Pr**(1/3) ## for turbulent boundary layer
            cc.Taw         = self.T_inf + self.recov_fac*self.U_inf**2/(2*self.cp)
            cc.lchar       = self.Re*self.nu_inf/self.U_inf
            cc.tchar       = self.lchar / self.U_inf
            cc.uchar       = self.U_inf
            cc.p_tot_inf   = self.p_inf   * (1 + (self.kappa-1)/2 * self.M_inf**2)**(self.kappa/(self.kappa-1))
            cc.T_tot_inf   = self.T_inf   * (1 + (self.kappa-1)/2 * self.M_inf**2)
            cc.rho_tot_inf = self.rho_inf * (1 + (self.kappa-1)/2 * self.M_inf**2)**(1/(self.kappa-1))
            
            ## (1) Assert that re-calculated derived attrs are equal to HDF5 top-level attrs of same name
            ## (2) If for some reason a derived attribute was not cached in HDF5, then
            ##      attach the re-calculated value silently to self
            for key in header_attr_keys_derived:
                if hasattr(cc,key):
                    if hasattr(self,key):
                        np.testing.assert_allclose(
                            getattr(self,key),
                            getattr(cc,key),
                            rtol=1e-8,
                            atol=1e-8,
                            )
                    else:
                        setattr(self,key,getattr(cc, key))
            
            #if verbose: print(72*'-')
            if verbose: even_print('Ma'          , '%0.2f [-]'           % self.Ma          )
            if verbose: even_print('Re'          , '%0.1f [-]'           % self.Re          )
            if verbose: even_print('Pr'          , '%0.3f [-]'           % self.Pr          )
            if verbose: even_print('T_inf'       , '%0.3f [K]'           % self.T_inf       )
            if verbose: even_print('p_inf'       , '%0.1f [Pa]'          % self.p_inf       )
            if verbose: even_print('kappa'       , '%0.3f [-]'           % self.kappa       )
            if verbose: even_print('R'           , '%0.3f [J/(kg·K)]'    % self.R           )
            if verbose: even_print('mu_Suth_ref' , '%0.6E [kg/(m·s)]'    % self.mu_Suth_ref )
            if verbose: even_print('T_Suth_ref'  , '%0.2f [K]'           % self.T_Suth_ref  )
            if verbose: even_print('S_Suth'      , '%0.2f [K]'           % self.S_Suth      )
            if verbose: even_print('C_Suth'      , '%0.5e [kg/(m·s·√K)]' % self.C_Suth      )
            
            if verbose: print(72*'-')
            if verbose: even_print('rho_inf'         , '%0.3f [kg/m³]'    % self.rho_inf   )
            if verbose: even_print('mu_inf'          , '%0.6E [kg/(m·s)]' % self.mu_inf    )
            if verbose: even_print('nu_inf'          , '%0.6E [m²/s]'     % self.nu_inf    )
            if verbose: even_print('a_inf'           , '%0.6f [m/s]'      % self.a_inf     )
            if verbose: even_print('U_inf'           , '%0.6f [m/s]'      % self.U_inf     )
            if verbose: even_print('cp'              , '%0.3f [J/(kg·K)]' % self.cp        )
            if verbose: even_print('cv'              , '%0.3f [J/(kg·K)]' % self.cv        )
            #if verbose: even_print('recovery factor' , '%0.6f [-]'        % self.recov_fac )
            if verbose: even_print('Taw'             , '%0.3f [K]'        % self.Taw       )
            if verbose: even_print('lchar'           , '%0.6E [m]'        % self.lchar     )
            if verbose: even_print('tchar'           , '%0.6E [s]'        % self.tchar     )
            if verbose: print(72*'-')
            #if verbose: print(72*'-'+'\n')
            
            ## Pack & attach a udef dict for convenience
            udef_keys = [
                'Ma','Re','Pr','kappa','R','p_inf','T_inf',
                'S_Suth','mu_Suth_ref','T_Suth_ref',
                'C_Suth','mu_inf','rho_inf','nu_inf',
                'a_inf','U_inf','cp','cv','recov_fac','Taw',
                'lchar','tchar','uchar','M_inf',
                'p_tot_inf','T_tot_inf','rho_tot_inf',
                ]
            self.udef = {}
            for key in udef_keys:
                if hasattr(self,key):
                    self.udef[key] = getattr(self,key)
        
        # === Coordinate vectors
        # - (full) grid will only be read (to every rank!!!) if read_grid=True
        # - reading the grid is often not necessary. when it is necessary, just read it directly from the HDF5/h5py handle in
        #    the corresponding function
        
        if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
            
            if read_grid:
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                ## read 3D/1D coordinate arrays
                ## ( dont transpose right away --> allows for 1D storage )
                
                if self.usingmpi:
                    
                    dset = self['dims/x']
                    with dset.collective:
                        x = self.x = np.copy( dset[()] )
                    
                    self.comm.Barrier()
                    
                    dset = self['dims/y']
                    with dset.collective:
                        y = self.y = np.copy( dset[()] )
                    
                    self.comm.Barrier()
                    
                    dset = self['dims/z']
                    with dset.collective:
                        z = self.z = np.copy( dset[()] )
                
                else:
                    
                    x = self.x = np.copy( self['dims/x'][()] )
                    y = self.y = np.copy( self['dims/y'][()] )
                    z = self.z = np.copy( self['dims/z'][()] )
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                data_gb = ( x.nbytes + y.nbytes + z.nbytes ) * self.n_ranks / 1024**3
                if verbose:
                    even_print('read x,y,z (full)', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                if True: ## transpose the coordinate arrays
                    
                    '''
                    nx,ny,nz should probably just be stored as attributes
                    '''
                    
                    if (x.ndim==1):
                        nx = self.nx = x.shape[0]
                    elif (x.ndim==3):
                        x  = self.x  = np.copy( x.T )
                        nx = self.nx = x.shape[0]
                    else:
                        raise AssertionError('x.ndim=%i'%(x.ndim,))
                    
                    if (y.ndim==1):
                        ny = self.ny = y.shape[0]
                    elif (y.ndim==3):
                        y  = self.y  = np.copy( y.T )
                        ny = self.ny = y.shape[1]
                    else:
                        raise AssertionError('y.ndim=%i'%(y.ndim,))
                    
                    if (z.ndim==1):
                        nz = self.nz = z.shape[0]
                    elif (z.ndim==3):
                        z  = self.z  = np.copy( z.T )
                        nz = self.nz = z.shape[2]
                    else:
                        raise AssertionError('z.ndim=%i'%(z.ndim,))
            
            else:
                
                x_shp = self['dims/x'].shape
                y_shp = self['dims/y'].shape
                z_shp = self['dims/z'].shape
                
                ## assert 3D
                if (self['dims/x'].ndim != 3):
                    raise ValueError
                
                ## assert shapes agree 
                if (x_shp != y_shp):
                    raise AssertionError
                if (y_shp != z_shp):
                    raise AssertionError
                
                ## datasets are stored transposed!
                nz,ny,nx = self['dims/x'].shape
                self.nx = nx
                self.ny = ny
                self.nz = nz
                
                self.x = None
                self.y = None
                self.z = None
            
            ngp = self.ngp = nx*ny*nz
            
            if verbose: even_print('nx',  '%i'%nx  )
            if verbose: even_print('ny',  '%i'%ny  )
            if verbose: even_print('nz',  '%i'%nz  )
            if verbose: even_print('ngp', '%i'%ngp )
            #if verbose: print(72*'-')
            
            if False:
                if verbose: even_print('x_min', '%0.2f'%x.min())
                if verbose: even_print('x_max', '%0.2f'%x.max())
                if verbose: even_print('dx begin : end', '%0.3E : %0.3E'%( (x[1]-x[0]), (x[-1]-x[-2]) ))
                if verbose: even_print('y_min', '%0.2f'%y.min())
                if verbose: even_print('y_max', '%0.2f'%y.max())
                if verbose: even_print('dy begin : end', '%0.3E : %0.3E'%( (y[1]-y[0]), (y[-1]-y[-2]) ))
                if verbose: even_print('z_min', '%0.2f'%z.min())
                if verbose: even_print('z_max', '%0.2f'%z.max())        
                if verbose: even_print('dz begin : end', '%0.3E : %0.3E'%( (z[1]-z[0]), (z[-1]-z[-2]) ))
                if verbose: print(72*'-'+'\n')
        
        # === 1D grid filters
        
        self.hasGridFilter=False
        if ('dims/xfi' in self):
            self.xfi  = np.copy(self['dims/xfi'][()])
            if not np.array_equal(self.xfi, np.arange(nx,dtype=np.int64)):
                self.hasGridFilter=True
            if ('dims/xfiR' in self):
                self.xfiR = np.copy(self['dims/xfiR'][()])
        if ('dims/yfi' in self):
            self.yfi  = np.copy(self['dims/yfi'][()])
            if not np.array_equal(self.yfi, np.arange(ny,dtype=np.int64)):
                self.hasGridFilter=True
            if ('dims/yfiR' in self):
                self.yfiR = np.copy(self['dims/yfiR'][()])
        if ('dims/zfi' in self):
            self.zfi  = np.copy(self['dims/zfi'][()])
            if not np.array_equal(self.zfi, np.arange(nz,dtype=np.int64)):
                self.hasGridFilter=True
            if ('dims/zfiR' in self):
                self.zfiR = np.copy(self['dims/zfiR'][()])
        
        # === Time vector
        
        if ('dims/t' in self):
            self.t = np.copy(self['dims/t'][()])
            
            if ('data' in self): ## check t dim and data arr agree
                nt,_,_,_ = self['data/%s'%list(self['data'].keys())[0]].shape ## 4D
                if (nt!=self.t.size):
                    raise AssertionError('nt!=self.t.size : %i!=%i'%(nt,self.t.size))
            
            nt = self.t.size
            
            try:
                self.dt = self.t[1] - self.t[0]
            except IndexError:
                self.dt = 0.
            
            self.nt       = self.t.size
            self.duration = self.t[-1] - self.t[0]
            self.ti       = np.array(range(self.nt), dtype=np.int64)
        
        elif all([('data' in self),('dims/t' not in self)]): ## data but no time
            self.scalars = list(self['data'].keys())
            nt,_,_,_ = self['data/%s'%self.scalars[0]].shape
            self.nt  = nt
            self.t   = np.arange(self.nt, dtype=np.float64)
            self.ti  = np.arange(self.nt, dtype=np.int64)
            self.dt  = 1.
            self.duration = self.t[-1]-self.t[0]
        
        else: ## no data, no time
            self.t  = np.array([], dtype=np.float64)
            self.ti = np.array([], dtype=np.int64)
            self.nt = nt = 0
            self.dt = 0.
            self.duration = 0.
        
        #if verbose: print(72*'-')
        if verbose: even_print('nt', '%i'%self.nt )
        if verbose: even_print('dt', '%0.6f'%self.dt)
        if verbose: even_print('duration', '%0.2f'%self.duration )
        if hasattr(self, 'duration_avg'):
            if verbose: even_print('duration_avg', '%0.2f'%self.duration_avg )
        #if verbose: print(72*'-'+'\n')
        
        if hasattr(self,'rectilinear'):
            if verbose: even_print('rectilinear', str(self.rectilinear) )
        if hasattr(self,'curvilinear'):
            if verbose: even_print('curvilinear', str(self.curvilinear) )
        
        # === ts group names & scalars
        
        if ('data' in self):
            self.scalars = list(self['data'].keys()) ## 4D : string names of scalars : ['u','v','w'],...
            self.n_scalars = len(self.scalars)
            self.scalars_dtypes = []
            for scalar in self.scalars:
                self.scalars_dtypes.append(self[f'data/{scalar}'].dtype)
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes)) ## dict {<<scalar>>: <<dtype>>}
        else:
            self.scalars = []
            self.n_scalars = 0
            self.scalars_dtypes = []
            self.scalars_dtypes_dict = dict(zip(self.scalars, self.scalars_dtypes))
        
        return
    
    def init_from_eas4(self, fn_eas4, **kwargs):
        '''
        Initialize a CGD from an EAS4 (NS3D output format)
        '''
        
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        chunk_kb = kwargs.get('chunk_kb',4*1024) ## 4 [MB]
        verbose  = kwargs.get('verbose',True)
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        
        if (rx*ry*rz != self.n_ranks):
            raise AssertionError('rx*ry*rz != self.n_ranks')
        
        if (self.rank!=0):
            verbose=False
        
        ## spatial resolution filter : take every nth grid point
        #sx = kwargs.get('sx',1)
        #sy = kwargs.get('sy',1)
        #sz = kwargs.get('sz',1)
        
        ## spatial resolution filter : set x/y/z bounds
        
        #x_min = kwargs.get('x_min',None)
        #y_min = kwargs.get('y_min',None)
        #z_min = kwargs.get('z_min',None)
        
        #x_max = kwargs.get('x_max',None)
        #y_max = kwargs.get('y_max',None)
        #z_max = kwargs.get('z_max',None)
        
        #xi_min = kwargs.get('xi_min',None)
        #yi_min = kwargs.get('yi_min',None)
        #zi_min = kwargs.get('zi_min',None)
        
        #xi_max = kwargs.get('xi_max',None)
        #yi_max = kwargs.get('yi_max',None)
        #zi_max = kwargs.get('zi_max',None)
        
        ## Set default attributes
        self.attrs['fsubtype'] = 'unsteady'
        self.attrs['fclass']   = 'cgd'
        
        if verbose: print('\n'+'cgd.init_from_eas4()'+'\n'+72*'-')
        
        if not (os.path.isfile(fn_eas4) or (os.path.islink(fn_eas4) and os.path.isfile(os.path.realpath(fn_eas4)))):
            raise FileNotFoundError(f'{fn_eas4} is not a file or a symlink to an existing file')
        
        with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=MPI.COMM_WORLD, read_3d_grid=False) as hf_eas4:
            
            if verbose: even_print('infile', os.path.basename(fn_eas4))
            if verbose: even_print('infile size', '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3))
            
            if verbose: even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1, gmode_dict[hf_eas4.gmode_dim1] ) )
            if verbose: even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2, gmode_dict[hf_eas4.gmode_dim2] ) )
            if verbose: even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3, gmode_dict[hf_eas4.gmode_dim3] ) )
            
            if verbose: even_print( 'nx' , f'{hf_eas4.nx:d}' )
            if verbose: even_print( 'ny' , f'{hf_eas4.ny:d}' )
            if verbose: even_print( 'nz' , f'{hf_eas4.nz:d}' )
            if verbose: print(72*'-')
            if verbose: even_print('outfile', self.fname)
            
            if verbose: even_print( 'rectilinear' , str(hf_eas4.is_rectilinear) )
            if verbose: even_print( 'curvilinear' , str(hf_eas4.is_curvilinear) )
            self.attrs['rectilinear'] = hf_eas4.is_rectilinear
            self.attrs['curvilinear'] = hf_eas4.is_curvilinear
            
            # === Copy over freestream parameters
            
            header_attr_keys = [
                'Ma','Re','Pr',
                'kappa','R',
                'p_inf','T_inf',
                'S_Suth','mu_Suth_ref','T_Suth_ref',
                ]
            
            ## assert that top-level attributes don't already exist
            #if any([ key in self.attrs for key in header_attr_keys ]):
            #    raise ValueError('some udef keys are already present in target file.')
            
            ## udef dict from EAS4
            udef = hf_eas4.udef
            
            ## Strip dict into 2x arrays (keys,values) and save to HDF5
            udef_real    = list(udef.values())
            udef_char    = list(udef.keys())
            udef_real_h5 = np.array(udef_real, dtype=np.float64)
            udef_char_h5 = np.array([s.encode('ascii', 'ignore') for s in udef_char], dtype='S128')
            
            if ('header/udef_real' in self):
                del self['header/udef_real']
            if ('header/udef_char' in self):
                del self['header/udef_char']
            
            self.create_dataset('header/udef_real', data=udef_real_h5, dtype=np.float64)
            self.create_dataset('header/udef_char', data=udef_char_h5, dtype='S128')
            
            ## Assert that all primary udef keys are available in EAS4
            ##  --> this could be fed into 'freestream_parameters()' instead
            for key in header_attr_keys:
                if key not in udef.keys():
                    raise ValueError(f"key '{key}' not found in udef of {fn_eas4}")
            
            ## Write (primary) udef members as top-level attributes of HDF5 file
            for key in header_attr_keys:
                self.attrs[key] = udef[key]
            
            ## Standard freestream parameters
            Ma          = udef['Ma']
            Re          = udef['Re']
            Pr          = udef['Pr']
            kappa       = udef['kappa']
            R           = udef['R']
            p_inf       = udef['p_inf']
            T_inf       = udef['T_inf']
            S_Suth      = udef['S_Suth']
            mu_Suth_ref = udef['mu_Suth_ref']
            T_Suth_ref  = udef['T_Suth_ref']
            
            ## Compute derived freestream parameters
            C_Suth    = mu_Suth_ref/(T_Suth_ref**(3/2))*(T_Suth_ref + S_Suth) ## [kg/(m·s·√K)]
            mu_inf    = mu_Suth_ref*(T_inf/T_Suth_ref)**(3/2) * ((T_Suth_ref+S_Suth)/(T_inf+S_Suth))
            rho_inf   = p_inf/(R*T_inf)
            nu_inf    = mu_inf/rho_inf
            a_inf     = np.sqrt(kappa*R*T_inf)
            U_inf     = Ma*a_inf
            cp        = R*kappa/(kappa-1.)
            cv        = cp/kappa
            recov_fac = Pr**(1/3)
            Taw       = T_inf + recov_fac*U_inf**2/(2*cp)
            lchar     = Re*nu_inf/U_inf
            tchar     = lchar / U_inf
            
            ## Aliases
            uchar = U_inf
            M_inf = Ma
            
            p_tot_inf   = p_inf   * (1 + (kappa-1)/2 * M_inf**2)**(kappa/(kappa-1))
            T_tot_inf   = T_inf   * (1 + (kappa-1)/2 * M_inf**2)
            rho_tot_inf = rho_inf * (1 + (kappa-1)/2 * M_inf**2)**(1/(kappa-1))
            
            ## Write (derived) freestream parameters as top-level attributes of HDF5 file
            self.attrs['C_Suth']      = C_Suth
            self.attrs['mu_inf']      = mu_inf
            self.attrs['rho_inf']     = rho_inf
            self.attrs['nu_inf']      = nu_inf
            self.attrs['a_inf']       = a_inf
            self.attrs['U_inf']       = U_inf
            self.attrs['cp']          = cp
            self.attrs['cv']          = cv
            self.attrs['recov_fac']   = recov_fac
            self.attrs['Taw']         = Taw
            self.attrs['lchar']       = lchar
            self.attrs['tchar']       = tchar
            self.attrs['uchar']       = uchar
            self.attrs['M_inf']       = M_inf
            self.attrs['p_tot_inf']   = p_tot_inf
            self.attrs['T_tot_inf']   = T_tot_inf
            self.attrs['rho_tot_inf'] = rho_tot_inf
            
            # === Copy over dims info
            
            if all([('dims/x' in self),('dims/y' in self),('dims/z' in self)]):
                pass
                ## future: 2D/3D handling here
            else:
                
                self.nx = nx = hf_eas4.nx
                self.ny = ny = hf_eas4.ny
                self.nz = nz = hf_eas4.nz
                
                # === Rank 3D grid ranges
                
                if self.usingmpi:
                    
                    comm4d = self.comm.Create_cart(dims=[rx,ry,rz], periods=[False,False,False], reorder=False)
                    t4d = comm4d.Get_coords(self.rank)
                    
                    rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
                    ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
                    rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
                    #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
                    
                    rxl = [[b[0],b[-1]+1] for b in rxl_ ]
                    ryl = [[b[0],b[-1]+1] for b in ryl_ ]
                    rzl = [[b[0],b[-1]+1] for b in rzl_ ]
                    #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
                    
                    rx1, rx2 = rxl[t4d[0]]; nxr = rx2 - rx1
                    ry1, ry2 = ryl[t4d[1]]; nyr = ry2 - ry1
                    rz1, rz2 = rzl[t4d[2]]; nzr = rz2 - rz1
                    #rt1, rt2 = rtl[t4d[3]]; ntr = rt2 - rt1
                
                else:
                    
                    rx1 = 0
                    rx2 = self.nx
                    ry1 = 0
                    ry2 = self.ny
                    rz1 = 0
                    rz2 = self.nz
                    
                    nxr = self.nx
                    nyr = self.ny
                    nzr = self.nz
                
                # === Broadcast mesh to full 3D
                
                x = np.full((nxr,nyr,nzr),np.nan,dtype=np.float64)
                y = np.full((nxr,nyr,nzr),np.nan,dtype=np.float64)
                z = np.full((nxr,nyr,nzr),np.nan,dtype=np.float64)
                
                na = np.newaxis
                
                ## [x]
                ds = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim01']
                if ds.ndim==1:
                    x[:,:,:] = ds[rx1:rx2][:,na,na]
                elif ds.ndim==3:
                    x[:,:,:] = ds[rx1:rx2,ry1:ry2,rz1:rz2]
                else:
                    raise NotImplementedError
                
                ## [y]
                ds = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim02']
                if ds.ndim==1:
                    y[:,:,:] = ds[ry1:ry2][na,:,na]
                elif ds.ndim==3:
                    y[:,:,:] = ds[rx1:rx2,ry1:ry2,rz1:rz2]
                else:
                    raise NotImplementedError
                
                ## [z]
                ds = hf_eas4[f'Kennsatz/GEOMETRY/{hf_eas4.domainName}/dim03']
                if ds.ndim==1:
                    z[:,:,:] = ds[rz1:rz2][na,na,:]
                elif ds.ndim==3:
                    z[:,:,:] = ds[rx1:rx2,ry1:ry2,rz1:rz2]
                else:
                    raise NotImplementedError
                
                if np.isnan(x).any():
                    raise ValueError('x has nans')
                if np.isnan(y).any():
                    raise ValueError('y has nans')
                if np.isnan(z).any():
                    raise ValueError('z has nans')
                
                shape  = (nz,ny,nx)
                chunks = h5_chunk_sizer(nxi=shape, constraint=(None,None,None), size_kb=chunk_kb, base=4, itemsize=8)
                
                # === Write coord arrays
                
                if ('dims/x' in self):
                    del self['dims/x']
                if ('dims/y' in self):
                    del self['dims/y']
                if ('dims/z' in self):
                    del self['dims/z']
                
                ## Initialize datasets
                dset_x = self.create_dataset('dims/x', shape=shape, chunks=chunks, dtype=x.dtype)
                dset_y = self.create_dataset('dims/y', shape=shape, chunks=chunks, dtype=y.dtype)
                dset_z = self.create_dataset('dims/z', shape=shape, chunks=chunks, dtype=z.dtype)
                
                chunk_kb_ = np.prod(dset_x.chunks)*8 / 1024. ## actual
                if verbose:
                    even_print('chunk shape (z,y,x)','%s'%str(dset_x.chunks))
                    even_print('chunk size','%i [KB]'%int(round(chunk_kb_)))
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if self.usingmpi: 
                    with dset_x.collective:
                        #dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_x[rz1:rz2,ry1:ry2,rx1:rx2] = x.T
                else:
                    dset_x[:,:,:] = x.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_y.collective:
                        #dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_y[rz1:rz2,ry1:ry2,rx1:rx2] = y.T
                else:
                    dset_y[:,:,:] = y.T
                
                if self.usingmpi: self.comm.Barrier()
                
                if self.usingmpi: 
                    with dset_z.collective:
                        #dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z[rx1:rx2,ry1:ry2,rz1:rz2].T
                        dset_z[rz1:rz2,ry1:ry2,rx1:rx2] = z.T
                else:
                    dset_z[:,:,:] = z.T
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                
                #data_gb = ( x.nbytes + y.nbytes + z.nbytes ) / 1024**3
                data_gb = 3 * 8 * nx * ny * nz / 1024**3
                
                if verbose:
                    even_print('Write x,y,z', '%0.3f [GB]  %0.3f [s]  %0.3f [GB/s]'%(data_gb,t_delta,(data_gb/t_delta)))
                
                self.nx  = nx
                self.ny  = ny
                self.nz  = nz
                self.ngp = nx*ny*nz
        
        if verbose: print(72*'-')
        self.get_header(verbose=True, read_grid=False)
        if verbose: print(72*'-')
        
        return
    
    def import_eas4(self, fn_eas4_list, **kwargs):
        '''
        Import data from a series of EAS4 files to a CGD
        '''
        
        if (self.rank!=0):
            verbose=False
        else:
            verbose=True
        
        gmode_dict = {1:'EAS4_NO_G', 2:'EAS4_X0DX_G', 3:'EAS4_UDEF_G', 4:'EAS4_ALL_G', 5:'EAS4_FULL_G'}
        
        if verbose: print('\n'+'cgd.import_eas4()'+'\n'+72*'-')
        t_start_func = timeit.default_timer()
        
        ntbuf = kwargs.get('ntbuf',1) ## [t] R/W buffer size
        if not isinstance(ntbuf, int):
            raise ValueError('ntbuf must be type int')
        if (ntbuf<1):
            raise ValueError('ntbuf<1')
        
        if self.open_mode=='r':
            raise ValueError('cant do import or file initialization if file has been opened in read-only mode.')
        
        report_reads  = kwargs.get('report_reads',False)
        report_writes = kwargs.get('report_writes',True)
        
        ti_min = kwargs.get('ti_min',None)
        ti_max = kwargs.get('ti_max',None)
        tt_min = kwargs.get('tt_min',None)
        tt_max = kwargs.get('tt_max',None)
        
        ## dont actually copy over data, just initialize datasets with 0's
        init_dsets_only = kwargs.get('init_dsets_only',False)
        
        ## delete EAS4s after import --> DANGER!
        delete_after_import = kwargs.get('delete_after_import',False)
        
        ## if you're just initializing, don't allow delete
        if (init_dsets_only and delete_after_import):
            raise ValueError("if init_dsets_only=True, then delete_after_import should not be activated!")
        
        ## delete only allowed if no time ranges are selected
        if delete_after_import and any([(ti_min is not None),(ti_max is not None),(tt_min is not None),(tt_max is not None)]):
            raise ValueError("if delete_after_import=True, then ti_min,ti_max,tt_min,tt_max are not supported")
        
        chunk_kb    = kwargs.get('chunk_kb',2*1024) ## h5 chunk size: default 2 [MB]
        #chunk_kb   = kwargs.get('chunk_kb',64*1024) ## h5 chunk size: default 64 [MB]
        #chunk_base = kwargs.get('chunk_base',2)
        
        ## used later when determining whether to re-initialize datasets
        #chunk_constraint = kwargs.get('chunk_constraint',(1,None,None,None))
        chunk_constraint = kwargs.get('chunk_constraint',None)
        if chunk_constraint is None:
            chunk_constraint_was_provided = False
        else:
            chunk_constraint_was_provided = True
        
        ## float precision when copying
        ## default is 'single' i.e. cast data to single
        ## 'same' will preserve the floating point precision from the EAS4 file
        prec = kwargs.get('prec',None)
        if (prec is None):
            prec = 'single'
        elif (prec=='single'):
            pass
        elif (prec=='same'):
            pass
        else:
            raise ValueError('prec not set correctly')
        
        ## check for an often made mistake
        ## 'ts_min' / 'ts_max' should NOT be allowed as inputs
        ts_min = kwargs.get('ts_min',None)
        ts_max = kwargs.get('ts_max',None)
        if (ts_min is not None):
            raise ValueError('ts_min is not an option --> did you mean ti_min or tt_min?')
        if (ts_max is not None):
            raise ValueError('ts_max is not an option --> did you mean ti_max or tt_max?')
        del ts_min
        del ts_max
        
        ## check that the passed iterable of EAS4 files is OK
        if not hasattr(fn_eas4_list, '__iter__'):
            raise ValueError('first arg \'fn_eas4_list\' must be iterable')
        for fn_eas4 in fn_eas4_list:
            if not os.path.isfile(fn_eas4):
                raise FileNotFoundError('%s not found!'%fn_eas4)
        
        ## ranks per direction
        rx = kwargs.get('rx',1)
        ry = kwargs.get('ry',1)
        rz = kwargs.get('rz',1)
        rt = kwargs.get('rt',1)
        
        ## check validity of rank declaration
        if not all(isinstance(rr,int) and rr>0 for rr in (rx,ry,rz,rt)):
            raise ValueError('rx,ry,rz,rt must be positive integers')
        if (rx*ry*rz*rt != self.n_ranks):
            raise ValueError('rx*ry*rz*rt != self.n_ranks')
        if (rx>self.nx):
            raise ValueError('rx>self.nx')
        if (ry>self.ny):
            raise ValueError('ry>self.ny')
        if (rz>self.nz):
            raise ValueError('rz>self.nz')
        if not self.usingmpi:
            if rx>1:
                if verbose: print(f'WARNING: file not opened in MPI mode but rx={rx:d}... setting rx=1')
                rx = 1
            if ry>1:
                if verbose: print(f'WARNING: file not opened in MPI mode but ry={ry:d}... setting ry=1')
                ry = 1
            if rz>1:
                if verbose: print(f'WARNING: file not opened in MPI mode but rz={rz:d}... setting rz=1')
                rz = 1
        
        ## st = timestep skip
        ## spatial [x,y,z] skips done in init_from_XXX()
        st = kwargs.get('st',1)
        
        if not isinstance(st, int):
            raise ValueError('time skip parameter st should be type int')
        if (st<1):
            raise ValueError('st<1')
        
        ## update this CGD's header and attributes
        self.get_header(verbose=False, read_grid=False)
        
        if self.usingmpi:
            comm_eas4 = MPI.COMM_WORLD ## communicator for opening EAS4s
        else:
            comm_eas4 = None
        
        ## get all time info & check
        if (self.rank==0):
            t = np.array([], dtype=np.float64)
            for fn_eas4 in fn_eas4_list:
                with eas4(fn_eas4, 'r', verbose=False, read_3d_grid=False) as hf_eas4:
                    t_ = np.copy(hf_eas4.t)
                t = np.concatenate((t,t_))
        else:
            t = np.array([],dtype=np.float64) ## 't' must exist on all ranks prior to bcast
        
        if self.usingmpi:
            self.comm.Barrier()
        
        ## broadcast concatenated time vector to all ranks
        if self.usingmpi:
            t = self.comm.bcast(t, root=0)
        
        if verbose: even_print('n EAS4 files','%i'%len(fn_eas4_list))
        if verbose: even_print('nt all files','%i'%t.size)
        if verbose: even_print('delete after import',str(delete_after_import))
        if verbose: print(72*'-')
        
        if (t.size>1):
            
            ## check no zero distance elements
            if np.any(np.diff(t) == 0):
                raise ValueError('t arr has zero-distance elements')
            else:
                if verbose: even_print('check: Δt!=0','passed')
            
            ## check monotonically increasing
            if not np.all(np.diff(t) > 0.):
                raise ValueError('t arr not monotonically increasing')
            else:
                if verbose: even_print('check: t mono increasing','passed')
            
            ## check constant Δt
            dt0 = np.diff(t)[0]
            if not np.all(np.isclose(np.diff(t), dt0, rtol=1e-3)):
                if (self.rank==0): print(np.diff(t))
                raise ValueError('t arr not uniformly spaced')
            else:
                if verbose: even_print('check: constant Δt','passed')
        
        # === get all grid info & check
        
        # TODO : compare coordinate arrays for series of EAS4 files
        
        ## [t] resolution filter (skip every N timesteps)
        tfi = np.arange(t.size, dtype=np.int64)
        if (st!=1):
            if verbose:
                even_print('st', f'{st:d}')
                print(72*'-')
            tfi = np.copy( tfi[::st] )
        
        ## initialize 'doRead' vector --> boolean vector to be updated
        doRead = np.full((t.size,), True, dtype=bool)
        
        ## skip filter
        if (st!=1):
            doRead[np.isin(np.arange(t.size),tfi,invert=True)] = False
        
        ## min/max index filter
        if (ti_min is not None):
            if not isinstance(ti_min, int):
                raise TypeError('ti_min must be type int')
            doRead[:ti_min] = False
        if (ti_max is not None):
            if not isinstance(ti_max, int):
                raise TypeError('ti_max must be type int')
            doRead[ti_max:] = False
        if (tt_min is not None):
            if (tt_min>=0.):
                doRead[np.where((t-t.min())<tt_min)] = False
            elif (tt_min<0.):
                doRead[np.where((t-t.max())<tt_min)] = False
        if (tt_max is not None):
            if (tt_max>=0.):
                doRead[np.where((t-t.min())>tt_max)] = False
            elif (tt_max<0.):
                doRead[np.where((t-t.max())>tt_max)] = False
        
        ## CGD time attributes
        self.t  = np.copy(t[doRead]) ## filter times by True/False from boolean vector doRead
        self.nt = self.t.shape[0]
        self.ti = np.arange(self.nt, dtype=np.int64)
        
        # ## update [t]
        # if ('dims/t' in self):
        #     t_ = np.copy(self['dims/t'][()])
        #     if not np.allclose(t_, self.t, rtol=1e-8, atol=1e-8):
        #         if verbose:
        #             print('>>> [t] in file not match [t] that has been determined ... overwriting')
        #         del self['dims/t']
        #         self.create_dataset('dims/t', data=self.t)
        # else:
        #     self.create_dataset('dims/t', data=self.t)
        
        ## update [t]
        if ('dims/t' in self):
            del self['dims/t']
        self.create_dataset('dims/t', data=self.t)
        
        ## divide spatial OUTPUT grid by ranks
        ## if no grid filter present, then INPUT = OUTPUT
        if self.usingmpi:
            comm4d = self.comm.Create_cart(dims=[rx,ry,rz,rt], periods=[False,False,False,False], reorder=False)
            t4d = comm4d.Get_coords(self.rank)
            
            rxl_ = np.array_split(np.arange(self.nx,dtype=np.int64),min(rx,self.nx))
            ryl_ = np.array_split(np.arange(self.ny,dtype=np.int64),min(ry,self.ny))
            rzl_ = np.array_split(np.arange(self.nz,dtype=np.int64),min(rz,self.nz))
            #rtl_ = np.array_split(np.arange(self.nt,dtype=np.int64),min(rt,self.nt))
            
            rxl = [[b[0],b[-1]+1] for b in rxl_ ]
            ryl = [[b[0],b[-1]+1] for b in ryl_ ]
            rzl = [[b[0],b[-1]+1] for b in rzl_ ]
            #rtl = [[b[0],b[-1]+1] for b in rtl_ ]
            
            rx1, rx2 = rxl[t4d[0]]
            ry1, ry2 = ryl[t4d[1]]
            rz1, rz2 = rzl[t4d[2]]
            #rt1, rt2 = rtl[t4d[3]]
            
            nxr = rx2 - rx1
            nyr = ry2 - ry1
            nzr = rz2 - rz1
            #ntr = rt2 - rt1
        
        else:
            nxr = self.nx
            nyr = self.ny
            nzr = self.nz
            #ntr = self.nt
        
        ## divide spatial READ/INPUT grid by ranks --> if grid filters present
        if self.hasGridFilter:
            if self.usingmpi:
                
                ## this rank's indices to read from FULL file, indices in global context
                xfi_ = np.copy( self.xfi[rx1:rx2] )
                yfi_ = np.copy( self.yfi[ry1:ry2] )
                zfi_ = np.copy( self.zfi[rz1:rz2] )
                
                ## this rank's global read RANGE from FULL file
                rx1R,rx2R = xfi_.min() , xfi_.max()+1
                ry1R,ry2R = yfi_.min() , yfi_.max()+1
                rz1R,rz2R = zfi_.min() , zfi_.max()+1
                
                ## this rank's LOCAL index filter to cut down read data
                xfi_local = np.copy( xfi_ - rx1R )
                yfi_local = np.copy( yfi_ - ry1R )
                zfi_local = np.copy( zfi_ - rz1R )
        
        ## Determine CGD scalars (from EAS4 scalars)
        if not hasattr(self,'scalars') or (len(self.scalars)==0):
            
            with eas4(fn_eas4_list[0], 'r', verbose=False, driver=self.driver, comm=comm_eas4) as hf_eas4:
                self.scalars   = hf_eas4.scalars
                self.n_scalars = len(self.scalars)
                
                ## decide dtypes
                for scalar in hf_eas4.scalars:
                    
                    ti = 0
                    dsn = f'Data/{hf_eas4.domainName}/ts_{ti:06d}/par_{hf_eas4.scalar_n_map[scalar]:06d}'
                    dset = hf_eas4[dsn]
                    dtype = dset.dtype
                    
                    if (prec=='same'):
                        self.scalars_dtypes_dict[scalar] = dtype
                    elif (prec=='single'):
                        if (dtype!=np.float32) and (dtype!=np.float64): ## make sure its either a single or double float
                            raise ValueError
                        self.scalars_dtypes_dict[scalar] = np.dtype(np.float32)
                    else:
                        raise ValueError
        
        if self.usingmpi:
            comm_eas4.Barrier()
        
        # ==============================================================
        # Initialize datasets
        # ==============================================================
        
        if verbose:
            progress_bar = tqdm(
                total=len(self.scalars),
                ncols=100,
                desc='initialize dsets',
                leave=True,
                file=sys.stdout,
                mininterval=0.1,
                smoothing=0.,
                #bar_format="\033[B{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}\033[A\n\b",
                bar_format="{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}",
                ascii="░█",
                colour='#FF6600',
                )
        
        for scalar in self.scalars:
            
            dtype   = self.scalars_dtypes_dict[scalar]
            data_gb = dtype.itemsize * self.nt*self.nz*self.ny*self.nx / 1024**3
            shape   = (self.nt,self.nz,self.ny,self.nx)
            
            ## the user provided a chunk_constraint, so calculate it
            if chunk_constraint_was_provided:
                chunks = h5_chunk_sizer(nxi=shape, constraint=chunk_constraint, size_kb=chunk_kb, itemsize=dtype.itemsize)
            else:
                if self.usingmpi:
                    chunks = h5_chunk_sizer(nxi=shape, constraint=(1,('max',self.nz//rz),('max',self.ny//ry),('max',self.nx//rx)), size_kb=chunk_kb, itemsize=dtype.itemsize)
                else:
                    chunks = h5_chunk_sizer(nxi=shape, constraint=(1,None,None,None), size_kb=chunk_kb, itemsize=dtype.itemsize)
            
            do_dset_initialize = True ## default value which, if all conditions are met, will be turned False
            
            dsn = f'data/{scalar}'
            
            ## check if dataset already exists and matches conditions
            ## ... if conditions are met then skip re-initializing dataset
            if self.open_mode in ('a','r+') and (dsn in self):
                dset = self[dsn]
                if verbose:
                    tqdm.write(even_print(f'dset {dsn} already exists', str(True), s=True))
                
                shape_matches = dset.shape == shape
                dtype_matches = dset.dtype == dtype
                
                ## either 1) no constraint was given or 2) constraint was given AND it matches
                chunks_match = not chunk_constraint_was_provided or dset.chunks == chunks
                
                ## if no constraint was given, copy back existing chunk
                if not chunk_constraint_was_provided:
                    chunks = dset.chunks
                
                if verbose:
                    tqdm.write(even_print(f'dset {dsn} shape matches', str(shape_matches), s=True))
                    tqdm.write(even_print(f'dset {dsn} dtype matches', str(dtype_matches), s=True))
                    if chunk_constraint_was_provided:
                        tqdm.write(even_print(f'dset {dsn} chunks match', str(chunks_match), s=True))
                
                if shape_matches and dtype_matches and chunks_match:
                    do_dset_initialize = False
            
            if do_dset_initialize:
                
                if (f'data/{scalar}' in self):
                    del self[f'data/{scalar}']
                
                if self.usingmpi: self.comm.Barrier()
                t_start = timeit.default_timer()
                
                if verbose:
                    tqdm.write(even_print(f'initializing data/{scalar}', f'{data_gb:0.2f} [GB]', s=True))
                
                ## !!!!!!!!!!! this has a tendency to hang !!!!!!!!!!!
                ## --> increasing Lustre stripe size tends to fix this (kwarg 'stripe_size_mb' upon 'w' open)
                dset = self.create_dataset(
                                    dsn,
                                    shape=shape,
                                    dtype=dtype,
                                    chunks=chunks,
                                    )
                
                ## write dummy data to dataset to ensure that it is truly initialized
                if not self.usingmpi:
                    h5_ds_force_allocate_chunks(dset,verbose=verbose)
                
                if self.usingmpi: self.comm.Barrier()
                t_delta = timeit.default_timer() - t_start
                if verbose:
                    tqdm.write(even_print(f'initialize data/{scalar}', f'{data_gb:0.2f} [GB]  {t_delta:0.2f} [s]  {(data_gb/t_delta):0.3f} [GB/s]', s=True))
            
            chunk_kb_ = np.prod(dset.chunks) * dset.dtype.itemsize / 1024. ## actual
            if verbose:
                tqdm.write(even_print('chunk shape (t,z,y,x)', str(dset.chunks), s=True))
                tqdm.write(even_print('chunk size', f'{int(round(chunk_kb_)):d} [KB]', s=True))
            
            if verbose:
                progress_bar.update()
        
        if self.usingmpi:
            self.comm.Barrier()
        if verbose:
            progress_bar.close()
            print(72*'-')
        
        ## Report size of CGD after initialization
        if verbose: tqdm.write(even_print(os.path.basename(self.fname), f'{os.path.getsize(self.fname)/1024**3:0.2f} [GB]', s=True))
        if verbose: print(72*'-')
        
        # ==============================================================
        # Open & read EAS4s, read data into RAM, write to CGD
        # ==============================================================
        
        if not init_dsets_only:
            
            ## should we tell the EAS4 to open with MPIIO hint 'romio_no_indep_rw' ?
            if self.usingmpi:
                eas4_no_indep_rw = True
            else:
                eas4_no_indep_rw = False
            
            ## Get main dtype and confirm all scalar dtypes are same (limitation)
            dtype = self.scalars_dtypes_dict[self.scalars[0]]
            for scalar in self.scalars:
                if not ( np.dtype(self.scalars_dtypes_dict[scalar]) == np.dtype(dtype) ):
                    raise NotImplementedError('dtype of scalars in output HDF5 file are not same. update!')
            
            ## Current limitation of read buffer due to uncreative implementation
            if (self.nt%ntbuf!=0):
                raise ValueError(f'n timesteps to be read ({self.nt}) is not divisible by ntbuf ({ntbuf:d})')
            
            ## Initialize read/write buffer
            databuf = np.zeros(shape=(ntbuf,nzr,nyr,nxr), dtype={'names':self.scalars, 'formats':[ dtype for s in self.scalars ]})
            buffer_nts_loaded = 0 ## counter for number of timesteps loaded in buffer
            buffers_written = -1 ## counter for number of buffers that have been written
            
            #print(f'rank {self.rank:d} databuf shape : {str(databuf["u"].shape)}')
            
            if self.usingmpi:
                self.comm.Barrier()
            
            ## Report read/write buffer size
            if verbose:
                even_print( 'R/W buffer size (global)' , f'{ntbuf*np.prod(shape[1:])*len(self.scalars)*dtype.itemsize/1024**3:0.2f} [GB]' )
                print(72*'-')
            
            if verbose:
                progress_bar = tqdm(
                    #total=self.nt*self.n_scalars,
                    total=self.nt//ntbuf, ## N buffer writes
                    ncols=100,
                    desc='import',
                    leave=True,
                    file=sys.stdout,
                    mininterval=0.1,
                    smoothing=0.,
                    #bar_format="\033[B{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}\033[A\n\b",
                    bar_format="{l_bar}{bar}| {n}/{total} [{percentage:.1f}%] {elapsed}/{remaining}",
                    ascii="░█",
                    colour='#FF6600',
                    )
            
            ## counters / timers
            data_gb_read  = 0.
            data_gb_write = 0.
            t_read  = 0.
            t_write = 0.
            
            tii  = -1 ## counter full series
            tiii = -1 ## counter CGD-local
            for fn_eas4 in fn_eas4_list: ## this has to stay the outer-most loop for file deletion purposes
                with eas4(fn_eas4, 'r', verbose=False, driver=self.driver, comm=comm_eas4, no_indep_rw=eas4_no_indep_rw) as hf_eas4:
                    
                    if verbose: tqdm.write(even_print(os.path.basename(fn_eas4), '%0.2f [GB]'%(os.path.getsize(fn_eas4)/1024**3), s=True))
                    
                    # if verbose: tqdm.write(even_print('gmode_dim1' , '%i'%hf_eas4.gmode_dim1  , s=True))
                    # if verbose: tqdm.write(even_print('gmode_dim2' , '%i'%hf_eas4.gmode_dim2  , s=True))
                    # if verbose: tqdm.write(even_print('gmode_dim3' , '%i'%hf_eas4.gmode_dim3  , s=True))
                    
                    if verbose: tqdm.write(even_print( 'gmode dim1' , '%i / %s'%( hf_eas4.gmode_dim1, gmode_dict[hf_eas4.gmode_dim1] ), s=True ))
                    if verbose: tqdm.write(even_print( 'gmode dim2' , '%i / %s'%( hf_eas4.gmode_dim2, gmode_dict[hf_eas4.gmode_dim2] ), s=True ))
                    if verbose: tqdm.write(even_print( 'gmode dim3' , '%i / %s'%( hf_eas4.gmode_dim3, gmode_dict[hf_eas4.gmode_dim3] ), s=True ))
                    
                    if verbose: tqdm.write(even_print('duration' , '%0.2f'%hf_eas4.duration , s=True))
                    
                    # ===
                    
                    for ti in range(hf_eas4.nt): ## this EAS4's time indices
                        tii += 1 ## full EAS4 series counter
                        if doRead[tii]:
                            tiii += 1 ## output CGD counter (takes into account skip, min/max)
                            
                            buffer_nts_loaded += 1
                            
                            #if verbose: tqdm.write(f'writing to buffer at index {tiii%ntbuf:d}') ## debug
                            
                            ## Perform collective read, write to RAM buffer
                            for scalar in hf_eas4.scalars:
                                if (scalar in self.scalars):
                                    
                                    ## dset handle in EAS4
                                    dset = hf_eas4[f'Data/{hf_eas4.domainName}/ts_{ti:06d}/par_{hf_eas4.scalar_n_map[scalar]:06d}']
                                    
                                    if hf_eas4.dform==1:
                                        ds_nx,ds_ny,ds_nz = dset.shape
                                    elif hf_eas4.dform==2:
                                        ds_nz,ds_ny,ds_nx = dset.shape
                                    else:
                                        raise RuntimeError
                                    
                                    ## EAS4 has a 'collapsed' dimension but >1 ranks in that dim
                                    if ( ds_nx < rx ):
                                        raise ValueError(f'dset shape in [x] is <rx : dset.shape={str(dset.shape)} , rx={rx:d}')
                                    if ( ds_ny < ry ):
                                        raise ValueError(f'dset shape in [y] is <ry : dset.shape={str(dset.shape)} , ry={ry:d}')
                                    if ( ds_nz < rz ):
                                        raise ValueError(f'dset shape in [z] is <rz : dset.shape={str(dset.shape)} , rz={rz:d}')
                                    
                                    if hf_eas4.usingmpi: comm_eas4.Barrier()
                                    t_start = timeit.default_timer()
                                    
                                    if hf_eas4.usingmpi:
                                        if self.hasGridFilter:
                                            with dset.collective:
                                                if hf_eas4.dform==1:
                                                    d_ = dset[rx1R:rx2R,ry1R:ry2R,rz1R:rz2R]
                                                elif hf_eas4.dform==2:
                                                    d_ = dset[rz1R:rz2R,ry1R:ry2R,rx1R:rx2R]
                                                else:
                                                    raise RuntimeError
                                            
                                            if ( ds_nx == 1 ):
                                                xfi_local = [0,]
                                            if ( ds_ny == 1 ):
                                                yfi_local = [0,]
                                            if ( ds_nz == 1 ):
                                                zfi_local = [0,]
                                            
                                            databuf[scalar][tiii%ntbuf,:,:,:] = d_[ np.ix_(xfi_local,yfi_local,zfi_local) ].T
                                        
                                        else:
                                            with dset.collective:
                                                
                                                if hf_eas4.dform==1:
                                                    databuf[scalar][tiii%ntbuf,:,:,:] = dset[rx1:rx2,ry1:ry2,rz1:rz2].T
                                                elif hf_eas4.dform==2:
                                                    databuf[scalar][tiii%ntbuf,:,:,:] = dset[rz1:rz2,ry1:ry2,rx1:rx2]
                                                else:
                                                    raise RuntimeError
                                    
                                    else:
                                        if self.hasGridFilter:
                                            d_ = dset[()]
                                            databuf[scalar][tiii%ntbuf,:,:,:] = d_[ np.ix_(self.xfi,self.yfi,self.zfi) ].T
                                        else:
                                            if hf_eas4.dform==1:
                                                databuf[scalar][tiii%ntbuf,:,:,:] = dset[()].T
                                            elif hf_eas4.dform==2:
                                                databuf[scalar][tiii%ntbuf,:,:,:] = dset[()]
                                            else:
                                                raise RuntimeError
                                    
                                    if hf_eas4.usingmpi: comm_eas4.Barrier()
                                    t_delta = timeit.default_timer() - t_start
                                    
                                    data_gb       = dset.dtype.itemsize * np.prod(dset.shape) / 1024**3
                                    t_read       += t_delta
                                    data_gb_read += data_gb
                                    
                                    if report_reads and verbose:
                                        txt = even_print(f'read: {scalar}', f'{data_gb:0.3f} [GB]  {t_delta:0.3f} [s]  {data_gb/t_delta:0.3f} [GB/s]', s=True)
                                        tqdm.write(txt)
                            
                            ## collective write
                            if (buffer_nts_loaded%ntbuf==0): ## if buffer is full... initiate write
                                buffer_nts_loaded = 0 ## reset
                                buffers_written += 1 ## increment
                                
                                ## the time index range in CGD to write contents of R/W buffer to
                                ti1 = ntbuf*buffers_written
                                ti2 = ti1+ntbuf
                                #if verbose: tqdm.write(f'performing write: {ti1:d}:{ti2:d}') ## debug
                                
                                for scalar in self.scalars:
                                    
                                    dset = self[f'data/{scalar}'] ## dset in CGD
                                    
                                    if self.usingmpi: self.comm.Barrier()
                                    t_start = timeit.default_timer()
                                    if self.usingmpi:
                                        with dset.collective:
                                            dset[ti1:ti2,rz1:rz2,ry1:ry2,rx1:rx2] = databuf[scalar][:,:,:,:]
                                    else:
                                        dset[ti1:ti2,:,:,:] = databuf[scalar][:,:,:,:]
                                    
                                    if self.usingmpi: self.comm.Barrier()
                                    t_delta = timeit.default_timer() - t_start
                                    
                                    t_write       += t_delta
                                    data_gb       = ntbuf * databuf[scalar].dtype.itemsize * np.prod(dset.shape[1:]) / 1024**3
                                    data_gb_write += data_gb
                                    
                                    if report_writes and verbose:
                                        txt = even_print(f'write: {scalar}', f'{data_gb:0.3f} [GB]  {t_delta:0.3f} [s]  {data_gb/t_delta:0.3f} [GB/s]', s=True)
                                        tqdm.write(txt)
                                    
                                    ## write zeros to buffer (optional)
                                    databuf[scalar][:,:,:,:] = 0.
                                
                                if verbose: progress_bar.update() ## progress bar counts buffer dumps
                
                ## (optionally) delete source EAS4 file
                ## 'do_delete.txt' must be present to actually initiate deletion
                if delete_after_import:
                    if (self.rank==0):
                        if os.path.isfile('do_delete.txt'):
                            tqdm.write(even_print('deleting', fn_eas4, s=True))
                            os.remove(fn_eas4)
                    self.comm.Barrier()
            
            if verbose: progress_bar.close()
            
            if hf_eas4.usingmpi: comm_eas4.Barrier()
            if self.usingmpi: self.comm.Barrier()
        
        self.get_header(verbose=False)
        
        # ## get read read/write stopwatch totals all ranks
        # if not init_dsets_only:
        #     if self.usingmpi:
        #         G = self.comm.gather([data_gb_read, data_gb_write, self.rank], root=0)
        #         G = self.comm.bcast(G, root=0)
        #         data_gb_read  = sum([x[0] for x in G])
        #         data_gb_write = sum([x[1] for x in G])
        
        if init_dsets_only:
            if verbose: print('>>> init_dsets_only=True, so no EAS4 data was imported')
        
        if verbose: print(72*'-')
        if verbose: even_print('nt',       '%i'%self.nt )
        if verbose: even_print('dt',       '%0.8f'%self.dt )
        if verbose: even_print('duration', '%0.2f'%self.duration )
        
        if not init_dsets_only:
            if verbose: print(72*'-')
            if verbose: even_print('time read',format_time_string(t_read))
            if verbose: even_print('time write',format_time_string(t_write))
            if verbose: even_print('read total avg', f'{data_gb_read:0.2f} [GB]  {t_read:0.2f} [s]  {(data_gb_read/t_read):0.3f} [GB/s]')
            if verbose: even_print('write total avg', f'{data_gb_write:0.2f} [GB]  {t_write:0.2f} [s]  {(data_gb_write/t_write):0.3f} [GB/s]')
        
        ## Report file
        if self.usingmpi:
            self.comm.Barrier()
        if verbose:
            print(72*'-')
            even_print( os.path.basename(self.fname), f'{(os.path.getsize(self.fname)/1024**3):0.2f} [GB]')
        if verbose: print(72*'-')
        if verbose: print('total time : cgd.import_eas4() : %s'%format_time_string((timeit.default_timer() - t_start_func)))
        if verbose: print(72*'-')
        return
    
    def make_xdmf(self, **kwargs):
        '''
        Generate an XDMF/XMF2 from CGD for processing with Paraview
        -----
        https://www.xdmf.org/index.php/XDMF_Model_and_Format
        '''
        
        if (self.rank==0):
            verbose = True
        else:
            verbose = False
        
        def _fmt_block(s,indent=0):
            ''' Text block formatter helper '''
            s = textwrap.dedent(s).strip("\n")
            lines = [ln.rstrip() for ln in s.splitlines()]
            s = "\n".join(lines)
            if s and not s.endswith("\n"):
                s += "\n"
            return textwrap.indent(s, indent*" ")
        
        makeVectors = kwargs.get('makeVectors',True) ## write vectors (e.g. velocity, vorticity) to XDMF
        #makeTensors = kwargs.get('makeTensors',True) ## write 3x3 tensors (e.g. stress, strain) to XDMF
        
        fname_path            = os.path.dirname(self.fname)
        fname_base            = os.path.basename(self.fname)
        fname_root, fname_ext = os.path.splitext(fname_base)
        fname_xdmf_base       = fname_root+'.xmf2'
        fname_xdmf            = os.path.join(fname_path, fname_xdmf_base)
        
        if verbose: print('\n'+'cgd.make_xdmf()'+'\n'+72*'-')
        
        dataset_precision_dict = {} ## holds dtype.itemsize ints i.e. 4,8
        dataset_numbertype_dict = {} ## holds string description of dtypes i.e. 'Float','Integer'
        
        # === 1D coordinate dimension vectors --> get dtype.name
        for scalar in ['x','y','z']:
            if ('dims/'+scalar in self):
                data = self['dims/'+scalar]
                
                txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
                if verbose: even_print(scalar, txt)
                
                dataset_precision_dict[scalar] = data.dtype.itemsize
                if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                    dataset_numbertype_dict[scalar] = 'Float'
                elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                    dataset_numbertype_dict[scalar] = 'Integer'
                else:
                    raise ValueError('dtype not recognized, please update script accordingly')
        
        ## Refresh header
        self.get_header(verbose=False, read_grid=False)
        
        for scalar in self.scalars:
            data = self['data/%s'%scalar]
            
            dataset_precision_dict[scalar] = data.dtype.itemsize
            txt = '%s%s%s%s%s'%(data.dtype.itemsize, ' '*(4-len(str(data.dtype.itemsize))), data.dtype.name, ' '*(10-len(str(data.dtype.name))), data.dtype.byteorder)
            if verbose: even_print(scalar, txt)
            
            if (data.dtype.name=='float32') or (data.dtype.name=='float64'):
                dataset_numbertype_dict[scalar] = 'Float'
            elif (data.dtype.name=='int8') or (data.dtype.name=='int16') or (data.dtype.name=='int32') or (data.dtype.name=='int64'):
                dataset_numbertype_dict[scalar] = 'Integer'
            else:
                raise TypeError('dtype not recognized, please update script accordingly')
        
        if verbose: print(72*'-')
        
        # === Write to .xdmf/.xmf2 file
        if (self.rank==0):
            
            if not os.path.isfile(fname_xdmf): ## if doesnt exist...
                Path(fname_xdmf).touch() ## touch XDMF file
                perms_h5 = oct(os.stat(self.fname).st_mode)[-3:] ## get permissions of CGD file
                os.chmod(fname_xdmf, int(perms_h5, base=8)) ## change permissions of XDMF file
            
            #with open(fname_xdmf,'w') as xdmf:
            with io.open(fname_xdmf,'w',newline='\n') as xdmf:
                
                xdmf_str='''
                         <?xml version="1.0" encoding="utf-8"?>
                         <!DOCTYPE Xdmf SYSTEM "Xdmf.dtd" []>
                         <Xdmf xmlns:xi="http://www.w3.org/2001/XInclude" Version="2.0">
                           <Domain>
                         '''
                xdmf.write(_fmt_block(xdmf_str,indent=0))
                
                ## <Topology TopologyType="3DRectMesh" NumberOfElements="{self.nz:d} {self.ny:d} {self.nx:d}"/>
                ## <Geometry GeometryType="VxVyVz">
                
                xdmf_str=f'''
                         <Topology TopologyType="3DSMesh" NumberOfElements="{self.nz:d} {self.ny:d} {self.nx:d}"/>
                         <Geometry GeometryType="X_Y_Z">
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['x']}" Precision="{dataset_precision_dict['x']:d}" Format="HDF">
                             {fname_base}:/dims/{'x'}
                           </DataItem>
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['y']}" Precision="{dataset_precision_dict['y']:d}" Format="HDF">
                             {fname_base}:/dims/{'y'}
                           </DataItem>
                           <DataItem Dimensions="{self.nx:d} {self.ny:d} {self.nz:d}" NumberType="{dataset_numbertype_dict['z']}" Precision="{dataset_precision_dict['z']:d}" Format="HDF">
                             {fname_base}:/dims/{'z'}
                           </DataItem>
                         </Geometry>
                         '''
                xdmf.write(_fmt_block(xdmf_str,indent=4))
                
                # === Time series
                
                xdmf_str='''
                         <!-- ==================== Time series ==================== -->
                         '''
                xdmf.write(_fmt_block(xdmf_str,indent=4))
                
                xdmf_str='''
                         <Grid Name="TimeSeries" GridType="Collection" CollectionType="Temporal">
                         '''
                xdmf.write(_fmt_block(xdmf_str,indent=4))
                
                for ti in range(len(self.t)):
                    
                    dset_name = 'ts_%08d'%ti
                    
                    xdmf_str='''
                             <!-- ============================================================ -->
                             '''
                    xdmf.write(_fmt_block(xdmf_str,indent=6))
                    
                    # =====
                    
                    xdmf_str=f'''
                             <Grid Name="{dset_name}" GridType="Uniform">
                               <Time TimeType="Single" Value="{self.t[ti]:0.8E}"/>
                               <Topology Reference="/Xdmf/Domain/Topology[1]" />
                               <Geometry Reference="/Xdmf/Domain/Geometry[1]" />
                             '''
                    xdmf.write(_fmt_block(xdmf_str,indent=6))
                    
                    # ===== .xdmf : <Grid> per 3D coordinate array
                    
                    for scalar in ['x','y','z']:
                        dset_hf_path = 'dims/%s'%scalar
                        scalar_name = scalar
                        xdmf_str=f'''
                                 <!-- ===== Scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                     {fname_base}:/{dset_hf_path}
                                   </DataItem>
                                 </Attribute>
                                 '''
                        xdmf.write(_fmt_block(xdmf_str,indent=8))
                    
                    # ===== .xdmf : <Grid> per scalar
                    
                    for scalar in self.scalars:
                        dset_hf_path = 'data/%s'%scalar
                        scalar_name = scalar
                        xdmf_str=f'''
                                 <!-- ===== Scalar : {scalar} ===== -->
                                 <Attribute Name="{scalar_name}" AttributeType="Scalar" Center="Node">
                                   <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                     <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                       {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                       {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                       {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                     </DataItem>
                                     <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict[scalar]}" Precision="{dataset_precision_dict[scalar]:d}" Format="HDF">
                                       {fname_base}:/{dset_hf_path}
                                     </DataItem>
                                   </DataItem>
                                 </Attribute>
                                 '''
                        xdmf.write(_fmt_block(xdmf_str,indent=8))
                    
                    if makeVectors:
                        
                        # === .xdmf : <Grid> per vector : velocity vector
                        
                        if ('u' in self.scalars) and ('v' in self.scalars) and ('w' in self.scalars):
                            
                            scalar_name    = 'velocity'
                            dset_hf_path_i = 'data/u'
                            dset_hf_path_j = 'data/v'
                            dset_hf_path_k = 'data/w'
                            
                            xdmf_str = f'''
                            <!-- ===== Vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['u']}" Precision="{dataset_precision_dict['u']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['v']}" Precision="{dataset_precision_dict['v']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['w']}" Precision="{dataset_precision_dict['w']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(_fmt_block(xdmf_str,indent=8))
                        
                        # === .xdmf : <Grid> per vector : vorticity vector
                        
                        if ('vort_x' in self.scalars) and ('vort_y' in self.scalars) and ('vort_z' in self.scalars):
                            
                            scalar_name    = 'vorticity'
                            dset_hf_path_i = 'data/vort_x'
                            dset_hf_path_j = 'data/vort_y'
                            dset_hf_path_k = 'data/vort_z'
                            
                            xdmf_str = f'''
                            <!-- ===== vector : {scalar_name} ===== -->
                            <Attribute Name="{scalar_name}" AttributeType="Vector" Center="Node">
                              <DataItem Dimensions="{self.nz:d} {self.ny:d} {self.nx:d} {3:d}" Function="JOIN($0, $1, $2)" ItemType="Function">
                                <!-- 1 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_x']}" Precision="{dataset_precision_dict['vort_x']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_i}
                                  </DataItem>
                                </DataItem>
                                <!-- 2 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_y']}" Precision="{dataset_precision_dict['vort_y']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_j}
                                  </DataItem>
                                </DataItem>
                                <!-- 3 -->
                                <DataItem ItemType="HyperSlab" Dimensions="{self.nz:d} {self.ny:d} {self.nx:d}" Type="HyperSlab">
                                  <DataItem Dimensions="3 4" NumberType="Integer" Format="XML">
                                    {ti:<6d} {0:<6d} {0:<6d} {0:<6d}
                                    {1:<6d} {1:<6d} {1:<6d} {1:<6d}
                                    {1:<6d} {self.nz:<6d} {self.ny:<6d} {self.nx:<6d}
                                  </DataItem>
                                  <DataItem Dimensions="{self.nt:d} {self.nz:d} {self.ny:d} {self.nx:d}" NumberType="{dataset_numbertype_dict['vort_z']}" Precision="{dataset_precision_dict['vort_z']:d}" Format="HDF">
                                    {fname_base}:/{dset_hf_path_k}
                                  </DataItem>
                                </DataItem>
                                <!-- - -->
                              </DataItem>
                            </Attribute>
                            '''
                            
                            xdmf.write(_fmt_block(xdmf_str,indent=8))
                    
                    # === .xdmf : end Grid for this timestep
                    
                    xdmf_str='''
                             </Grid>
                             '''
                    xdmf.write(_fmt_block(xdmf_str,indent=6))
                
                # ===
                
                xdmf_str='''
                             </Grid>
                           </Domain>
                         </Xdmf>
                         '''
                xdmf.write(_fmt_block(xdmf_str,indent=0))
        
        if self.usingmpi:
            self.comm.Barrier()
        if verbose: print(f'--w-> {fname_xdmf_base}')
        return
    
    # ==================================================================
    # External attachments
    # ==================================================================
    
    # ...
