#!/usr/bin/env python
#   This file is part of nexdatas - Tango Server for NeXus data writer
#
#    Copyright (C) 2012-2018 DESY, Jan Kotanski <jkotan@mail.desy.de>
#
#    nexdatas is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    nexdatas is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with nexdatas.  If not, see <http://www.gnu.org/licenses/>.
# \package test nexdatas
# \file XMLConfiguratorTest.py
# unittests for field Tags running Tango Server
#
import unittest
import os
import sys
import random
import struct
import binascii
import shutil
import fabio
import numpy as np
# import time
# import threading
# import PyTango
import json
from nxstools import nxscollect
from nxstools import filewriter
try:
    from pninexus import h5cpp
    H5CPP = True
except ImportError:
    H5CPP = False


try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO


if sys.version_info > (3,):
    unicode = str
    long = int

WRITERS = {}

try:
    from nxstools import h5pywriter
    WRITERS["h5py"] = h5pywriter
except Exception:
    pass

try:
    from nxstools import h5cppwriter
    WRITERS["h5cpp"] = h5cppwriter
except Exception:
    pass


# if 64-bit machione
IS64BIT = (struct.calcsize("P") == 8)

# from nxsconfigserver.XMLConfigurator  import XMLConfigurator
# from nxsconfigserver.Merger import Merger
# from nxsconfigserver.Errors import (
# NonregisteredDBRecordError, UndefinedTagError,
#                                    IncompatibleNodeError)
# import nxsconfigserver


def myinput(w, text):
    myio = os.fdopen(w, 'w')
    myio.write(text)

    # myio.close()


# test fixture
class NXSCollectTest(unittest.TestCase):

    # constructor
    # \param methodName name of the test method
    def __init__(self, methodName):
        unittest.TestCase.__init__(self, methodName)
        # self.maxDiff = None

        self.helperror = "Error: too few arguments\n"

        self.helpinfo = """usage: nxscollect [-h] {append,link,vds} ...

  Command-line tool to merge images of external file-formats """ + \
            """into the master NeXus file

positional arguments:
  {append,link,vds}  sub-command help
    append           append images to the master file
    link             create an external or internal link in the master file
    vds              create a virual dataset in the master file

optional arguments:
  -h, --help         show this help message and exit

For more help:
  nxscollect <sub-command> -h

"""

        try:
            # random seed
            self.seed = long(binascii.hexlify(os.urandom(16)), 16)
        except NotImplementedError:
            import time
            # random seed
            self.seed = long(time.time() * 256)  # use fractional seconds

        self.__rnd = random.Random(self.seed)

        self._bint = "int64" if IS64BIT else "int32"
        self._buint = "uint64" if IS64BIT else "uint32"
        self._bfloat = "float64" if IS64BIT else "float32"

        if "h5cpp" in WRITERS.keys():
            self.writer = "h5cpp"
        else:
            self.writer = "h5py"

        self.flags = ""
        self.externalfilters = True

    # test starter
    # \brief Common set up
    def setUp(self):
        print("\nsetting up...")
        print("SEED = %s" % self.seed)

    # test closer
    # \brief Common tear down
    def tearDown(self):
        print("tearing down ...")

    # Exception tester
    # \param exception expected exception
    # \param method called method
    # \param args list with method arguments
    # \param kwargs dictionary with method arguments
    def myAssertRaise(self, exception, method, *args, **kwargs):
        try:
            error = False
            method(*args, **kwargs)
        except exception:
            error = True
        self.assertEqual(error, True)

    # sets selection configuration
    # \param selectionc configuration instance
    # \param selection selection configuration string
    def setSelection(self, selectionc, selection):
        selectionc.selection = selection

    # gets selectionconfiguration
    # \param selectionc configuration instance
    # \returns selection configuration string
    def getSelection(self, selectionc):
        return selectionc.selection

    def test_default(self):
        """ test nxsconfig default
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        old_stdout = sys.stdout
        old_stderr = sys.stderr
        sys.stdout = mystdout = StringIO()
        sys.stderr = mystderr = StringIO()
        old_argv = sys.argv
        sys.argv = ['nxscollect']
        with self.assertRaises(SystemExit):
            nxscollect.main()

        sys.argv = old_argv
        sys.stdout = old_stdout
        sys.stderr = old_stderr
        vl = mystdout.getvalue()
        er = mystderr.getvalue()
        self.assertEqual("".join(self.helpinfo.split()),
                         "".join(vl.split()))
        self.assertEqual(self.helperror, er)

    def test_help(self):
        """ test nxsconfig help
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        helps = ['-h', '--help']
        for hl in helps:
            old_stdout = sys.stdout
            old_stderr = sys.stderr
            sys.stdout = mystdout = StringIO()
            sys.stderr = mystderr = StringIO()
            old_argv = sys.argv
            sys.argv = ['nxscollect', hl]
            with self.assertRaises(SystemExit):
                nxscollect.main()

            sys.argv = old_argv
            sys.stdout = old_stdout
            sys.stderr = old_stderr
            vl = mystdout.getvalue()
            er = mystderr.getvalue()
            self.assertEqual("".join(self.helpinfo[0:-1].split()),
                             "".join(vl.split()))
            self.assertEqual('', er)

    def test_append_test_emptyfile(self):
        """ test nxsconfig append empty file
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)

        commands = [
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s -r %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            nxsfile = filewriter.create_file(filename, overwrite=True)
            nxsfile.close()

            for cmd in commands:
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertEqual('', vl)

        finally:
            os.remove(filename)

    def test_append_test_nofile(self):
        """ test nxsconfig append empty file
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        commands = [
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append -s %s %s' % (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append -s %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s -r %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        nxsfile = filewriter.create_file(
            filename, overwrite=True)
        nxsfile.close()
        os.remove(filename)

        for cmd in commands:
            old_stdout = sys.stdout
            old_stderr = sys.stderr
            sys.stdout = mystdout = StringIO()
            sys.stderr = mystderr = StringIO()
            old_argv = sys.argv
            sys.argv = cmd
            self.myAssertRaise(IOError, nxscollect.main)

            sys.argv = old_argv
            sys.stdout = old_stdout
            sys.stderr = old_stderr
            vl = mystdout.getvalue()
            er = mystderr.getvalue()

            self.assertEqual('', er)
            self.assertEqual('', vl)

    def test_append_test_file_withdata(self):
        """ test nxsconfig append file with data field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        commands = [
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -s  %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s  %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s -r %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            nxsfile = filewriter.create_file(
                filename, overwrite=True)
            rt = nxsfile.root()
            entry = rt.create_group("entry12345", "NXentry")
            ins = entry.create_group("instrument", "NXinstrument")
            det = ins.create_group("detector", "NXdetector")
            entry.create_group("data", "NXdata")
            det.create_field("intimage", "uint32", [0, 30], [1, 30])
            nxsfile.close()

            for cmd in commands:
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertEqual('', vl)
                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)

        finally:
            os.remove(filename)

    def test_append_test_file_withpostrun_nofile(self):
        """ test nxsconfig append file with data field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append -r %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test -r %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test -r %s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            nxsfile = filewriter.create_file(
                filename, overwrite=True)
            rt = nxsfile.root()
            entry = rt.create_group("entry12345", "NXentry")
            ins = entry.create_group("instrument", "NXinstrument")
            det = ins.create_group("detector", "NXdetector")
            entry.create_group("data", "NXdata")
            col = det.create_group("collection", "NXcollection")
            postrun = col.create_field("postrun", "string")
            postrun.write("test1_%05d.cbf:0:5")
            nxsfile.close()

            for cmd in commands:
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                # if '-r' not in cmd:
                #     os.remove("%s.__nxscollect_old__" % filename)

        finally:
            os.remove(filename)

    def test_append_file_withpostrun_tif(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file0.tif', './test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif', './test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif', './test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif', './test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif', './test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif', './test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                for i in range(1, 6):

                    self.assertTrue(svl[i].startswith(' * append '))
                    self.assertTrue(
                        svl[i].endswith('test1_%05d.tif ' % (i - 1)))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open('./test1_%05d.tif' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.tif')
            os.remove('./test1_00001.tif')
            os.remove('./test1_00002.tif')
            os.remove('./test1_00003.tif')
            os.remove('./test1_00004.tif')
            os.remove('./test1_00005.tif')

    def test_append_file_withpostrun_tif_pilatus300k_comp(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r -s -c1  %s' %
             (filename, self.flags)).split(),
            ('nxscollect append  %s -c2 %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -c3 %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -c4 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -c5 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append  %s -s -c6 %s' %
             (filename, self.flags)).split(),

            ('nxscollect append %s -s -c7 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s -c8 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s -c9' %
             (filename, self.flags)).split(),
        ]
        extra_commands = [
            ('nxscollect append  %s -r -s -c32008:0,2 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s -c32008:0,2' %
             (filename, self.flags)).split(),
        ]
        if self.externalfilters and H5CPP:
            if hasattr(h5cpp.filter, "is_filter_available") \
               and h5cpp.filter.is_filter_available(32008):
                commands.extend(extra_commands)

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file0.tif',
                         './testcollect/pilatus300k/test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif',
                         './testcollect/pilatus300k/test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif',
                         './testcollect/pilatus300k/test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif',
                         './testcollect/pilatus300k/test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif',
                         './testcollect/pilatus300k/test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif',
                         './testcollect/pilatus300k/test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                for i in range(1, 6):
                    self.assertTrue(
                        svl[i],
                        ' * append /home/jkotan/ndts/nexdatas.tools/'
                        'test1_%05d.tif ' % (i - 1)
                    )

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open(
                        './testcollect/pilatus300k/test1_%05d.tif' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.tif')
            os.remove('./testcollect/pilatus300k/test1_00001.tif')
            os.remove('./testcollect/pilatus300k/test1_00002.tif')
            os.remove('./testcollect/pilatus300k/test1_00003.tif')
            os.remove('./testcollect/pilatus300k/test1_00004.tif')
            os.remove('./testcollect/pilatus300k/test1_00005.tif')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_append_file_withpostrun_tif_pilatus300k_skip(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file0.tif',
                         './testcollect/pilatus300k/test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif',
                         './testcollect/pilatus300k/test1_00001.tif')
            # shutil.copy2('test/files/test_file2.tif',
            #              './testcollect/pilatus300k/test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif',
                         './testcollect/pilatus300k/test1_00003.tif')
            # shutil.copy2('test/files/test_file4.tif',
            #              './testcollect/pilatus300k/test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif',
                         './testcollect/pilatus300k/test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                for i in range(1, 6):
                    if i not in [3, 5]:
                        self.assertEqual(
                            svl[i],
                            ' * append testcollect/pilatus300k/'
                            'test1_%05d.tif ' % (i - 1)
                        )
                    else:
                        self.assertTrue(
                            svl[i].startswith("Cannot open any of "))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (4, 195, 487))
                ii = 0
                for i in range(6):
                    if i not in [2, 4]:
                        fbuffer = fabio.open(
                            './testcollect/pilatus300k/test1_%05d.tif' % i)
                        fimage = fbuffer.data[...]
                        image = buffer[ii, :, :]
                        self.assertTrue((image == fimage).all())
                        ii += 1
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.tif')
            os.remove('./testcollect/pilatus300k/test1_00001.tif')
            # os.remove('./testcollect/pilatus300k/test1_00002.tif')
            os.remove('./testcollect/pilatus300k/test1_00003.tif')
            # os.remove('./testcollect/pilatus300k/test1_00004.tif')
            os.remove('./testcollect/pilatus300k/test1_00005.tif')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_append_file_withpostrun_tif_pilatus300k_wait(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                shutil.copy2('test/files/test_file0.tif',
                             './testcollect/pilatus300k/test1_00000.tif')
                shutil.copy2('test/files/test_file1.tif',
                             './testcollect/pilatus300k/test1_00001.tif')
                shutil.copy2('test/files/test_file2.tif',
                             './testcollect/pilatus300k/test1_00002.tif')

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 8)

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (3, 195, 487))
                ii = 0
                for i in range(3):
                    fbuffer = fabio.open(
                        './testcollect/pilatus300k/test1_%05d.tif' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[ii, :, :]
                    self.assertTrue((image == fimage).all())
                    ii += 1
                nxsfile.close()

                shutil.copy2('test/files/test_file3.tif',
                             './testcollect/pilatus300k/test1_00003.tif')
                shutil.copy2('test/files/test_file4.tif',
                             './testcollect/pilatus300k/test1_00004.tif')

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (5, 195, 487))
                ii = 0
                for i in range(5):
                    fbuffer = fabio.open(
                        './testcollect/pilatus300k/test1_%05d.tif' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[ii, :, :]
                    self.assertTrue((image == fimage).all())
                    ii += 1
                nxsfile.close()

                os.remove('./testcollect/pilatus300k/test1_00000.tif')
                os.remove('./testcollect/pilatus300k/test1_00001.tif')
                os.remove('./testcollect/pilatus300k/test1_00002.tif')
                os.remove('./testcollect/pilatus300k/test1_00003.tif')
                os.remove('./testcollect/pilatus300k/test1_00004.tif')
                os.remove(filename)

        finally:
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_append_file_withpostrun_tif_pilatus300k_missing(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file0.tif',
                         './testcollect/pilatus300k/test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif',
                         './testcollect/pilatus300k/test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif',
                         './testcollect/pilatus300k/test1_00002.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 6)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                for i in range(1, 5):
                    if i not in [4]:
                        self.assertEqual(
                            svl[i],
                            ' * append testcollect/pilatus300k/'
                            'test1_%05d.tif ' % (i - 1)
                        )
                    else:
                        self.assertTrue(
                            svl[i].startswith("Cannot open any of "))

                # if '-r' not in cmd:
                #      os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.tif')
            os.remove('./testcollect/pilatus300k/test1_00001.tif')
            os.remove('./testcollect/pilatus300k/test1_00002.tif')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_append_file_withpostrun_cbf(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file.cbf', './test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                for i in range(1, 6):

                    self.assertTrue(svl[i].startswith(' * append '))
                    self.assertTrue(
                        svl[i].endswith('test1_%05d.cbf ' % (i - 1)))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open('./test1_%05d.cbf' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.cbf')
            os.remove('./test1_00001.cbf')
            os.remove('./test1_00002.cbf')
            os.remove('./test1_00003.cbf')
            os.remove('./test1_00004.cbf')
            os.remove('./test1_00005.cbf')

    def test_append_file_withpostrun_cbf_pilatus300k_comp(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r -s -c1 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append  %s -c2 %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -c3 %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -c4 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -c5 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append  %s -s -c6 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -s -c7 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s -c8 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s -c9' %
             (filename, self.flags)).split(),
        ]
        extra_commands = [
            ('nxscollect append  %s -r -s -c32008:0,2 %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s -c32008:0,2' %
             (filename, self.flags)).split(),
        ]
        if self.externalfilters and H5CPP:
            if hasattr(h5cpp.filter, "is_filter_available") \
               and h5cpp.filter.is_filter_available(32008):
                commands.extend(extra_commands)

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()
                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()
                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 8:
                    print(svl)
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                for i in range(1, 6):
                    self.assertTrue(
                        svl[i],
                        ' * append /home/jkotan/ndts/nexdatas.tools/'
                        'test1_%05d.cbf ' % (i - 1)
                    )

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open(
                        './testcollect/pilatus300k/test1_%05d.cbf' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.cbf')
            os.remove('./testcollect/pilatus300k/test1_00001.cbf')
            os.remove('./testcollect/pilatus300k/test1_00002.cbf')
            os.remove('./testcollect/pilatus300k/test1_00003.cbf')
            os.remove('./testcollect/pilatus300k/test1_00004.cbf')
            os.remove('./testcollect/pilatus300k/test1_00005.cbf')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_append_file_withpostrun_cbf_pilatus300k_skip(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00001.cbf')
            # shutil.copy2('test/files/test_file.cbf',
            #              './testcollect/pilatus300k/test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00003.cbf')
            # shutil.copy2('test/files/test_file.cbf',
            #              './testcollect/pilatus300k/test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                for i in range(1, 6):
                    if i not in [3, 5]:
                        self.assertEqual(
                            svl[i],
                            ' * append testcollect/pilatus300k/'
                            'test1_%05d.cbf ' % (i - 1)
                        )
                    else:
                        self.assertTrue(
                            svl[i].startswith("Cannot open any of "))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (4, 195, 487))
                ii = 0
                for i in range(6):
                    if i not in [2, 4]:
                        fbuffer = fabio.open(
                            './testcollect/pilatus300k/test1_%05d.cbf' % i)
                        fimage = fbuffer.data[...]
                        image = buffer[ii, :, :]
                        self.assertTrue((image == fimage).all())
                        ii += 1
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.cbf')
            os.remove('./testcollect/pilatus300k/test1_00001.cbf')
            # os.remove('./testcollect/pilatus300k/test1_00002.cbf')
            os.remove('./testcollect/pilatus300k/test1_00003.cbf')
            # os.remove('./testcollect/pilatus300k/test1_00004.cbf')
            os.remove('./testcollect/pilatus300k/test1_00005.cbf')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_append_file_withpostrun_cbf_pilatus300k_wait(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00000.cbf')
                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00001.cbf')
                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00002.cbf')

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 8)

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (3, 195, 487))
                ii = 0
                for i in range(3):
                    fbuffer = fabio.open(
                        './testcollect/pilatus300k/test1_%05d.cbf' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[ii, :, :]
                    self.assertTrue((image == fimage).all())
                    ii += 1
                nxsfile.close()

                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00003.cbf')
                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00004.cbf')

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (5, 195, 487))
                ii = 0
                for i in range(5):
                    fbuffer = fabio.open(
                        './testcollect/pilatus300k/test1_%05d.cbf' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[ii, :, :]
                    self.assertTrue((image == fimage).all())
                    ii += 1
                nxsfile.close()

                os.remove('./testcollect/pilatus300k/test1_00000.cbf')
                os.remove('./testcollect/pilatus300k/test1_00001.cbf')
                os.remove('./testcollect/pilatus300k/test1_00002.cbf')
                os.remove('./testcollect/pilatus300k/test1_00003.cbf')
                os.remove('./testcollect/pilatus300k/test1_00004.cbf')
                os.remove(filename)

        finally:
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_append_file_withpostrun_cbf_pilatus300k_missing(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00002.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 6)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                for i in range(1, 5):
                    if i not in [4]:
                        self.assertEqual(
                            svl[i],
                            ' * append testcollect/pilatus300k/'
                            'test1_%05d.cbf ' % (i - 1)
                        )
                    else:
                        self.assertTrue(
                            svl[i].startswith("Cannot open any of "))

                # if '-r' not in cmd:
                #      os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.cbf')
            os.remove('./testcollect/pilatus300k/test1_00001.cbf')
            os.remove('./testcollect/pilatus300k/test1_00002.cbf')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_append_file_withpostrun_raw(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    with open("rawtest1_%05d.dat" % i, "w") as fl:
                        attrs[k][0][i].tofile(fl)
                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    col = det.create_group("collection", "NXcollection")
                    postrun = col.create_field("postrun", "string")
                    postrun.write("rawtest1_%05d.dat:0:5")
                    atts = postrun.attributes
                    atts.create("fielddtype", "string").write(attrs[k][2])
                    atts.create("fieldshape", "string").write(
                        json.dumps(attrs[k][0].shape[1:]))
                    nxsfile.close()

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = cmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    # print(svl)
                    self.assertEqual(len(svl), 8)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")
                    for i in range(1, 6):
                        self.assertTrue(svl[i].startswith(' * append '))
                        self.assertTrue(
                            svl[i].endswith('test1_%05d.dat ' % (i - 1)))

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    for i in range(6):
                        fimage = attrs[k][0][i]
                        image = buffer[i, :, :]
                        self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                for i in range(6):
                    os.remove("rawtest1_%05d.dat" % i)

    def test_append_file_withpostrun_h5(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    fl = filewriter.create_file("h5test1_%05d.h5" % i,
                                                overwrite=True)
                    rt = fl.root()
                    shp = attrs[k][0][i].shape
                    data = rt.create_field("data", attrs[k][2], shp, shp)
                    data.write(attrs[k][0][i])
                    data.close()
                    fl.close()
                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    col = det.create_group("collection", "NXcollection")
                    postrun = col.create_field("postrun", "string")
                    postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = cmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    self.assertEqual(len(svl), 8)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")
                    for i in range(1, 6):
                        self.assertTrue(svl[i].startswith(' * append '))
                        self.assertTrue(
                            svl[i].endswith('test1_%05d.h5 ' % (i - 1)))

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    for i in range(6):
                        fimage = attrs[k][0][i]
                        image = buffer[i, :, :]
                        self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                pass
                for i in range(6):
                    os.remove("h5test1_%05d.h5" % i)

    def test_append_file_parameters_tif(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_%05d.tif:0:5'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append  %s %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file0.tif', './test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif', './test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif', './test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif', './test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif', './test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif', './test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 8:
                    print(svl)
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                for i in range(1, 6):

                    self.assertTrue(svl[i].startswith(' * append '))
                    self.assertTrue(
                        svl[i].endswith('test1_%05d.tif ' % (i - 1)))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open('./test1_%05d.tif' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.tif')
            os.remove('./test1_00001.tif')
            os.remove('./test1_00002.tif')
            os.remove('./test1_00003.tif')
            os.remove('./test1_00004.tif')
            os.remove('./test1_00005.tif')

    def test_append_file_parameters_tif_list(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_00000.tif,test1_00001.tif,test1_00002.tif,' \
            'test1_00003.tif,test1_00004.tif,test1_00005.tif'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append  %s %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file0.tif', './test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif', './test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif', './test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif', './test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif', './test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif', './test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 8:
                    print(svl)
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                for i in range(1, 6):

                    self.assertTrue(svl[i].startswith(' * append '))
                    self.assertTrue(
                        svl[i].endswith('test1_%05d.tif ' % (i - 1)))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open('./test1_%05d.tif' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.tif')
            os.remove('./test1_00001.tif')
            os.remove('./test1_00002.tif')
            os.remove('./test1_00003.tif')
            os.remove('./test1_00004.tif')
            os.remove('./test1_00005.tif')

    def test_append_file_parameters_tif_list_sep(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_00000.tif:test1_00001.tif:test1_00002.tif:' \
            'test1_00003.tif:test1_00004.tif:test1_00005.tif'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append  %s %s -i %s -p %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s %s -i %s --path %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r %s -i %s -p %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r %s -i %s --path %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -s %s --input-files %s -p %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -s %s --input-files %s --path %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r -s %s --input-files %s -p %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r -s %s --input-files %s --path %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file0.tif', './test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif', './test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif', './test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif', './test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif', './test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif', './test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 8:
                    print(svl)
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                for i in range(1, 6):

                    self.assertTrue(svl[i].startswith(' * append '))
                    self.assertTrue(
                        svl[i].endswith('test1_%05d.tif ' % (i - 1)))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open('./test1_%05d.tif' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.tif')
            os.remove('./test1_00001.tif')
            os.remove('./test1_00002.tif')
            os.remove('./test1_00003.tif')
            os.remove('./test1_00004.tif')
            os.remove('./test1_00005.tif')

    def test_append_file_parameters_cbf(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_%05d.cbf:0:5'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append  %s %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file.cbf', './test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 8:
                    print(svl)
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                for i in range(1, 6):

                    self.assertTrue(svl[i].startswith(' * append '))
                    self.assertTrue(
                        svl[i].endswith('test1_%05d.cbf ' % (i - 1)))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open('./test1_%05d.cbf' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.cbf')
            os.remove('./test1_00001.cbf')
            os.remove('./test1_00002.cbf')
            os.remove('./test1_00003.cbf')
            os.remove('./test1_00004.cbf')
            os.remove('./test1_00005.cbf')

    def test_append_file_parameters_cbf_list(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_00000.cbf,test1_00001.cbf,test1_00002.cbf,' \
            'test1_00003.cbf,test1_00004.cbf,test1_00005.cbf'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append  %s %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file.cbf', './test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 8:
                    print(svl)
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                for i in range(1, 6):

                    self.assertTrue(svl[i].startswith(' * append '))
                    self.assertTrue(
                        svl[i].endswith('test1_%05d.cbf ' % (i - 1)))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open('./test1_%05d.cbf' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.cbf')
            os.remove('./test1_00001.cbf')
            os.remove('./test1_00002.cbf')
            os.remove('./test1_00003.cbf')
            os.remove('./test1_00004.cbf')
            os.remove('./test1_00005.cbf')

    def test_append_file_parameters_cbf_list_sep(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_00000.cbf:test1_00001.cbf:test1_00002.cbf:' \
            'test1_00003.cbf:test1_00004.cbf:test1_00005.cbf'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append  %s %s -i %s -p %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s %s -i %s --path %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r %s -i %s -p %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r %s -i %s --path %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -s %s --input-files %s -p %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -s %s --input-files %s --path %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append  %s -r -s %s --input-files %s -p %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append %s -r -s %s --input-files %s --path %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file.cbf', './test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 8:
                    print(svl)
                self.assertEqual(len(svl), 8)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                for i in range(1, 6):

                    self.assertTrue(svl[i].startswith(' * append '))
                    self.assertTrue(
                        svl[i].endswith('test1_%05d.cbf ' % (i - 1)))

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                buffer = dt.read()
                self.assertEqual(buffer.shape, (6, 195, 487))
                for i in range(6):
                    fbuffer = fabio.open('./test1_%05d.cbf' % i)
                    fimage = fbuffer.data[...]
                    image = buffer[i, :, :]
                    self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.cbf')
            os.remove('./test1_00001.cbf')
            os.remove('./test1_00002.cbf')
            os.remove('./test1_00003.cbf')
            os.remove('./test1_00004.cbf')
            os.remove('./test1_00005.cbf')

    def test_append_file_parameters_raw(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    with open("rawtest1_%05d.dat" % i, "w") as fl:
                        attrs[k][0][i].tofile(fl)
                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    # to be created
                    # ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    nxsfile.close()
                    pcmd = cmd
                    pcmd.extend(["-i", "rawtest1_%05d.dat:0:5"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])
                    pcmd.extend(
                        ["--shape", json.dumps(attrs[k][0].shape[1:])])
                    pcmd.extend(
                        ["--dtype", attrs[k][2]])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    self.assertEqual(len(svl), 8)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")
                    for i in range(1, 6):
                        self.assertTrue(svl[i].startswith(' * append '))
                        self.assertTrue(
                            svl[i].endswith('test1_%05d.dat ' % (i - 1)))

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    for i in range(6):
                        fimage = attrs[k][0][i]
                        image = buffer[i, :, :]
                        self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                for i in range(6):
                    os.remove("rawtest1_%05d.dat" % i)

    def test_append_file_parameters_nxs(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    fl = filewriter.create_file("h5test1_%05d.nxs" % i,
                                                overwrite=True)
                    rt = fl.root()

                    at = rt.attributes.create("default", "string")
                    at.write("entry12345")
                    at.close()

                    entry = rt.create_group("entry12345", "NXentry")
                    at = entry.attributes.create("default", "string")
                    at.write("data")
                    at.close()

                    dt = entry.create_group("data", "NXdata")
                    at = dt.attributes.create("signal", "string")
                    at.write("data")
                    at.close()

                    shp = attrs[k][0][i].shape
                    data = dt.create_field("data", attrs[k][2], shp, shp)
                    data.write(attrs[k][0][i])
                    data.close()

                    dt.close()
                    entry.close()
                    fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(["-i", "h5test1_%05d.nxs:0:5"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 8:
                        print(svl)
                    self.assertEqual(len(svl), 8)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")
                    for i in range(1, 6):
                        self.assertTrue(svl[i].startswith(' * append '))
                        self.assertTrue(
                            svl[i].endswith('test1_%05d.nxs ' % (i - 1)))

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    for i in range(6):
                        fimage = attrs[k][0][i]
                        image = buffer[i, :, :]
                        self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                pass
                for i in range(6):
                    os.remove("h5test1_%05d.nxs" % i)

    def test_append_file_parameters_nxs_3d_many(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 20),
                    self.__rnd.randint(10, 20),
                    self.__rnd.randint(10, 20)]

            attrs[k][0] = np.array(
                [[[[attrs[k][0] * self.__rnd.randint(0, 3)
                    for d in range(mlen[2])]
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    fl = filewriter.create_file("h5test1_%05d.nxs" % i,
                                                overwrite=True)
                    rt = fl.root()

                    at = rt.attributes.create("default", "string")
                    at.write("entry12345")
                    at.close()

                    entry = rt.create_group("entry12345", "NXentry")
                    at = entry.attributes.create("default", "string")
                    at.write("data")
                    at.close()

                    dt = entry.create_group("data", "NXdata")
                    at = dt.attributes.create("signal", "string")
                    at.write("data")
                    at.close()

                    shp = attrs[k][0][i].shape
                    data = dt.create_field("data", attrs[k][2], shp, shp)
                    data.write(attrs[k][0][i])
                    data.close()

                    dt.close()
                    entry.close()
                    fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(["-i", "h5test1_%05d.nxs:0:5"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 8:
                        print(svl)
                    self.assertEqual(len(svl), 8)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")
                    for i in range(1, 6):
                        self.assertTrue(svl[i].startswith(' * append '))
                        self.assertTrue(
                            svl[i].endswith('test1_%05d.nxs ' % (i - 1)))
                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)

                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(
                        buffer.shape[0],
                        attrs[k][0].shape[0] * attrs[k][0].shape[1])
                    self.assertEqual(buffer.shape[1:],
                                     attrs[k][0].shape[2:])
                    for i in range(6):
                        fimage = attrs[k][0][i]
                        image = buffer[
                            i * attrs[k][0].shape[1]:
                            (i + 1) * attrs[k][0].shape[1], :, :]
                        self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                pass
                for i in range(6):
                    os.remove("h5test1_%05d.nxs" % i)

    def test_append_file_parameters_nxs_1d(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[attrs[k][0] * self.__rnd.randint(0, 3)
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
            )
            try:
                for i in range(6):
                    fl = filewriter.create_file("h5test1_%05d.nxs" % i,
                                                overwrite=True)
                    rt = fl.root()

                    at = rt.attributes.create("default", "string")
                    at.write("entry12345")
                    at.close()

                    entry = rt.create_group("entry12345", "NXentry")
                    at = entry.attributes.create("default", "string")
                    at.write("data")
                    at.close()

                    dt = entry.create_group("data", "NXdata")
                    at = dt.attributes.create("signal", "string")
                    at.write("data")
                    at.close()

                    shp = attrs[k][0][i].shape
                    data = dt.create_field("data", attrs[k][2], shp, shp)
                    data.write(attrs[k][0][i])
                    data.close()

                    dt.close()
                    entry.close()
                    fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(["-i", "h5test1_%05d.nxs:0:5"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 8:
                        print(svl)
                    self.assertEqual(len(svl), 8)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")
                    for i in range(1, 6):
                        self.assertTrue(svl[i].startswith(' * append '))
                        self.assertTrue(
                            svl[i].endswith('test1_%05d.nxs ' % (i - 1)))

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    for i in range(6):
                        fimage = attrs[k][0][i]
                        image = buffer[i, :]
                        self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                pass
                for i in range(6):
                    os.remove("h5test1_%05d.nxs" % i)

    def test_append_file_parameters_nxs_3d(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                fl = filewriter.create_file("h5test1_00001.nxs",
                                            overwrite=True)
                rt = fl.root()

                at = rt.attributes.create("default", "string")
                at.write("entry12345")
                at.close()

                entry = rt.create_group("entry12345", "NXentry")
                at = entry.attributes.create("default", "string")
                at.write("data")
                at.close()

                dt = entry.create_group("data", "NXdata")
                at = dt.attributes.create("signal", "string")
                at.write("data")
                at.close()

                shp = attrs[k][0].shape
                data = dt.create_field("data", attrs[k][2], shp, shp)
                data.write(attrs[k][0])
                data.close()

                dt.close()
                entry.close()
                fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(["-i", "h5test1_00001.nxs"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 3:
                        print(svl)
                    self.assertEqual(len(svl), 3)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['h5test1_00001.nxs']")
                    self.assertTrue(svl[1].startswith(' * append '))
                    self.assertTrue(
                            svl[1].endswith('h5test1_00001.nxs '))

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    for i in range(6):
                        fimage = attrs[k][0][i]
                        image = buffer[i, :, :]
                        self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                os.remove("h5test1_00001.nxs")

    def test_append_file_parameters_nxs_path(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append  %s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -r %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append %s -s %s' % (filename, self.flags)).split(),
            ('nxscollect append  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append %s -r -s %s' % (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    fl = filewriter.create_file("h5test1_%05d.nxs" % i,
                                                overwrite=True)
                    rt = fl.root()
                    entry = rt.create_group("entry345", "NXentry")

                    dt = entry.create_group("data", "NXdata")

                    shp = attrs[k][0][i].shape
                    data = dt.create_field("data", attrs[k][2], shp, shp)
                    data.write(attrs[k][0][i])
                    data.close()

                    dt.close()
                    entry.close()
                    fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(
                        ["-i", "h5test1_%05d.nxs://entry345/data/data:0:5"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 8:
                        print(svl)
                    self.assertEqual(len(svl), 8)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.nxs:0:5']")
                    for i in range(1, 6):
                        self.assertTrue(svl[i].startswith(' * append '))
                        self.assertTrue(
                            svl[i].endswith('test1_%05d.nxs ' % (i - 1)))

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    for i in range(6):
                        fimage = attrs[k][0][i]
                        image = buffer[i, :, :]
                        self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                pass
                for i in range(6):
                    os.remove("h5test1_%05d.nxs" % i)

    def test_test_file_withpostrun_tif(self):
        """ test nxsconfig test file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file0.tif', './test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif', './test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif', './test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif', './test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif', './test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif', './test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.tif')
            os.remove('./test1_00001.tif')
            os.remove('./test1_00002.tif')
            os.remove('./test1_00003.tif')
            os.remove('./test1_00004.tif')
            os.remove('./test1_00005.tif')

    def test_test_file_withpostrun_tif_pilatus300k(self):
        """ test nxsconfig test file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file0.tif',
                         './testcollect/pilatus300k/test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif',
                         './testcollect/pilatus300k/test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif',
                         './testcollect/pilatus300k/test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif',
                         './testcollect/pilatus300k/test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif',
                         './testcollect/pilatus300k/test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif',
                         './testcollect/pilatus300k/test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.tif')
            os.remove('./testcollect/pilatus300k/test1_00001.tif')
            os.remove('./testcollect/pilatus300k/test1_00002.tif')
            os.remove('./testcollect/pilatus300k/test1_00003.tif')
            os.remove('./testcollect/pilatus300k/test1_00004.tif')
            os.remove('./testcollect/pilatus300k/test1_00005.tif')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_test_file_withpostrun_tif_pilatus300k_skip(self):
        """ test nxsconfig test file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file0.tif',
                         './testcollect/pilatus300k/test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif',
                         './testcollect/pilatus300k/test1_00001.tif')
            # shutil.copy2('test/files/test_file2.tif',
            #              './testcollect/pilatus300k/test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif',
                         './testcollect/pilatus300k/test1_00003.tif')
            # shutil.copy2('test/files/test_file4.tif',
            #              './testcollect/pilatus300k/test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif',
                         './testcollect/pilatus300k/test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                print(svl)
                self.assertEqual(len(svl), 4)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.tif')
            os.remove('./testcollect/pilatus300k/test1_00001.tif')
            # os.remove('./testcollect/pilatus300k/test1_00002.tif')
            os.remove('./testcollect/pilatus300k/test1_00003.tif')
            # os.remove('./testcollect/pilatus300k/test1_00004.tif')
            os.remove('./testcollect/pilatus300k/test1_00005.tif')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_test_file_withpostrun_tif_pilatus300k_wait(self):
        """ test nxsconfig test file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                shutil.copy2('test/files/test_file0.tif',
                             './testcollect/pilatus300k/test1_00000.tif')
                shutil.copy2('test/files/test_file1.tif',
                             './testcollect/pilatus300k/test1_00001.tif')
                shutil.copy2('test/files/test_file2.tif',
                             './testcollect/pilatus300k/test1_00002.tif')

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()

                shutil.copy2('test/files/test_file3.tif',
                             './testcollect/pilatus300k/test1_00003.tif')
                shutil.copy2('test/files/test_file4.tif',
                             './testcollect/pilatus300k/test1_00004.tif')

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()

                os.remove('./testcollect/pilatus300k/test1_00000.tif')
                os.remove('./testcollect/pilatus300k/test1_00001.tif')
                os.remove('./testcollect/pilatus300k/test1_00002.tif')
                os.remove('./testcollect/pilatus300k/test1_00003.tif')
                os.remove('./testcollect/pilatus300k/test1_00004.tif')
                os.remove(filename)

        finally:
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_test_file_withpostrun_tif_pilatus300k_missing(self):
        """ test nxsconfig test file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file0.tif',
                         './testcollect/pilatus300k/test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif',
                         './testcollect/pilatus300k/test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif',
                         './testcollect/pilatus300k/test1_00002.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 3)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")

                # if '-r' not in cmd:
                #      os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.tif')
            os.remove('./testcollect/pilatus300k/test1_00001.tif')
            os.remove('./testcollect/pilatus300k/test1_00002.tif')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_test_file_withpostrun_cbf(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file.cbf', './test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.cbf')
            os.remove('./test1_00001.cbf')
            os.remove('./test1_00002.cbf')
            os.remove('./test1_00003.cbf')
            os.remove('./test1_00004.cbf')
            os.remove('./test1_00005.cbf')

    def test_test_file_withpostrun_cbf_pilatus300k(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.cbf')
            os.remove('./testcollect/pilatus300k/test1_00001.cbf')
            os.remove('./testcollect/pilatus300k/test1_00002.cbf')
            os.remove('./testcollect/pilatus300k/test1_00003.cbf')
            os.remove('./testcollect/pilatus300k/test1_00004.cbf')
            os.remove('./testcollect/pilatus300k/test1_00005.cbf')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_test_file_withpostrun_cbf_pilatus300k_skip(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00001.cbf')
            # shutil.copy2('test/files/test_file.cbf',
            #              './testcollect/pilatus300k/test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00003.cbf')
            # shutil.copy2('test/files/test_file.cbf',
            #              './testcollect/pilatus300k/test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 4)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.cbf')
            os.remove('./testcollect/pilatus300k/test1_00001.cbf')
            # os.remove('./testcollect/pilatus300k/test1_00002.cbf')
            os.remove('./testcollect/pilatus300k/test1_00003.cbf')
            # os.remove('./testcollect/pilatus300k/test1_00004.cbf')
            os.remove('./testcollect/pilatus300k/test1_00005.cbf')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_test_file_withpostrun_cbf_pilatus300k_wait(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00000.cbf')
                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00001.cbf')
                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00002.cbf')

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 5)

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()

                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00003.cbf')
                shutil.copy2('test/files/test_file.cbf',
                             './testcollect/pilatus300k/test1_00004.cbf')

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()

                os.remove('./testcollect/pilatus300k/test1_00000.cbf')
                os.remove('./testcollect/pilatus300k/test1_00001.cbf')
                os.remove('./testcollect/pilatus300k/test1_00002.cbf')
                os.remove('./testcollect/pilatus300k/test1_00003.cbf')
                os.remove('./testcollect/pilatus300k/test1_00004.cbf')
                os.remove(filename)

        finally:
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_test_file_withpostrun_cbf_pilatus300k_missing(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        dircreated = False
        try:
            if not os.path.exists("./testcollect/pilatus300k"):
                os.makedirs("./testcollect/pilatus300k")
                dircreated = True

            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf',
                         './testcollect/pilatus300k/test1_00002.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                col = det.create_group("collection", "NXcollection")
                postrun = col.create_field("postrun", "string")
                postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                self.assertEqual(len(svl), 3)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")
                # if '-r' not in cmd:
                #      os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./testcollect/pilatus300k/test1_00000.cbf')
            os.remove('./testcollect/pilatus300k/test1_00001.cbf')
            os.remove('./testcollect/pilatus300k/test1_00002.cbf')
            if dircreated:
                shutil.rmtree("./testcollect")

    def test_test_file_withpostrun_raw(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    with open("rawtest1_%05d.dat" % i, "w") as fl:
                        attrs[k][0][i].tofile(fl)
                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    col = det.create_group("collection", "NXcollection")
                    postrun = col.create_field("postrun", "string")
                    postrun.write("rawtest1_%05d.dat:0:5")
                    atts = postrun.attributes
                    atts.create("fielddtype", "string").write(attrs[k][2])
                    atts.create("fieldshape", "string").write(
                        json.dumps(attrs[k][0].shape[1:]))
                    nxsfile.close()

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = cmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    self.assertEqual(len(svl), 2)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    self.assertTrue('data' not in det.names())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                for i in range(6):
                    os.remove("rawtest1_%05d.dat" % i)

    def test_test_file_withpostrun_h5(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    fl = filewriter.create_file("h5test1_%05d.h5" % i,
                                                overwrite=True)
                    rt = fl.root()
                    shp = attrs[k][0][i].shape
                    data = rt.create_field("data", attrs[k][2], shp, shp)
                    data.write(attrs[k][0][i])
                    data.close()
                    fl.close()
                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    col = det.create_group("collection", "NXcollection")
                    postrun = col.create_field("postrun", "string")
                    postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = cmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    self.assertEqual(len(svl), 2)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    self.assertTrue('data' not in det.names())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                pass
                for i in range(6):
                    os.remove("h5test1_%05d.h5" % i)

    def test_test_file_parameters_tif(self):
        """ test nxsconfig append file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_%05d.tif:0:5'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append --test  %s %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r -s %s --input-files %s --path %s'
             % (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file0.tif', './test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif', './test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif', './test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif', './test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif', './test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif', './test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.tif')
            os.remove('./test1_00001.tif')
            os.remove('./test1_00002.tif')
            os.remove('./test1_00003.tif')
            os.remove('./test1_00004.tif')
            os.remove('./test1_00005.tif')

    def test_test_file_parameters_tif_list(self):
        """ test nxsconfig test file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_00000.tif,test1_00001.tif,test1_00002.tif,' \
            'test1_00003.tif,test1_00004.tif,test1_00005.tif'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append --test  %s %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r -s %s --input-files %s --path %s'
             % (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file0.tif', './test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif', './test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif', './test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif', './test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif', './test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif', './test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.tif')
            os.remove('./test1_00001.tif')
            os.remove('./test1_00002.tif')
            os.remove('./test1_00003.tif')
            os.remove('./test1_00004.tif')
            os.remove('./test1_00005.tif')

    def test_test_file_parameters_tif_list_sep(self):
        """ test nxsconfig test file with a tif postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_00000.tif:test1_00001.tif:test1_00002.tif:' \
            'test1_00003.tif:test1_00004.tif:test1_00005.tif'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append --test  %s %s -i %s -p %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s %s -i %s --path %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r %s -i %s -p %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r %s -i %s --path %s --separator :'
             % (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -s %s --input-files %s -p %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -s %s --input-files %s --path %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r -s %s --input-files %s -p %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r -s %s --input-files %s --path %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file0.tif', './test1_00000.tif')
            shutil.copy2('test/files/test_file1.tif', './test1_00001.tif')
            shutil.copy2('test/files/test_file2.tif', './test1_00002.tif')
            shutil.copy2('test/files/test_file3.tif', './test1_00003.tif')
            shutil.copy2('test/files/test_file4.tif', './test1_00004.tif')
            shutil.copy2('test/files/test_file5.tif', './test1_00005.tif')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.tif:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.tif:0:5']")
                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.tif')
            os.remove('./test1_00001.tif')
            os.remove('./test1_00002.tif')
            os.remove('./test1_00003.tif')
            os.remove('./test1_00004.tif')
            os.remove('./test1_00005.tif')

    def test_test_file_parameters_cbf(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_%05d.cbf:0:5'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append --test  %s %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r -s %s --input-files %s --path %s'
             % (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file.cbf', './test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.cbf')
            os.remove('./test1_00001.cbf')
            os.remove('./test1_00002.cbf')
            os.remove('./test1_00003.cbf')
            os.remove('./test1_00004.cbf')
            os.remove('./test1_00005.cbf')

    def test_test_file_parameters_cbf_list(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_00000.cbf,test1_00001.cbf,test1_00002.cbf,' \
            'test1_00003.cbf,test1_00004.cbf,test1_00005.cbf'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append --test  %s %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r %s -i %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r %s -i %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -s %s --input-files %s --path %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r -s %s --input-files %s -p %s' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r -s %s --input-files %s --path %s'
             % (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file.cbf', './test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.cbf')
            os.remove('./test1_00001.cbf')
            os.remove('./test1_00002.cbf')
            os.remove('./test1_00003.cbf')
            os.remove('./test1_00004.cbf')
            os.remove('./test1_00005.cbf')

    def test_test_file_parameters_cbf_list_sep(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        ifiles = 'test1_00000.cbf:test1_00001.cbf:test1_00002.cbf:' \
            'test1_00003.cbf:test1_00004.cbf:test1_00005.cbf'
        path = '/entry12345/instrument/pilatus300k/data'
        commands = [
            ('nxscollect append --test  %s %s -i %s -p %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s %s -i %s --path %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r %s -i %s -p %s --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r %s -i %s --path %s --separator :'
             % (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -s %s --input-files %s -p %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -s %s --input-files %s --path %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test  %s -r -s %s --input-files %s -p %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
            ('nxscollect append --test %s -r -s %s --input-files %s --path %s'
             ' --separator :' %
             (filename, self.flags, ifiles, path)).split(),
        ]

        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule

        try:
            shutil.copy2('test/files/test_file.cbf', './test1_00000.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00001.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00002.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00003.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00004.cbf')
            shutil.copy2('test/files/test_file.cbf', './test1_00005.cbf')
            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("test1_%05d.cbf:0:5")
                nxsfile.close()

                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = cmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                # er =
                mystderr.getvalue()

                # self.assertEqual('', er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                self.assertEqual(len(svl), 2)
                self.assertTrue(
                    svl[0],
                    "populate: /entry12345:NXentry/instrument:NXinstrument/"
                    "pilatus300k:NXdetector/data with ['test1_%05d.cbf:0:5']")

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                self.assertTrue('data' not in det.names())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove('./test1_00000.cbf')
            os.remove('./test1_00001.cbf')
            os.remove('./test1_00002.cbf')
            os.remove('./test1_00003.cbf')
            os.remove('./test1_00004.cbf')
            os.remove('./test1_00005.cbf')

    def test_test_file_parameters_raw(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test   %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    with open("rawtest1_%05d.dat" % i, "w") as fl:
                        attrs[k][0][i].tofile(fl)
                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    # to be created
                    # ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    nxsfile.close()
                    pcmd = cmd
                    pcmd.extend(["-i", "rawtest1_%05d.dat:0:5"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])
                    pcmd.extend(
                        ["--shape", json.dumps(attrs[k][0].shape[1:])])
                    pcmd.extend(
                        ["--dtype", attrs[k][2]])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    # print(svl)
                    self.assertEqual(len(svl), 2)

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    self.assertTrue('instrument' not in entry.names())
                    # ins = entry.open("instrument")
                    # det = ins.open("pilatus300k")
                    # dt = det.open("data")
                    nxsfile.close()
                    os.remove(filename)

            finally:
                for i in range(6):
                    os.remove("rawtest1_%05d.dat" % i)

    def test_test_file_parameters_nxs(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    fl = filewriter.create_file("h5test1_%05d.nxs" % i,
                                                overwrite=True)
                    rt = fl.root()

                    at = rt.attributes.create("default", "string")
                    at.write("entry12345")
                    at.close()

                    entry = rt.create_group("entry12345", "NXentry")
                    at = entry.attributes.create("default", "string")
                    at.write("data")
                    at.close()

                    dt = entry.create_group("data", "NXdata")
                    at = dt.attributes.create("signal", "string")
                    at.write("data")
                    at.close()

                    shp = attrs[k][0][i].shape
                    data = dt.create_field("data", attrs[k][2], shp, shp)
                    data.write(attrs[k][0][i])
                    data.close()

                    dt.close()
                    entry.close()
                    fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(["-i", "h5test1_%05d.nxs:0:5"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 2:
                        print(svl)
                    self.assertEqual(len(svl), 2)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    self.assertTrue('pilatus300k' not in ins.names())
                    # det = ins.open("pilatus300k")
                    # dt = det.open("data")
                    nxsfile.close()
                    os.remove(filename)

            finally:
                pass
                for i in range(6):
                    os.remove("h5test1_%05d.nxs" % i)

    def test_test_file_parameters_nxs_path(self):
        """ test nxsconfig test file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
            "int8": [12, "NX_INT8", "int8", (1,)],
            "int16": [-123, "NX_INT16", "int16", (1,)],
            "int32": [12345, "NX_INT32", "int32", (1,)],
            "int64": [-12345, "NX_INT64", "int64", (1,)],
            "uint": [123, "NX_UINT", "uint64", (1,)],
            "uint8": [12, "NX_UINT8", "uint8", (1,)],
            "uint16": [123, "NX_UINT16", "uint16", (1,)],
            "uint32": [12345, "NX_UINT32", "uint32", (1,)],
            "uint64": [12345, "NX_UINT64", "uint64", (1,)],
            "float": [-12.345, "NX_FLOAT", "float64", (1,), 1.e-14],
            "number": [-12.345e+2, "NX_NUMBER", "float64", (1,), 1.e-14],
            "float32": [-12.345e-1, "NX_FLOAT32", "float32", (1,), 1.e-5],
            "float64": [-12.345, "NX_FLOAT64", "float64", (1,), 1.e-14],
        }

        commands = [
            ('nxscollect append --test  %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test  %s -r -s %s' %
             (filename, self.flags)).split(),
            ('nxscollect append --test %s -r -s %s' %
             (filename, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[[attrs[k][0] * self.__rnd.randint(0, 3)
                   for c in range(mlen[1])]
                  for i in range(mlen[0])]
                 for _ in range(6)],
                dtype=attrs[k][2]
                )
            try:
                for i in range(6):
                    fl = filewriter.create_file("h5test1_%05d.nxs" % i,
                                                overwrite=True)
                    rt = fl.root()
                    entry = rt.create_group("entry345", "NXentry")

                    dt = entry.create_group("data", "NXdata")

                    shp = attrs[k][0][i].shape
                    data = dt.create_field("data", attrs[k][2], shp, shp)
                    data.write(attrs[k][0][i])
                    data.close()

                    dt.close()
                    entry.close()
                    fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(
                        ["-i", "h5test1_%05d.nxs://entry345/data/data:0:5"])
                    pcmd.extend(
                        ["-p", '/entry12345/instrument/pilatus300k/data'])

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 2:
                        print(svl)
                    self.assertEqual(len(svl), 2)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.nxs:0:5']")

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    self.assertTrue('pilatus300k' not in ins.names())
                    # det = ins.open("pilatus300k")
                    # dt = det.open("data")
                    nxsfile.close()
                    os.remove(filename)

            finally:
                pass
                for i in range(6):
                    os.remove("h5test1_%05d.nxs" % i)

    def test_link_external_nxs(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
        }

        commands = [
            ('nxscollect link %s' % (self.flags)).split(),
            ('nxscollect link -r %s' % (self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[attrs[k][0] * self.__rnd.randint(0, 3)
                  for c in range(mlen[1])]
                 for i in range(mlen[0])],
                dtype=attrs[k][2]
                )
            try:
                fl = filewriter.create_file("h5test1_00001.nxs",
                                            overwrite=True)
                rt = fl.root()

                entry = rt.create_group("entry345", "NXentry")
                dt = entry.create_group("data", "NXdata")
                shp = attrs[k][0].shape
                data = dt.create_field("data", attrs[k][2], shp, shp)
                data.write(attrs[k][0])
                data.close()

                dt.close()
                entry.close()
                fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(
                        ['%s://entry12345/instrument/pilatus300k:NXdetector'
                         % filename])
                    pcmd.extend(["--target",
                                 "h5test1_00001.nxs://entry345/data/data"])
                    pcmd.extend(["--name", "data"])

                    # print(pcmd)
                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 2:
                        print(svl)
                    self.assertEqual(len(svl), 2)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")
                    # print(svl)
                    self.assertTrue(svl[0].startswith('link: '))
                    self.assertTrue('h5test1_00001.nxs://' in svl[0])

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    fimage = attrs[k][0]
                    image = buffer[:, :]
                    self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                os.remove("h5test1_00001.nxs")

    def test_link_external_nxs_noname(self):
        """ test nxsconfig append file with a cbf postrun field
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))

        filename = 'testcollect.nxs'
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
        }

        commands = [
            ('nxscollect link %s' % (self.flags)).split(),
            ('nxscollect link -r %s' % (self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]

            attrs[k][0] = np.array(
                [[attrs[k][0] * self.__rnd.randint(0, 3)
                  for c in range(mlen[1])]
                 for i in range(mlen[0])],
                dtype=attrs[k][2]
                )
            try:
                fl = filewriter.create_file("h5test1_00001.nxs",
                                            overwrite=True)
                rt = fl.root()

                entry = rt.create_group("entry345", "NXentry")
                dt = entry.create_group("data", "NXdata")
                shp = attrs[k][0].shape
                data = dt.create_field("data", attrs[k][2], shp, shp)
                data.write(attrs[k][0])
                data.close()

                dt.close()
                entry.close()
                fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(
                        ['%s://entry12345/instrument/pilatus300k:NXdetector'
                         % filename])
                    pcmd.extend(["--target",
                                 "h5test1_00001.nxs://entry345/data/data"])

                    # print(pcmd)
                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertEqual('', er)
                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 2:
                        print(svl)
                    self.assertEqual(len(svl), 2)
                    self.assertTrue(
                        svl[0],
                        "populate: /entry12345:NXentry/"
                        "instrument:NXinstrument/pilatus300k:NXdetector"
                        "/data with ['test1_%05d.cbf:0:5']")
                    print(svl)
                    self.assertTrue(svl[0].startswith('link: '))
                    self.assertTrue('h5test1_00001.nxs://' in svl[0])

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    fimage = attrs[k][0]
                    image = buffer[:, :]
                    self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                os.remove("h5test1_00001.nxs")

    def test_vds_single(self):
        """ test nxscollect vds
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))
        if self.writer == "h5py":
            import nxstools.h5pywriter as H5PYWriter
            if not H5PYWriter.is_vds_supported():
                print("VDS not supported: skipping the test")
                return

        filename = 'testcollect.nxs'
        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        attrs = {
            "int": [-123, "NX_INT", "int64", (1,)],
        }

        commands = [
            ('nxscollect vds %s' % (self.flags)).split(),
            ('nxscollect vds -r %s' % (self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        for k in attrs.keys():
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]
            print(mlen)
            attrs[k][0] = np.array(
                [[attrs[k][0] * self.__rnd.randint(0, 3)
                  for c in range(mlen[1])]
                 for i in range(mlen[0])],
                dtype=attrs[k][2]
                )
            try:
                fl = filewriter.create_file("h5test1_00001.nxs",
                                            overwrite=True)
                rt = fl.root()

                entry = rt.create_group("entry345", "NXentry")
                dt = entry.create_group("data", "NXdata")
                shp = attrs[k][0].shape
                data = dt.create_field("data", attrs[k][2], shp, shp)
                data.write(attrs[k][0])
                data.close()

                dt.close()
                entry.close()
                fl.close()

                for cmd in commands:
                    nxsfile = filewriter.create_file(
                        filename, overwrite=True)
                    rt = nxsfile.root()
                    entry = rt.create_group("entry12345", "NXentry")
                    ins = entry.create_group("instrument", "NXinstrument")
                    # det = ins.create_group("pilatus300k", "NXdetector")
                    entry.create_group("data", "NXdata")
                    # col = det.create_group("collection", "NXcollection")
                    # postrun = col.create_field("postrun", "string")
                    # postrun.write("h5test1_%05d.h5:0:5")
                    nxsfile.close()

                    pcmd = cmd
                    pcmd.extend(
                        ['%s://entry12345/instrument/pilatus300k:NXdetector/'
                         'data' % filename])
                    pcmd.extend(["--target-fields",
                                 "h5test1_00001.nxs://entry345/data/data"])
                    pcmd.extend(["--shape",
                                 "%s" % ','.join([str(s) for s in shp])])
                    pcmd.extend(["--shapes",
                                 "%s" % ','.join([str(s) for s in shp])])
                    pcmd.extend(["--dtype",
                                 "%s" % attrs[k][2]])

                    # print(pcmd)
                    old_stdout = sys.stdout
                    old_stderr = sys.stderr
                    sys.stdout = mystdout = StringIO()
                    sys.stderr = mystderr = StringIO()
                    old_argv = sys.argv
                    sys.argv = pcmd
                    nxscollect.main()

                    sys.argv = old_argv
                    sys.stdout = old_stdout
                    sys.stderr = old_stderr
                    vl = mystdout.getvalue()
                    er = mystderr.getvalue()

                    self.assertTrue(vl)
                    svl = vl.split("\n")
                    if len(svl) != 2:
                        print(svl)
                    print(svl)
                    self.assertEqual(len(svl), 2)
                    self.assertEqual('', er)
                    self.assertTrue(svl[0].startswith('vds: '))
                    self.assertTrue('h5test1_00001.nxs' in svl[0])

                    if '-r' not in cmd:
                        os.remove("%s.__nxscollect_old__" % filename)
                    nxsfile = filewriter.open_file(filename, readonly=True)
                    rt = nxsfile.root()
                    entry = rt.open("entry12345")
                    ins = entry.open("instrument")
                    det = ins.open("pilatus300k")
                    dt = det.open("data")
                    buffer = dt.read()
                    self.assertEqual(buffer.shape, attrs[k][0].shape)
                    fimage = attrs[k][0]
                    image = buffer[:, :]
                    self.assertTrue((image == fimage).all())
                    nxsfile.close()
                    os.remove(filename)

            finally:
                os.remove("h5test1_00001.nxs")

    def test_vds_concatinate(self):
        """ test nxscollect vds
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))
        if self.writer == "h5py":
            import nxstools.h5pywriter as H5PYWriter
            if not H5PYWriter.is_vds_supported():
                print("VDS not supported: skipping the test")
                return

        filename = 'testcollect.nxs'
        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        attrs = {
            "int1": [-123, "NX_INT", "int64", (1,)],
            "int2": [12, "NX_INT", "int64", (1,)],
            "int3": [52, "NX_INT", "int64", (1,)],
        }

        commands = [
            ('nxscollect vds %s' % (self.flags)).split(),
            ('nxscollect vds -r %s' % (self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        try:
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]
            # print(mlen)
            for ii, k in enumerate(sorted(attrs.keys())):
                attrs[k][0] = np.array(
                    [[attrs[k][0] * self.__rnd.randint(0, 3)
                      for c in range(mlen[1])]
                     for i in range(mlen[0])],
                    dtype=attrs[k][2]
                )
                fl = filewriter.create_file("eh5test1_%05d.nxs" % (ii + 1),
                                            overwrite=True)
                rt = fl.root()

                entry = rt.create_group("entry345", "NXentry")
                dt = entry.create_group("data", "NXdata")
                shp = attrs[k][0].shape
                data = dt.create_field("data", attrs[k][2], shp, shp)
                data.write(attrs[k][0])
                data.close()

                dt.close()
                entry.close()
                fl.close()

            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                # det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("h5test1_%05d.h5:0:5")
                nxsfile.close()

                pcmd = cmd
                pcmd.extend(
                    ['%s://entry12345/instrument/pilatus300k:NXdetector/'
                     'data' % filename])
                tfields = ",".join(
                    ["eh5test1_%05d.nxs://entry345/data/data" %
                     (i + 1) for i in range(len(attrs))])
                pcmd.extend(["--target-fields", "%s" % tfields])
                pcmd.extend(["--shape",
                             "%s,%s" % (shp[0] * len(attrs), shp[1])])
                pcmd.extend(["--dtype",
                             "%s" % attrs["int1"][2]])
                tshapes = ";".join(
                    [("%s,%s" % (shp[0], shp[1])) for _ in range(len(attrs))])
                pcmd.extend(["--shapes", "%s" % tshapes])
                offsets = ";".join(
                    [('%s,%s' % (shp[0] * i, 0)) for i in range(len(attrs))])

                pcmd.extend(["--offsets", "%s" % offsets])
                # print(pcmd)
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = pcmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                print(svl)
                self.assertEqual(len(svl), 4)
                self.assertEqual('', er)
                self.assertTrue(svl[0].startswith('vds: '))
                self.assertTrue('h5test1_0000' in svl[0])

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                ibuffer = dt.read()
                # print(attrs["int1"][0].shape)
                # print(ibuffer.shape)
                tshape = (attrs["int1"][0].shape[0] * len(attrs),
                          attrs["int1"][0].shape[1])

                self.assertEqual(ibuffer.shape, tshape)
                fimage = np.concatenate(
                    (attrs["int1"][0], attrs["int2"][0], attrs["int3"][0]))
                image = ibuffer[:, :]
                self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove("eh5test1_00001.nxs")
            os.remove("eh5test1_00002.nxs")
            os.remove("eh5test1_00003.nxs")

    def test_vds_append(self):
        """ test nxscollect vds
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))
        if self.writer == "h5py":
            import nxstools.h5pywriter as H5PYWriter
            if not H5PYWriter.is_vds_supported():
                print("VDS not supported: skipping the test")
                return

        # filename = 'testcollect.nxs'
        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        attrs = {
            "int1": [-123, "NX_INT", "int64", (1,)],
            "int2": [12, "NX_INT", "int64", (1,)],
            "int3": [52, "NX_INT", "int64", (1,)],
        }

        commands = [
            ('nxscollect vds %s' % (self.flags)).split(),
            ('nxscollect vds -r %s' % (self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        try:
            mlen = [self.__rnd.randint(10, 200),
                    self.__rnd.randint(10, 200)]
            # print(mlen)
            for ii, k in enumerate(sorted(attrs.keys())):
                attrs[k][0] = np.array(
                    [[attrs[k][0] * self.__rnd.randint(0, 3)
                      for c in range(mlen[1])]
                     for i in range(mlen[0])],
                    dtype=attrs[k][2]
                )
                fl = filewriter.create_file("eh5test1_%05d.nxs" % (ii + 1),
                                            overwrite=True)
                rt = fl.root()

                entry = rt.create_group("entry345", "NXentry")
                dt = entry.create_group("data", "NXdata")
                shp = attrs[k][0].shape
                data = dt.create_field("data", attrs[k][2], shp, shp)
                data.write(attrs[k][0])
                data.close()

                dt.close()
                entry.close()
                fl.close()

            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                # det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("h5test1_%05d.h5:0:5")
                nxsfile.close()

                pcmd = cmd
                pcmd.extend(
                    ['%s://entry12345/instrument/pilatus300k:NXdetector/'
                     'data' % filename])
                tfields = ",".join(
                    ["eh5test1_%05d.nxs://entry345/data/data" %
                     (i + 1) for i in range(len(attrs))])
                pcmd.extend(["--target-fields", "%s" % tfields])
                pcmd.extend(["--shape",
                             "%s,%s,%s" % (len(attrs), shp[0], shp[1])])
                pcmd.extend(["--dtype",
                             "%s" % attrs["int1"][2]])
                tshapes = ";".join(
                    [("1,%s,%s" % (shp[0], shp[1]))
                     for _ in range(len(attrs))])
                pcmd.extend(["--shapes", "%s" % tshapes])
                offsets = ";".join(
                    [('%s,0,0' % i) for i in range(len(attrs))])

                pcmd.extend(["--offsets", "%s" % offsets])
                # print(pcmd)
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = pcmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                # print(er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                print(svl)
                self.assertEqual(len(svl), 4)
                self.assertEqual('', er)
                self.assertTrue(svl[0].startswith('vds: '))
                self.assertTrue('h5test1_0000' in svl[0])

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                ibuffer = dt.read()
                tshape = (len(attrs),
                          attrs["int1"][0].shape[0],
                          attrs["int1"][0].shape[1])

                self.assertEqual(ibuffer.shape, tshape)
                image = ibuffer[:, :, :]
                self.assertTrue((image[0, :, :] == attrs["int1"][0]).all())
                self.assertTrue((image[1, :, :] == attrs["int2"][0]).all())
                self.assertTrue((image[2, :, :] == attrs["int3"][0]).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove("eh5test1_00001.nxs")
            os.remove("eh5test1_00002.nxs")
            os.remove("eh5test1_00003.nxs")

    def test_vds_append_gap(self):
        """ test nxscollect vds
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))
        if self.writer == "h5py":
            import nxstools.h5pywriter as H5PYWriter
            if not H5PYWriter.is_vds_supported():
                print("VDS not supported: skipping the test")
                return

        # filename = 'testcollect.nxs'
        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        attrs = {
            "int1": [-123, "NX_INT", "int64", (1,)],
            "int2": [12, "NX_INT", "int64", (1,)],
            "int3": [52, "NX_INT", "int64", (1,)],
        }

        fval = 1
        commands = [
            ('nxscollect vds -f %s %s' % (fval, self.flags)).split(),
            ('nxscollect vds -r -f %s %s' % (fval, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        try:
            mlen = [self.__rnd.randint(3, 10),
                    self.__rnd.randint(3, 10),
                    self.__rnd.randint(3, 10)]
            gap = 3
            imnr = len(list(attrs.keys()))
            lsh = list(mlen)
            lsh[1] = imnr * lsh[1] + (imnr - 1) * gap
            lsh = tuple(lsh)

            garr = np.array(
                [[[fval
                   for j in range(mlen[2])]
                  for c in range(gap)]
                 for i in range(mlen[0])],
                dtype=attrs['int1'][2]
            )

            # print(mlen)
            for ii, k in enumerate(sorted(attrs.keys())):
                attrs[k][0] = np.array(
                    [[[attrs[k][0] * self.__rnd.randint(0, 3)
                       for j in range(mlen[2])]
                      for c in range(mlen[1])]
                     for i in range(mlen[0])],
                    dtype=attrs[k][2]
                )
                fl = filewriter.create_file(
                    "eh5test1_%05d.nxs" % (ii + 1),
                    overwrite=True)
                rt = fl.root()

                entry = rt.create_group("entry345", "NXentry")
                dt = entry.create_group("data", "NXdata")
                shp = attrs[k][0].shape
                chk = list(shp)
                chk[0] = 1
                chk = tuple(chk)
                data = dt.create_field("data", attrs[k][2], shp, chk)
                data.write(attrs[k][0])
                data.close()

                dt.close()
                entry.close()
                fl.close()

            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                # det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("h5test1_%05d.h5:0:5")
                nxsfile.close()

                pcmd = cmd
                pcmd.extend(
                    ['%s://entry12345/instrument/pilatus300k:NXdetector/'
                     'data' % filename])
                tfields = ",".join(
                    ["eh5test1_%05d.nxs://entry345/data/data" %
                     (i + 1) for i in range(len(attrs))])
                pcmd.extend(["--target-fields", "%s" % tfields])
                pcmd.extend(["--shape",
                             "%s,%s,%s" % (lsh[0], lsh[1], lsh[2])])
                pcmd.extend(["--dtype",
                             "%s" % attrs["int1"][2]])
                tshapes = ";".join(
                    [("%s,%s,%s" % (shp[0], shp[1], shp[2]))
                     for _ in range(len(attrs))])
                pcmd.extend(["--shapes", "%s" % tshapes])
                offsets = ";".join(
                    [('0,%s,0' % (i*(mlen[1] + gap)))
                     for i in range(len(attrs))])

                pcmd.extend(["--offsets", "%s" % offsets])
                # print(pcmd)
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = pcmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                # print(er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                print(svl)
                self.assertEqual(len(svl), 4)
                self.assertEqual('', er)
                self.assertTrue(svl[0].startswith('vds: '))
                self.assertTrue('h5test1_0000' in svl[0])

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                ibuffer = dt.read()

                self.assertEqual(ibuffer.shape, lsh)
                image = ibuffer[:, :, :]
                fimage = np.concatenate(
                    (attrs["int1"][0], garr,
                     attrs["int2"][0], garr,
                     attrs["int3"][0]), 1)
                self.assertTrue((image == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove("eh5test1_00001.nxs")
            os.remove("eh5test1_00002.nxs")
            os.remove("eh5test1_00003.nxs")

    def test_vds_append_inter(self):
        """ test nxscollect vds
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))
        if self.writer == "h5py":
            import nxstools.h5pywriter as H5PYWriter
            if not H5PYWriter.is_vds_supported():
                print("VDS not supported: skipping the test")
                return

        # filename = 'testcollect.nxs'
        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        attrs = {
            "int1": [-123, "NX_FLOAT", "float64", (1,)],
            "int2": [12, "NX_FLOAT", "float64", (1,)],
            "int3": [52, "NX_FLOAT", "float64", (1,)],
            "int4": [22, "NX_FLOAT", "float64", (1,)],
        }

        commands = [
            ('nxscollect vds  %s' % (self.flags)).split(),
            ('nxscollect vds -r %s' % (self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        try:
            mlen = [self.__rnd.randint(3, 10),
                    self.__rnd.randint(3, 10),
                    self.__rnd.randint(3, 10)]
            # mlen = [3,3,4]
            imnr = len(list(attrs.keys()))
            lsh = list(mlen)
            lsh[0] = imnr * lsh[0]
            lsh = tuple(lsh)

            # print(mlen)
            for ii, k in enumerate(sorted(attrs.keys())):
                attrs[k][0] = np.array(
                    [[[attrs[k][0] * self.__rnd.randint(0, 3)
                       for j in range(mlen[2])]
                      for c in range(mlen[1])]
                     for i in range(mlen[0])],
                    dtype=attrs[k][2]
                )
                fl = filewriter.create_file(
                    "eh5test1_%05d.nxs" % (ii + 1),
                    overwrite=True)
                rt = fl.root()

                entry = rt.create_group("entry345", "NXentry")
                dt = entry.create_group("data", "NXdata")
                shp = attrs[k][0].shape
                chk = list(shp)
                chk[0] = 1
                chk = tuple(chk)
                data = dt.create_field("data", attrs[k][2], shp, chk)
                data.write(attrs[k][0])
                data.close()

                dt.close()
                entry.close()
                fl.close()

            fimage = np.array(
                [[[0
                   for j in range(mlen[2])]
                  for c in range(mlen[1])]
                 for i in range(4 * mlen[0])],
                dtype=attrs["int1"][2]
                )
            fimage[0:lsh[0]:4, :, :] = attrs["int1"][0]
            fimage[1:lsh[0]:4, :, :] = attrs["int2"][0]
            fimage[2:lsh[0]:4, :, :] = attrs["int3"][0]
            fimage[3:lsh[0]:4, :, :] = attrs["int4"][0]

            for cmd in commands:
                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                # det = ins.create_group("pilatus300k", "NXdetector")
                entry.create_group("data", "NXdata")
                # col = det.create_group("collection", "NXcollection")
                # postrun = col.create_field("postrun", "string")
                # postrun.write("h5test1_%05d.h5:0:5")
                nxsfile.close()

                pcmd = cmd
                pcmd.extend(
                    ['%s://entry12345/instrument/pilatus300k:NXdetector/'
                     'data' % filename])
                tfields = "eh5test1_%05d.nxs://entry345/data/data:1:4"
                pcmd.extend(["--target-fields", "%s" % tfields])
                pcmd.extend(["--shape",
                             "%s,%s,%s" % (lsh[0], lsh[1], lsh[2])])
                pcmd.extend(["--dtype",
                             "%s" % attrs["int1"][2]])
                tshapes = ";".join(
                    [("%s,%s,%s" % (shp[0], shp[1], shp[2]))
                     for _ in range(len(attrs))])
                pcmd.extend(["--shapes", "%s" % tshapes])
                offsets = ";".join(
                    [('%s,,' % i) for i in range(imnr)])
                pcmd.extend(["--offsets", "%s" % offsets])
                strides = ";".join(
                    [('4,,') for _ in range(imnr)])
                pcmd.extend(["--strides", "%s" % strides])

                # print(pcmd)
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = pcmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                # print(er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 2:
                    print(svl)
                print(svl)
                self.assertEqual(len(svl), 5)
                self.assertEqual('', er)
                self.assertTrue(svl[0].startswith('vds: '))
                self.assertTrue('h5test1_0000' in svl[0])

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dt = det.open("data")
                ibuffer = dt.read()

                self.assertEqual(ibuffer.shape, lsh)
                self.assertTrue((ibuffer == fimage).all())
                nxsfile.close()
                os.remove(filename)

        finally:
            os.remove("eh5test1_00001.nxs")
            os.remove("eh5test1_00002.nxs")
            os.remove("eh5test1_00003.nxs")
            os.remove("eh5test1_00004.nxs")

    def test_vds_append_unlimited(self):
        """ test nxscollect vds
        """
        fun = sys._getframe().f_code.co_name
        print("Run: %s.%s() " % (self.__class__.__name__, fun))
        if self.writer == "h5py":
            import nxstools.h5pywriter as H5PYWriter
            if not H5PYWriter.is_unlimited_vds_supported():
                print("VDS not supported: skipping the test")
                return

        # filename = 'testcollect.nxs'
        filename = '%s/%s%s.nxs' % (os.getcwd(),
                                    self.__class__.__name__, fun)
        attrs = {
            "int1": [-123, "NX_INT", "int64", (1,), 3],
            "int2": [12, "NX_INT", "int64", (1,), -2],
        }

        fval = 1
        commands = [
            ('nxscollect vds -f %s %s' % (fval, self.flags)).split(),
            # second command hangs on reading vds
            ('nxscollect vds -r -f %s %s' % (fval, self.flags)).split(),
        ]
        wrmodule = WRITERS[self.writer]
        filewriter.writer = wrmodule
        mlen = [self.__rnd.randint(3, 10),
                self.__rnd.randint(3, 10),
                self.__rnd.randint(3, 10)]
        imnr = len(list(attrs.keys()))
        lsh = list(mlen)
        lsh[1] = imnr * lsh[1]
        lsh = tuple(lsh)
        for ck, cmd in enumerate(commands):
            filename = '%s/%s%s_%s.nxs' % (
                os.getcwd(),
                self.__class__.__name__, fun, ck)

            filenames = []
            try:
                for ii, k in enumerate(sorted(attrs.keys())):
                    attrs[k][0] = np.array(
                        [[[self.__rnd.randint(0, 3)
                           for j in range(mlen[2])]
                          for c in range(mlen[1])]
                         for i in range(mlen[0])],
                        dtype=attrs[k][2]
                    )
                    filenames.append(
                        "eh5test1_%05d_%s.nxs" % (ii + 1, ck))
                    fl = filewriter.create_file(filenames[-1],
                                                overwrite=True)
                    rt = fl.root()

                    entry = rt.create_group("entry345", "NXentry")
                    dt = entry.create_group("data", "NXdata")
                    shp = attrs[k][0].shape
                    chk = list(shp)
                    chk[0] = 1
                    chk = tuple(chk)
                    data = dt.create_field("data", attrs[k][2], shp, chk)
                    data.write(attrs[k][0])
                    data.close()
                    dt.close()
                    entry.close()
                    fl.close()

                nxsfile = filewriter.create_file(
                    filename, overwrite=True)
                rt = nxsfile.root()
                entry = rt.create_group("entry12345", "NXentry")
                ins = entry.create_group("instrument", "NXinstrument")
                dt = entry.create_group("data", "NXdata")
                dt.close()
                ins.close()
                entry.close()
                rt.close()
                nxsfile.close()

                pcmd = cmd
                pcmd.extend(
                    ['%s://entry12345/instrument/pilatus300k:NXdetector/'
                     'data' % filename])
                tfields = ",".join(
                    ["%s://entry345/data/data" %
                     (filenames[i]) for i in range(len(attrs))])
                pcmd.extend(["--target-fields", "%s" % tfields])
                pcmd.extend(["--shape",
                             "%s,%s,%s" % (lsh[0], lsh[1], lsh[2])])
                pcmd.extend(["--dtype",
                             "%s" % attrs["int1"][2]])
                tshapes = ":".join(
                    [(",%s," % (shp[1]))
                     for _ in range(len(attrs))])
                pcmd.extend(["--shapes", "%s" % tshapes])
                offsets = ":".join(
                    [('0,%s,0' % (i*(mlen[1])))
                     for i in range(len(attrs))])
                pcmd.extend(["--offsets", "%s" % offsets])
                counts = ":".join(
                    ['U,,' for i in range(len(attrs))])
                pcmd.extend(["--counts", "%s" % counts])

                # print(pcmd)
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = mystdout = StringIO()
                sys.stderr = mystderr = StringIO()
                old_argv = sys.argv
                sys.argv = pcmd
                nxscollect.main()

                sys.argv = old_argv
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                vl = mystdout.getvalue()
                er = mystderr.getvalue()

                # print(er)
                self.assertTrue(vl)
                svl = vl.split("\n")
                if len(svl) != 3:
                    print(svl)
                print(svl)
                self.assertEqual(len(svl), 3)
                self.assertEqual('', er)
                self.assertTrue(svl[0].startswith('vds: '))
                self.assertTrue('h5test1_0000' in svl[0])

                if '-r' not in cmd:
                    os.remove("%s.__nxscollect_old__" % filename)
                nxsfile = filewriter.open_file(
                    filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dat = det.open("data")

                self.assertEqual(dat.shape, lsh)
                fimage = np.concatenate(
                    (attrs["int1"][0],
                     attrs["int2"][0]), 1)

                # second times read hangs
                # ibuffer = dat.read()

                image = dat[:, :, :]
                self.assertTrue(np.array_equal(image, fimage))

                dat.close()
                ins.close()
                entry.close()
                rt.close()
                nxsfile.close()

                for ii, k in enumerate(sorted(attrs.keys())):
                    attrs[k][4] = np.array(
                        [[[self.__rnd.randint(0, 30)
                           for j in range(mlen[2])]
                          for c in range(mlen[1])]
                         for i in range(mlen[0])],
                        dtype=attrs[k][2]
                    )
                    fl = filewriter.open_file(
                        filenames[ii],
                        readonly=False)
                    rt = fl.root()

                    entry = rt.open("entry345")
                    dt = entry.open("data")
                    shp = attrs[k][4].shape
                    chk = list(shp)
                    chk[0] = 1
                    chk = tuple(chk)
                    data = dt.open("data")
                    data.grow(0, shp[0])
                    data[shp[0]:, :, :] = attrs[k][4]
                    data.close()

                    dt.close()
                    entry.close()
                    fl.close()

                nxsfile = filewriter.open_file(filename, readonly=True)
                rt = nxsfile.root()
                entry = rt.open("entry12345")
                ins = entry.open("instrument")
                det = ins.open("pilatus300k")
                dat = det.open("data")
                ibuffer = dat.read()

                self.assertEqual(ibuffer.shape,
                                 (2 * lsh[0], lsh[1], lsh[2]))
                image = ibuffer[:, :, :]
                fimage1 = np.concatenate(
                    (
                        attrs["int1"][0],
                        attrs["int2"][0],
                    ), 1)
                fimage2 = np.concatenate(
                    (
                        attrs["int1"][4],
                        attrs["int2"][4],
                    ), 1)
                fimage = np.concatenate((fimage1, fimage2), 0)
                self.assertTrue((image == fimage).all())
                dat.close()
                ins.close()
                entry.close()
                rt.close()

            finally:
                os.remove(filename)
                for fn in filenames:
                    os.remove(fn)


if __name__ == '__main__':
    unittest.main()
