/*
 * E57FoundationImpl.cpp - implementation of private functions of E57 format
 *   reference implementation.
 *
 * Copyright 2009 - 2010 Kevin Ackley (kackley@gwi.net)
 *
 * Permission is hereby granted, free of charge, to any person or organization
 * obtaining a copy of the software and accompanying documentation covered by
 * this license (the "Software") to use, reproduce, display, distribute,
 * execute, and transmit the Software, and to prepare derivative works of the
 * Software, and to permit third-parties to whom the Software is furnished to
 * do so, all subject to the following:
 *
 * The copyright notices in the Software and this entire statement, including
 * the above license grant, this restriction and the following disclaimer,
 * must be included in all copies of the Software, in whole or in part, and
 * all derivative works of the Software, unless such copies or derivative
 * works are solely in the form of machine-executable object code generated by
 * a source language processor.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
 * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
 * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <cmath>

#include <xercesc/sax2/XMLReaderFactory.hpp>
XERCES_CPP_NAMESPACE_USE

#include "E57FoundationImpl.h"

#include "Decoder.h"
#include "Encoder.h"
#include "E57XmlParser.h"

using namespace e57;
using namespace std;

/// Section types:
#define E57_BLOB_SECTION                0
#define E57_COMPRESSED_VECTOR_SECTION   1

/// Packet types (in a compressed vector section)
#define E57_DATA_PACKET                 1
#define E57_INDEX_PACKET                0
#define E57_EMPTY_PACKET                2

#ifdef E57_BIGENDIAN
void E57FileHeader::swab()
{
    /// Byte swap fields in-place, if CPU is BIG_ENDIAN
    SWAB(&majorVersion);
    SWAB(&minorVersion);
    SWAB(&filePhysicalLength);
    SWAB(&xmlPhysicalOffset);
    SWAB(&xmlLogicalLength);
    SWAB(&pageSize);
};
#endif

#ifdef E57_DEBUG
void E57FileHeader::dump(int indent, std::ostream& os)
{
    os << space(indent) << "fileSignature:      "
       << fileSignature[0] << fileSignature[1] << fileSignature[2] << fileSignature[3]
       << fileSignature[4] << fileSignature[5] << fileSignature[6] << fileSignature[7] << endl;
    os << space(indent) << "majorVersion:       " << majorVersion << endl;
    os << space(indent) << "minorVersion:       " << minorVersion << endl;
    os << space(indent) << "filePhysicalLength: " << filePhysicalLength << endl;
    os << space(indent) << "xmlPhysicalOffset:  " << xmlPhysicalOffset << endl;
    os << space(indent) << "xmlLogicalLength:   " << xmlLogicalLength << endl;
    os << space(indent) << "pageSize:           " << pageSize << endl;
}
#endif

struct BlobSectionHeader {
    uint8_t     sectionId;              // = E57_BLOB_SECTION
    uint8_t     reserved1[7];           // must be zero
    uint64_t    sectionLogicalLength;   // byte length of whole section
#ifdef E57_BIGENDIAN
    void        swab();
#else
    void        swab(){}
#endif
#ifdef E57_DEBUG
    void        dump(int indent = 0, std::ostream& os = std::cout);
#endif
};

#ifdef E57_BIGENDIAN
void BlobSectionHeader::swab()
{
    /// Byte swap fields in-place if CPU is BIG_ENDIAN
    SWAB(&sectionLogicalLength);
};
#endif

#ifdef E57_DEBUG
void BlobSectionHeader::dump(int indent, std::ostream& os)
{
    os << space(indent) << "sectionId:            " << sectionId << endl;
    os << space(indent) << "sectionLogicalLength: " << sectionLogicalLength << endl;
}
#endif

///================================================================
///================================================================
///================================================================

NodeImpl::NodeImpl(weak_ptr<ImageFileImpl> destImageFile)
: destImageFile_(destImageFile),
  isAttached_(false)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);  // does checking for all node type ctors
}

void NodeImpl::checkImageFileOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName) const
{
    /// Throw an exception if destImageFile (destImageFile_) isn't open
    shared_ptr<ImageFileImpl> destImageFile(destImageFile_);
    if (!destImageFile->isOpen())
    {
        throw E57Exception(E57_ERROR_IMAGEFILE_NOT_OPEN,
                           "fileName=" + destImageFile->fileName(),
                           srcFileName,
                           srcLineNumber,
                           srcFunctionName);
    }
}

bool NodeImpl::isRoot() const
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    return parent_.expired();
};

std::shared_ptr<NodeImpl> NodeImpl::parent()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    if (isRoot())
    {
        /// If is root, then has self as parent (by convention)
        return shared_from_this();
    }
    else
    {
        std::shared_ptr<NodeImpl> myParent(parent_);

        return myParent;
    }
}

ustring NodeImpl::pathName() const
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    if (isRoot())
    {
        return("/");
    }
    else
    {
        shared_ptr<NodeImpl> p(parent_);

        if (p->isRoot())
        {
            return("/" + elementName_);
        }
        else
        {
            return(p->pathName() + "/" + elementName_);
        }
    }
}

ustring NodeImpl::relativePathName(shared_ptr<NodeImpl> origin, ustring childPathName) const
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    if (origin == shared_from_this())
        return(childPathName);

    if (isRoot())
        /// Got to top and didn't find origin, must be error
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " childPathName="+childPathName);
    else {
        /// Assemble relativePathName from right to left, recursively
        shared_ptr<NodeImpl> p(parent_);
        if (childPathName == "")
            return(p->relativePathName(origin, elementName_));
        else
            return(p->relativePathName(origin, elementName_ + "/" + childPathName));
    }
}

ustring NodeImpl::elementName() const
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    return elementName_;
}

shared_ptr<ImageFileImpl> NodeImpl::destImageFile()
{
    /// don't checkImageFileOpen
    shared_ptr<ImageFileImpl> imf(destImageFile_);
    return(imf);
}

bool NodeImpl::isAttached() const
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    return isAttached_;
}

void NodeImpl::setAttachedRecursive()
{
    /// Non-terminal node types (Structure, Vector, CompressedVector) will override this virtual function, to mark their children, codecs, prototypes
    isAttached_ = true;
}

ustring NodeImpl::imageFileName() const
{
    /// don't checkImageFileOpen
    shared_ptr<ImageFileImpl> imf(destImageFile_);

    return imf->fileName();
}

void NodeImpl::setParent(shared_ptr<NodeImpl> parent, const ustring& elementName)
{
    /// don't checkImageFileOpen

    /// First check if our parent_ is already set, throw E57_ERROR_ALREADY_HAS_PARENT
    /// The isAttached_ condition is to catch two errors:
    ///    1) if user attempts to use the ImageFile root as a child (e.g. root.set("x", root))
    ///    2) if user attempts to reuse codecs or prototype trees of a CompressedVectorNode
    ///       ??? what if CV not attached yet?
    if (!parent_.expired() || isAttached_) {
        /// ??? does caller do setParent first, so state is not messed up when throw?
        throw E57_EXCEPTION2(E57_ERROR_ALREADY_HAS_PARENT,
                             "this->pathName=" + this->pathName() +
                             " newParent->pathName=" + parent->pathName());
    }

    parent_      = parent;
    elementName_ = elementName;

    /// If parent is attached then we are attached (and all of our children)
    if (parent->isAttached())
        setAttachedRecursive();
}

shared_ptr<NodeImpl> NodeImpl::getRoot()
{
    /// don't checkImageFileOpen
    shared_ptr<NodeImpl> p(shared_from_this());
    while (!p->isRoot())
    {
        p = shared_ptr<NodeImpl>(p->parent_);  //??? check if bad ptr?
    }

    return p;
}

//??? use visitor?
bool NodeImpl::isTypeConstrained()
{
    /// don't checkImageFileOpen
    /// A node is type constrained if any of its parents is an homo VECTOR or COMPRESSED_VECTOR with more than one child
    shared_ptr<NodeImpl> p(shared_from_this());
    while (!p->isRoot()) {
        /// We have a parent since we are not root
        p = shared_ptr<NodeImpl>(p->parent_);  //??? check if bad ptr?

        switch (p->type()) {
            case E57_VECTOR:
                {
                    /// Downcast to shared_ptr<VectorNodeImpl>
                    shared_ptr<VectorNodeImpl> ai(dynamic_pointer_cast<VectorNodeImpl>(p));
                    if (!ai)  // check if failed
                        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName="+p->elementName());

                    /// If homogenous vector and have more than one child, then can't change them
                    if (!ai->allowHeteroChildren() && ai->childCount() > 1)
                        return(true);
                }
                break;
            case E57_COMPRESSED_VECTOR:
                /// Can't make any type changes to CompressedVector prototype.  ??? what if hasn't been written to yet
                return(true);
            default:
                break;
        }
    }
    /// Didn't find any constraining VECTORs or COMPRESSED_VECTORs in path above us, so our type is not constrained.
    return(false);
}

std::shared_ptr<NodeImpl> NodeImpl::get(const ustring& pathName)
{
    /// This is common virtual function for terminal E57 element types: Integer, ScaledInteger, Float, Blob.
    /// The non-terminal types override this virtual function.
    /// Only absolute pathNames make any sense here, because the terminal types can't have children, so relative pathNames are illegal.

    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    /// Parse to determine if pathName is absolute
    bool isRelative;
    vector<ustring> fields;
    shared_ptr<ImageFileImpl> imf(destImageFile_);
    imf->pathNameParse(pathName, isRelative, fields);  // throws if bad pathName

    /// If not an absolute path name, have error
    if (isRelative)
        throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "this->pathName=" + this->pathName() + " pathName=" + pathName);

    /// Find root of the tree
    shared_ptr<NodeImpl> root(shared_from_this()->getRoot());

    /// Check to make sure root node is non-terminal type (otherwise have stack overflow).
    switch (root->type()) {
        case E57_STRUCTURE:
        case E57_VECTOR: //??? COMPRESSED_VECTOR?
            break;
        default:
            throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "this->pathName=" + this->pathName() + " pathName=" + pathName);
    }

    /// Forward call to the non-terminal root node
    return(root->get(pathName));
}

void NodeImpl::set(const ustring& pathName, shared_ptr<NodeImpl> ni, bool autoPathCreate)
{
    /// This is common virtual function for terminal E57 element types: Integer, ScaledInteger, Float, Blob.
    /// The non-terminal types override this virtual function.
    /// Only absolute pathNames make any sense here, because the terminal types can't have children, so relative pathNames are illegal.

    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    /// Parse to determine if pathName is absolute
    bool isRelative;
    vector<ustring> fields;
    shared_ptr<ImageFileImpl> imf(destImageFile_);
    imf->pathNameParse(pathName, isRelative, fields);  // throws if bad pathName

    /// If not an absolute path name, have error
    if (isRelative)
        throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "this->pathName=" + this->pathName() + " pathName=" + pathName);

    /// Find root of the tree
    shared_ptr<NodeImpl> root(shared_from_this()->getRoot());

    /// Check to make sure root node is non-terminal type (otherwise have stack overflow).
    switch (root->type()) {
        case E57_STRUCTURE:
        case E57_VECTOR: //??? COMPRESSED_VECTOR?
            break;
        default:
            throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "this->pathName=" + this->pathName() + " pathName=" + pathName);
    }

    /// Forward call to the non-terminal root node
    root->set(pathName, ni, autoPathCreate);
}

void NodeImpl::set(const std::vector<ustring>& /*fields*/, unsigned /*level*/, std::shared_ptr<NodeImpl> /*ni*/, bool /*autoPathCreate*/)
{
    /// If get here, then tried to call set(fields...) on NodeImpl that wasn't a StructureNodeImpl, so that's an error
    throw E57_EXCEPTION1(E57_ERROR_BAD_PATH_NAME); //???
}

void NodeImpl::checkBuffers(const vector<SourceDestBuffer>& sdbufs, bool allowMissing)  //??? convert sdbufs to vector of shared_ptr
{
    /// this node is prototype of CompressedVector

    /// don't checkImageFileOpen

    std::set<ustring> pathNames;
    for (unsigned i = 0; i < sdbufs.size(); i++) {
        ustring pathName = sdbufs.at(i).impl()->pathName();

        /// Check that all buffers are same size
        if (sdbufs.at(i).impl()->capacity() != sdbufs.at(0).impl()->capacity()) {
            throw E57_EXCEPTION2(E57_ERROR_BUFFER_SIZE_MISMATCH,
                                 "this->pathName=" + this->pathName()
                                 + " sdbuf.pathName=" + pathName
                                 + " firstCapacity=" + toString(sdbufs.at(0).impl()->capacity())
                                 + " secondCapacity=" + toString(sdbufs.at(i).impl()->capacity()));
        }

        /// Add each pathName to set, error if already in set (a duplicate pathName in sdbufs)
        if (!pathNames.insert(pathName).second)
            throw E57_EXCEPTION2(E57_ERROR_BUFFER_DUPLICATE_PATHNAME, "this->pathName=" + this->pathName() + " sdbuf.pathName=" + pathName);

        /// Check no bad fields in sdbufs
        if (!isDefined(pathName))
            throw E57_EXCEPTION2(E57_ERROR_PATH_UNDEFINED, "this->pathName=" + this->pathName() + " sdbuf.pathName=" + pathName);
    }

    if (!allowMissing) {
        /// Traverse tree recursively, checking that all nodes are listed in sdbufs
        checkLeavesInSet(pathNames, shared_from_this());
    }
}

bool NodeImpl::findTerminalPosition(shared_ptr<NodeImpl> target, uint64_t& countFromLeft)
{
    /// don't checkImageFileOpen

    if (this == &*target) //??? ok?
        return(true);

    switch (type()) {
        case E57_STRUCTURE: {
                StructureNodeImpl* sni = dynamic_cast<StructureNodeImpl*>(this);

                /// Recursively visit child nodes
                if ( sni != nullptr )
                {
                   uint64_t childCount = sni->childCount();
                   for (uint64_t i = 0; i < childCount; ++i)
                   {
                       if (sni->get(i)->findTerminalPosition(target, countFromLeft))
                           return(true);
                   }
                }
            }
            break;
        case E57_VECTOR: {
                VectorNodeImpl* vni = dynamic_cast<VectorNodeImpl*>(this);

                /// Recursively visit child nodes
                if ( vni != nullptr )
                {
                   uint64_t childCount = vni->childCount();
                   for (uint64_t i = 0; i < childCount; ++i)
                   {
                       if (vni->get(i)->findTerminalPosition(target, countFromLeft))
                           return(true);
                   }
                }
            }
            break;
        case E57_COMPRESSED_VECTOR:
            break;  //??? for now, don't search into contents of compressed vector
        case E57_INTEGER:
        case E57_SCALED_INTEGER:
        case E57_FLOAT:
        case E57_STRING:
        case E57_BLOB:
            countFromLeft++;
            break;
    }

    return(false);
}

#ifdef E57_DEBUG
void NodeImpl::dump(int indent, ostream& os)
{
    /// don't checkImageFileOpen
    os << space(indent) << "elementName: " << elementName_ << endl;
    os << space(indent) << "isAttached:  " << isAttached_ << endl;
    os << space(indent) << "path:        " << pathName() << endl;
}
#endif

//================================================================================================
StructureNodeImpl::StructureNodeImpl(weak_ptr<ImageFileImpl> destImageFile)
: NodeImpl(destImageFile)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
}

NodeType StructureNodeImpl::type() const
{
    /// don't checkImageFileOpen
    return E57_STRUCTURE;
}

//??? use visitor?
bool StructureNodeImpl::isTypeEquivalent(shared_ptr<NodeImpl> ni)
{
    /// don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_STRUCTURE)
        return(false);

    /// Downcast to shared_ptr<StructureNodeImpl>, should succeed
    shared_ptr<StructureNodeImpl> si(dynamic_pointer_cast<StructureNodeImpl>(ni));
    if (!si)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->pathName=" + this->pathName() + " elementName="+ni->elementName());

    /// Same number of children?
    if (childCount() != si->childCount())
        return(false);

    /// Check each child is equivalent
    for (unsigned i = 0; i < childCount(); i++) {  //??? vector iterator?
        ustring myChildsFieldName = children_.at(i)->elementName();
        /// Check if matching field name is in same position (to speed things up)
        if (myChildsFieldName == si->children_.at(i)->elementName()) {
            if (!children_.at(i)->isTypeEquivalent(si->children_.at(i)))
                return(false);
        } else {
            /// Children in different order, so lookup by name and check if equal to our child
            if (!si->isDefined(myChildsFieldName))
                return(false);
            if (!children_.at(i)->isTypeEquivalent(si->lookup(myChildsFieldName)))
                return(false);
        }
    }

    /// Types match
    return(true);
}

bool StructureNodeImpl::isDefined(const ustring& pathName)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    shared_ptr<NodeImpl> ni(lookup(pathName));
    return(ni != nullptr);
}

void StructureNodeImpl::setAttachedRecursive()
{
    /// Mark this node as attached to an ImageFile
    isAttached_ = true;

    /// Not a leaf node, so mark all our children
    for (unsigned i = 0; i < children_.size(); i++)
        children_.at(i)->setAttachedRecursive();
}

int64_t StructureNodeImpl::childCount() const
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    return children_.size();
}
shared_ptr<NodeImpl> StructureNodeImpl::get(int64_t index)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
        if (index < 0 || index >= static_cast<int64_t>(children_.size())) { // %%% Possible truncation on platforms where size_t = uint64
        throw E57_EXCEPTION2(E57_ERROR_CHILD_INDEX_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " index=" + toString(index)
                             + " size=" + toString(children_.size()));
    }
    return(children_.at(static_cast<unsigned>(index)));
}


shared_ptr<NodeImpl> StructureNodeImpl::get(const ustring& pathName)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    shared_ptr<NodeImpl> ni(lookup(pathName));
    if (!ni)
        throw E57_EXCEPTION2(E57_ERROR_PATH_UNDEFINED, "this->pathName=" + this->pathName() + " pathName=" + pathName);
    return(ni);
}

shared_ptr<NodeImpl> StructureNodeImpl::lookup(const ustring& pathName)
{
    /// don't checkImageFileOpen
    //??? use lookup(fields, level) instead, for speed.
    bool isRelative;
    vector<ustring> fields;
    shared_ptr<ImageFileImpl> imf(destImageFile_);
    imf->pathNameParse(pathName, isRelative, fields);  // throws if bad pathName

    if (isRelative || isRoot()) {
        if (fields.size() == 0)
            if (isRelative) {
                return(shared_ptr<NodeImpl>());  /// empty pointer
            } else {
                shared_ptr<NodeImpl> root(getRoot());
                return(root);
            }
        else {
            /// Find child with elementName that matches first field in path
            unsigned i;
            for (i = 0; i < children_.size(); i++) {
                if (fields.at(0) == children_.at(i)->elementName())
                    break;
            }
            if (i == children_.size())
                return(shared_ptr<NodeImpl>());  /// empty pointer
            if (fields.size() == 1) {
                return(children_.at(i));
            } else {
                //??? use level here rather than unparse
                /// Remove first field in path
                fields.erase(fields.begin());

                /// Call lookup on child object with remaining fields in path name
                return(children_.at(i)->lookup(imf->pathNameUnparse(true, fields)));
            }
        }
    } else {  /// Absolute pathname and we aren't at the root
        /// Find root of the tree
        shared_ptr<NodeImpl> root(getRoot());

        /// Call lookup on root
        return(root->lookup(pathName));
    }
}

void StructureNodeImpl::set(int64_t index64, shared_ptr<NodeImpl> ni)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    unsigned index = static_cast<unsigned>(index64);

    /// Allow index == current number of elements, interpret as append
    if (index64 < 0 || index64 > UINT_MAX || index > children_.size()) {
        throw E57_EXCEPTION2(E57_ERROR_CHILD_INDEX_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " index=" + toString(index64)
                             + " size=" + toString(children_.size()));
    }

    /// Enforce "set once" policy, only allow append
    if (index != children_.size()) {
        throw E57_EXCEPTION2(E57_ERROR_SET_TWICE,
                             "this->pathName=" + this->pathName()
                             + " index=" + toString(index64));
    }

    /// Verify that child is destined for same ImageFile as this is
    shared_ptr<ImageFileImpl> thisDest(destImageFile());
    shared_ptr<ImageFileImpl> niDest(ni->destImageFile());
    if (thisDest != niDest) {
        throw E57_EXCEPTION2(E57_ERROR_DIFFERENT_DEST_IMAGEFILE,
                             "this->destImageFile" + thisDest->fileName()
                             + " ni->destImageFile" + niDest->fileName());
    }

    /// Field name is string version of index value, e.g. "14"
    stringstream elementName;
    elementName << index;

    /// If this struct is type constrained, can't add new child
    if (isTypeConstrained())
        throw E57_EXCEPTION2(E57_ERROR_HOMOGENEOUS_VIOLATION, "this->pathName=" + this->pathName());

    ni->setParent(shared_from_this(), elementName.str());
    children_.push_back(ni);
}

void StructureNodeImpl::set(const ustring& pathName, shared_ptr<NodeImpl> ni, bool autoPathCreate)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    //??? parse pathName! throw if impossible, absolute and multi-level paths...
    //??? enforce type constraints on path (non-zero index types match zero index types for VECTOR, COMPRESSED_VECTOR

#ifdef E57_MAX_VERBOSE
    cout << "StructureNodeImpl::set(pathName=" << pathName << ", ni, autoPathCreate=" << autoPathCreate << endl;
#endif

    bool isRelative;
    vector<ustring> fields;

    /// Path may be absolute or relative with several levels.  Break string into individual levels.
    shared_ptr<ImageFileImpl> imf(destImageFile_);
    imf->pathNameParse(pathName, isRelative, fields);  // throws if bad pathName
    if (isRelative) {
        /// Relative path, starting from current object, e.g. "foo/17/bar"
        set(fields, 0, ni, autoPathCreate);
    } else {
        /// Absolute path (starting from root), e.g. "/foo/17/bar"
        getRoot()->set(fields, 0, ni, autoPathCreate);
    }
}

void StructureNodeImpl::set(const vector<ustring>& fields, unsigned level, shared_ptr<NodeImpl> ni, bool autoPathCreate)
{
#ifdef E57_MAX_VERBOSE
    cout << "StructureNodeImpl::set: level=" << level << endl;
    for (unsigned i = 0; i < fields.size(); i++)
        cout << "  field[" << i << "]: " << fields.at(i) << endl;
#endif

    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    //??? check if field is numeric string (e.g. "17"), verify number is same as index, else throw bad_path

    /// Check if trying to set the root node "/", which is illegal
    if (level == 0 && fields.size() == 0)
        throw E57_EXCEPTION2(E57_ERROR_SET_TWICE, "this->pathName=" + this->pathName() + " element=/");

    /// Serial search for matching field name, if find match, have error since can't set twice
    for (unsigned i = 0; i < children_.size(); i++) {
        if (fields.at(level) == children_.at(i)->elementName()) {
            if (level == fields.size()-1) {
                /// Enforce "set once" policy, don't allow reset
                throw E57_EXCEPTION2(E57_ERROR_SET_TWICE, "this->pathName=" + this->pathName() + " element=" + fields[level]);
            } else {
                /// Recurse on child
                children_.at(i)->set(fields, level+1, ni);
            }
            return;
        }
    }
    /// Didn't find matching field name, so have a new child.

    /// If this struct is type constrained, can't add new child
    if (isTypeConstrained())
        throw E57_EXCEPTION2(E57_ERROR_HOMOGENEOUS_VIOLATION, "this->pathName=" + this->pathName());

    /// Check if we are at bottom level
    if (level == fields.size()-1){
        /// At bottom, so append node at end of children
        ni->setParent(shared_from_this(), fields.at(level));
        children_.push_back(ni);
    } else {
        /// Not at bottom level, if not autoPathCreate have an error
        if (!autoPathCreate) {
            throw E57_EXCEPTION2(E57_ERROR_PATH_UNDEFINED,
                             "this->pathName=" + this->pathName() + " field=" + fields.at(level));
        }
        //??? what if extra fields are numbers?

        /// Do autoPathCreate: Create nested Struct objects for extra field names in path
        shared_ptr<NodeImpl> parent(shared_from_this());
        for (;level != fields.size()-1; level++) {
            shared_ptr<StructureNodeImpl> child(new StructureNodeImpl(destImageFile_));
            parent->set(fields.at(level), child);
            parent = child;
        }
        parent->set(fields.at(level), ni);
    }
}

void StructureNodeImpl::append(shared_ptr<NodeImpl> ni)
{
    /// don't checkImageFileOpen, set() will do it

    /// Create new node at end of list with integer field name
    set(childCount(), ni);
}

//??? use visitor?
void StructureNodeImpl::checkLeavesInSet(const std::set<ustring>& pathNames, shared_ptr<NodeImpl> origin)
{
    /// don't checkImageFileOpen

    /// Not a leaf node, so check all our children
    for (unsigned i = 0; i < children_.size(); i++)
        children_.at(i)->checkLeavesInSet(pathNames, origin);
}

//??? use visitor?
void StructureNodeImpl::writeXml(std::shared_ptr<ImageFileImpl> imf, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    /// don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"Structure\"";

    /// If this struct is the root for the E57 file, add name space declarations
    /// Note the prototype of a CompressedVector is a separate tree, so don't want to write out namespaces if not the ImageFile root
    if (isRoot() && shared_from_this() == imf->root()) {
        bool gotDefaultNamespace = false;
        for (size_t i = 0; i < imf->extensionsCount(); i++) {
            const char* xmlnsExtension;
            if (imf->extensionsPrefix(i) == "") {
                gotDefaultNamespace = true;
                xmlnsExtension = "xmlns";
            } else
                xmlnsExtension = "xmlns:";
            cf << "\n" << space(indent+fieldName.length()+2) << xmlnsExtension << imf->extensionsPrefix(i) << "=\"" << imf->extensionsUri(i) << "\"";
        }

        /// If user didn't explicitly declare a default namespace, use the current E57 standard one.
        if (!gotDefaultNamespace)
            cf << "\n" << space(indent+fieldName.length()+2) << "xmlns=\"" << E57_V1_0_URI << "\"";
    }
    if (children_.size() > 0) {
        cf << ">\n";

        /// Write all children nested inside Structure element
        for (unsigned i = 0; i < children_.size(); i++)
            children_.at(i)->writeXml(imf, cf, indent+2);

        /// Write closing tag
        cf << space(indent) << "</" << fieldName << ">\n";
    } else {
        /// XML element has no child elements
        cf << "/>\n";
    }
}

//??? use visitor?
#ifdef E57_DEBUG
void StructureNodeImpl::dump(int indent, ostream& os)
{
    /// don't checkImageFileOpen
    os << space(indent) << "type:        Structure" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    for (unsigned i = 0; i < children_.size(); i++) {
        os << space(indent) << "child[" << i << "]:" << endl;
        children_.at(i)->dump(indent+2, os);
    }
}
#endif

//=============================================================================
VectorNodeImpl::VectorNodeImpl(weak_ptr<ImageFileImpl> destImageFile, bool allowHeteroChildren)
: StructureNodeImpl(destImageFile),
  allowHeteroChildren_(allowHeteroChildren)
{
    /// don't checkImageFileOpen, StructNodeImpl() will do it
}

NodeType VectorNodeImpl::type() const
{
    /// don't checkImageFileOpen
    return E57_VECTOR;
}

bool VectorNodeImpl::isTypeEquivalent(shared_ptr<NodeImpl> ni)
{
    /// don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_VECTOR)
        return(false);

    /// Downcast to shared_ptr<VectorNodeImpl>
    shared_ptr<VectorNodeImpl> ai(dynamic_pointer_cast<VectorNodeImpl>(ni));
    if (!ai)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// allowHeteroChildren must match
    if (allowHeteroChildren_ != ai->allowHeteroChildren_)
        return(false);

    /// Same number of children?
    if (childCount() != ai->childCount())
        return(false);

    /// Check each child, must be in same order
    for (unsigned i = 0; i < childCount(); i++) {
        if (!children_.at(i)->isTypeEquivalent(ai->children_.at(i)))
            return(false);
    }

    /// Types match
    return(true);
}

bool VectorNodeImpl::allowHeteroChildren() const
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return allowHeteroChildren_;
}

void VectorNodeImpl::set(int64_t index64, shared_ptr<NodeImpl> ni)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    if (!allowHeteroChildren_) {
        /// New node type must match all existing children
        for (unsigned i = 0; i < children_.size(); i++) {
            if (!children_.at(i)->isTypeEquivalent(ni))
                throw E57_EXCEPTION2(E57_ERROR_HOMOGENEOUS_VIOLATION, "this->pathName=" + this->pathName());
        }
    }

    ///??? for now, use base implementation
    StructureNodeImpl::set(index64, ni);
}

void VectorNodeImpl::writeXml(std::shared_ptr<ImageFileImpl> imf, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    /// don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"Vector\" allowHeterogeneousChildren=\"" << static_cast<int64_t>(allowHeteroChildren_) << "\">\n";
    for (unsigned i = 0; i < children_.size(); i++)
        children_.at(i)->writeXml(imf, cf, indent+2, "vectorChild");
    cf << space(indent) << "</"<< fieldName << ">\n";
}

#ifdef E57_DEBUG
void VectorNodeImpl::dump(int indent, ostream& os)
{
    /// don't checkImageFileOpen
    os << space(indent) << "type:        Vector" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "allowHeteroChildren: " << allowHeteroChildren() << endl;
    for (unsigned i = 0; i < children_.size(); i++) {
        os << space(indent) << "child[" << i << "]:" << endl;
        children_.at(i)->dump(indent+2, os);
    }
}
#endif

//=====================================================================================
SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, int8_t* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_INT8), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, uint8_t* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_UINT8), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, int16_t* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_INT16), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, uint16_t* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_UINT16), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, int32_t* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_INT32), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, uint32_t* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_UINT32), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, int64_t* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_INT64), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, bool* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_BOOL), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, float* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_REAL32), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, double* base, const size_t capacity, bool doConversion, bool doScaling, size_t stride)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_REAL64), base_(reinterpret_cast<char*>(base)),
  capacity_(capacity), doConversion_(doConversion), doScaling_(doScaling), stride_(stride), nextIndex_(0), ustrings_(0)
{
    /// don't checkImageFileOpen, checkState_ will do it
    checkState_();
}

SourceDestBufferImpl::SourceDestBufferImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring pathName, vector<ustring>* b)
: destImageFile_(destImageFile), pathName_(pathName), memoryRepresentation_(E57_USTRING), base_(0),
  capacity_(0/*updated below*/), doConversion_(false), doScaling_(false), stride_(0), nextIndex_(0), ustrings_(b)
{
    /// don't checkImageFileOpen, checkState_ will do it

    /// Set capacity_ after testing that b is OK
    if (b == NULL)
        throw E57_EXCEPTION2(E57_ERROR_BAD_BUFFER, "sdbuf.pathName=" + pathName);
    capacity_ = b->size();

    checkState_();

    /// Note that capacity_ is set to the size() of the vector<>, not its capacity().
    /// The size() of *ustrings_ will not be changed as strings are stored in it.
}

void SourceDestBufferImpl::checkState_() const
{
    /// Implement checkImageFileOpen functionality for SourceDestBufferImpl ctors
    /// Throw an exception if destImageFile (destImageFile_) isn't open
    shared_ptr<ImageFileImpl> destImageFile(destImageFile_);
    if (!destImageFile->isOpen())
        throw E57_EXCEPTION2(E57_ERROR_IMAGEFILE_NOT_OPEN, "fileName=" + destImageFile->fileName());

    /// Check pathName is well formed (can't verify path is defined until associate sdbuffer with CompressedVector later)
    shared_ptr<ImageFileImpl> imf(destImageFile_);
    imf->pathNameCheckWellFormed(pathName_);

    if (memoryRepresentation_ != E57_USTRING) {
        if (base_ == nullptr)
            throw E57_EXCEPTION2(E57_ERROR_BAD_BUFFER, "pathName=" + pathName_);
        if (stride_ == 0)
            throw E57_EXCEPTION2(E57_ERROR_BAD_BUFFER, "pathName=" + pathName_);
        //??? check base alignment, depending on CPU type
        //??? check if stride too small, positive or negative
    } else {
        if (ustrings_ == NULL)
            throw E57_EXCEPTION2(E57_ERROR_BAD_BUFFER, "pathName=" + pathName_);
    }
}

int64_t SourceDestBufferImpl::getNextInt64()
{
    /// don't checkImageFileOpen

    /// Verify index is within bounds
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Fetch value from source buffer.
    /// Convert from non-integer formats if requested.
    char* p = &base_[nextIndex_*stride_];
    int64_t value;
    switch (memoryRepresentation_) {
        case E57_INT8:
            value = static_cast<int64_t>(*reinterpret_cast<int8_t*>(p));
            break;
        case E57_UINT8:
            value = static_cast<int64_t>(*reinterpret_cast<uint8_t*>(p));
            break;
        case E57_INT16:
            value = static_cast<int64_t>(*reinterpret_cast<int16_t*>(p));
            break;
        case E57_UINT16:
            value = static_cast<int64_t>(*reinterpret_cast<uint16_t*>(p));
            break;
        case E57_INT32:
            value = static_cast<int64_t>(*reinterpret_cast<int32_t*>(p));
            break;
        case E57_UINT32:
            value = static_cast<int64_t>(*reinterpret_cast<uint32_t*>(p));
            break;
        case E57_INT64:
            value = *reinterpret_cast<int64_t*>(p);
            break;
        case E57_BOOL:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            /// Convert bool to 0/1, all non-zero values map to 1.0
            value = (*reinterpret_cast<bool*>(p)) ? 1 : 0;
            break;
        case E57_REAL32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            //??? fault if get special value: NaN, NegInf...
            value = static_cast<int64_t>(*reinterpret_cast<float*>(p));
            break;
        case E57_REAL64:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            //??? fault if get special value: NaN, NegInf...
            value = static_cast<int64_t>(*reinterpret_cast<double*>(p));
            break;
        case E57_USTRING:
            throw E57_EXCEPTION2(E57_ERROR_EXPECTING_NUMERIC, "pathName=" + pathName_);
        default:
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);
    }
    nextIndex_++;
    return(value);
}

int64_t SourceDestBufferImpl::getNextInt64(double scale, double offset)
{
    /// don't checkImageFileOpen

    /// Reverse scale (undo scaling) of a user's number to get raw value to put in file.

    /// Encorporating the scale is optional (requested by user when constructing the sdbuf).
    /// If the user did not request scaling, then we get raw values from user's buffer.
    if (!doScaling_) {
        /// Just return raw value.
        return(getNextInt64());
    }

    /// Double check non-zero scale.  Going to divide by it below.
    if (scale == 0)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Verify index is within bounds
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Fetch value from source buffer.
    /// Convert from non-integer formats if requested
    char* p = &base_[nextIndex_*stride_];
    double doubleRawValue;
    switch (memoryRepresentation_) {
        case E57_INT8:
            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<int8_t*>(p) - offset)/scale + 0.5);
            break;
        case E57_UINT8:
            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<uint8_t*>(p) - offset)/scale + 0.5);
            break;
        case E57_INT16:
            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<int16_t*>(p) - offset)/scale + 0.5);
            break;
        case E57_UINT16:
            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<uint16_t*>(p) - offset)/scale + 0.5);
            break;
        case E57_INT32:
            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<int32_t*>(p) - offset)/scale + 0.5);
            break;
        case E57_UINT32:
            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<uint32_t*>(p) - offset)/scale + 0.5);
            break;
        case E57_INT64:
            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<int64_t*>(p) - offset)/scale + 0.5);
            break;
        case E57_BOOL:
            if (*reinterpret_cast<bool*>(p))
                doubleRawValue = floor((1 - offset)/scale + 0.5);
            else
                doubleRawValue = floor((0 - offset)/scale + 0.5);
            break;
        case E57_REAL32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            //??? fault if get special value: NaN, NegInf...

            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<float*>(p) - offset)/scale + 0.5);
            break;
        case E57_REAL64:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            //??? fault if get special value: NaN, NegInf...

            /// Calc (x-offset)/scale rounded to nearest integer, but keep in floating point until sure is in bounds
            doubleRawValue = floor((*reinterpret_cast<double*>(p) - offset)/scale + 0.5);
            break;
        case E57_USTRING:
            throw E57_EXCEPTION2(E57_ERROR_EXPECTING_NUMERIC, "pathName=" + pathName_);
        default:
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);
    }
    /// Make sure that value is representable in an int64_t
    if (doubleRawValue < E57_INT64_MIN || E57_INT64_MAX < doubleRawValue) {
        throw E57_EXCEPTION2(E57_ERROR_SCALED_VALUE_NOT_REPRESENTABLE,
                             "pathName=" + pathName_
                             + " value=" + toString(doubleRawValue));
    }

    int64_t rawValue = static_cast<int64_t>(doubleRawValue);

    nextIndex_++;
    return(rawValue);
}

float SourceDestBufferImpl::getNextFloat()
{
    /// don't checkImageFileOpen

    /// Verify index is within bounds
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Fetch value from source buffer.
    /// Convert from other formats to floating point if requested
    char* p = &base_[nextIndex_*stride_];
    float value;
    switch (memoryRepresentation_) {
        case E57_INT8:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<float>(*reinterpret_cast<int8_t*>(p));
            break;
        case E57_UINT8:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<float>(*reinterpret_cast<uint8_t*>(p));
            break;
        case E57_INT16:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<float>(*reinterpret_cast<int16_t*>(p));
            break;
        case E57_UINT16:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<float>(*reinterpret_cast<uint16_t*>(p));
            break;
        case E57_INT32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<float>(*reinterpret_cast<int32_t*>(p));
            break;
        case E57_UINT32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<float>(*reinterpret_cast<uint32_t*>(p));
            break;
        case E57_INT64:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<float>(*reinterpret_cast<int64_t*>(p));
            break;
        case E57_BOOL:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);

            /// Convert bool to 0/1, all non-zero values map to 1.0
            value = (*reinterpret_cast<bool*>(p)) ? 1.0F : 0.0F;
            break;
        case E57_REAL32:
            value = *reinterpret_cast<float*>(p);
            break;
        case E57_REAL64: {
            /// Check that exponent of user's value is not too large for single precision number in file.
            double d = *reinterpret_cast<double*>(p);

            ///??? silently limit here?
            if (d < E57_DOUBLE_MIN || E57_DOUBLE_MAX < d)
                throw E57_EXCEPTION2(E57_ERROR_REAL64_TOO_LARGE, "pathName=" + pathName_ + " value=" + toString(d));
            value = static_cast<float>(d);
            break;
        }
        case E57_USTRING:
            throw E57_EXCEPTION2(E57_ERROR_EXPECTING_NUMERIC, "pathName=" + pathName_);
        default:
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);
    }
    nextIndex_++;
    return(value);
}

double SourceDestBufferImpl::getNextDouble()
{
    /// don't checkImageFileOpen

    /// Verify index is within bounds
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Fetch value from source buffer.
    /// Convert from other formats to floating point if requested
    char* p = &base_[nextIndex_*stride_];
    double value;
    switch (memoryRepresentation_) {
        case E57_INT8:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<double>(*reinterpret_cast<int8_t*>(p));
            break;
        case E57_UINT8:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<double>(*reinterpret_cast<uint8_t*>(p));
            break;
        case E57_INT16:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<double>(*reinterpret_cast<int16_t*>(p));
            break;
        case E57_UINT16:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<double>(*reinterpret_cast<uint16_t*>(p));
            break;
        case E57_INT32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<double>(*reinterpret_cast<int32_t*>(p));
            break;
        case E57_UINT32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<double>(*reinterpret_cast<uint32_t*>(p));
            break;
        case E57_INT64:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            value = static_cast<double>(*reinterpret_cast<int64_t*>(p));
            break;
        case E57_BOOL:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            /// Convert bool to 0/1, all non-zero values map to 1.0
            value = (*reinterpret_cast<bool*>(p)) ? 1.0 : 0.0;
            break;
        case E57_REAL32:
            value = static_cast<double>(*reinterpret_cast<float*>(p));
            break;
        case E57_REAL64:
            value = *reinterpret_cast<double*>(p);
            break;
        case E57_USTRING:
            throw E57_EXCEPTION2(E57_ERROR_EXPECTING_NUMERIC, "pathName=" + pathName_);
        default:
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);
    }
    nextIndex_++;
    return(value);
}

ustring SourceDestBufferImpl::getNextString()
{
    /// don't checkImageFileOpen

    /// Check have correct type buffer
    if (memoryRepresentation_ != E57_USTRING)
        throw E57_EXCEPTION2(E57_ERROR_EXPECTING_USTRING, "pathName=" + pathName_);

    /// Verify index is within bounds
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Get ustring from vector
    return((*ustrings_)[nextIndex_++]);
}

void  SourceDestBufferImpl::setNextInt64(int64_t value)
{
    /// don't checkImageFileOpen

    /// Verify have room
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Calc start of memory location, index into buffer using stride_ (the distance between elements).
    char* p = &base_[nextIndex_*stride_];

    switch (memoryRepresentation_) {
        case E57_INT8:
            if (value < E57_INT8_MIN || E57_INT8_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int8_t*>(p) = static_cast<int8_t>(value);
            break;
        case E57_UINT8:
            if (value < E57_UINT8_MIN || E57_UINT8_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint8_t*>(p) = static_cast<uint8_t>(value);
            break;
        case E57_INT16:
            if (value < E57_INT16_MIN || E57_INT16_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int16_t*>(p) = static_cast<int16_t>(value);
            break;
        case E57_UINT16:
            if (value < E57_UINT16_MIN || E57_UINT16_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint16_t*>(p) = static_cast<uint16_t>(value);
            break;
        case E57_INT32:
            if (value < E57_INT32_MIN || E57_INT32_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int32_t*>(p) = static_cast<int32_t>(value);
            break;
        case E57_UINT32:
            if (value < E57_UINT32_MIN || E57_UINT32_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint32_t*>(p) = static_cast<uint32_t>(value);
            break;
        case E57_INT64:
            *reinterpret_cast<int64_t*>(p) = static_cast<int64_t>(value);
            break;
        case E57_BOOL:
            *reinterpret_cast<bool*>(p) = (value ? false : true);
            break;
        case E57_REAL32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            //??? very large integers may lose some lowest bits here. error?
            *reinterpret_cast<float*>(p) = static_cast<float>(value);
            break;
        case E57_REAL64:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            *reinterpret_cast<double*>(p) = static_cast<double>(value);
            break;
        case E57_USTRING:
            throw E57_EXCEPTION2(E57_ERROR_EXPECTING_NUMERIC, "pathName=" + pathName_);
    }

    nextIndex_++;
}

void  SourceDestBufferImpl::setNextInt64(int64_t value, double scale, double offset)
{
    /// don't checkImageFileOpen

    /// Apply a scale and offset to numbers from file before puting in user's buffer.

    /// Encorporating the scale is optional (requested by user when constructing the sdbuf).
    /// If the user did not request scaling, then we send raw values to user's buffer.
    if (!doScaling_) {
        /// Use raw value routine, then bail out.
        setNextInt64(value);
        return;
    }

    /// Verify have room
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Calc start of memory location, index into buffer using stride_ (the distance between elements).
    char* p = &base_[nextIndex_*stride_];

    /// Calc x*scale+offset
    double scaledValue;
    if (memoryRepresentation_ == E57_REAL32 || memoryRepresentation_ == E57_REAL64) {
        /// Value will be stored in some floating point rep in user's buffer, so keep full resolution here.
        scaledValue = value*scale + offset;
    } else {
        /// Value will represented as some integer in user's buffer, so round to nearest integer here.
        /// But keep in floating point rep until we know that the value is representable in the user's buffer.
        scaledValue = floor(value*scale + offset + 0.5);
    }

    switch (memoryRepresentation_) {
        case E57_INT8:
            if (scaledValue < E57_INT8_MIN || E57_INT8_MAX < scaledValue)
                throw E57_EXCEPTION2(E57_ERROR_SCALED_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " scaledValue=" + toString(scaledValue));
            *reinterpret_cast<int8_t*>(p) = static_cast<int8_t>(scaledValue);
            break;
        case E57_UINT8:
            if (scaledValue < E57_UINT8_MIN || E57_UINT8_MAX < scaledValue)
                throw E57_EXCEPTION2(E57_ERROR_SCALED_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " scaledValue=" + toString(scaledValue));
            *reinterpret_cast<uint8_t*>(p) = static_cast<uint8_t>(scaledValue);
            break;
        case E57_INT16:
            if (scaledValue < E57_INT16_MIN || E57_INT16_MAX < scaledValue)
                throw E57_EXCEPTION2(E57_ERROR_SCALED_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " scaledValue=" + toString(scaledValue));
            *reinterpret_cast<int16_t*>(p) = static_cast<int16_t>(scaledValue);
            break;
        case E57_UINT16:
            if (scaledValue < E57_UINT16_MIN || E57_UINT16_MAX < scaledValue)
                throw E57_EXCEPTION2(E57_ERROR_SCALED_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " scaledValue=" + toString(scaledValue));
            *reinterpret_cast<uint16_t*>(p) = static_cast<uint16_t>(scaledValue);
            break;
        case E57_INT32:
            if (scaledValue < E57_INT32_MIN || E57_INT32_MAX < scaledValue)
                throw E57_EXCEPTION2(E57_ERROR_SCALED_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " scaledValue=" + toString(scaledValue));
            *reinterpret_cast<int32_t*>(p) = static_cast<int32_t>(scaledValue);
            break;
        case E57_UINT32:
            if (scaledValue < E57_UINT32_MIN || E57_UINT32_MAX < scaledValue)
                throw E57_EXCEPTION2(E57_ERROR_SCALED_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " scaledValue=" + toString(scaledValue));
            *reinterpret_cast<uint32_t*>(p) = static_cast<uint32_t>(scaledValue);
            break;
        case E57_INT64:
            *reinterpret_cast<int64_t*>(p) = static_cast<int64_t>(scaledValue);
            break;
        case E57_BOOL:
            *reinterpret_cast<bool*>(p) = (scaledValue ? false : true);
            break;
        case E57_REAL32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            /// Check that exponent of result is not too big for single precision float
            if (scaledValue < E57_DOUBLE_MIN || E57_DOUBLE_MAX < scaledValue)
                throw E57_EXCEPTION2(E57_ERROR_SCALED_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " scaledValue=" + toString(scaledValue));
            *reinterpret_cast<float*>(p) = static_cast<float>(scaledValue);
            break;
        case E57_REAL64:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            *reinterpret_cast<double*>(p) = scaledValue;
            break;
        case E57_USTRING:
            throw E57_EXCEPTION2(E57_ERROR_EXPECTING_NUMERIC, "pathName=" + pathName_);
    }

    nextIndex_++;
}

void SourceDestBufferImpl::setNextFloat(float value)
{
    /// don't checkImageFileOpen

    /// Verify have room
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Calc start of memory location, index into buffer using stride_ (the distance between elements).
    char* p = &base_[nextIndex_*stride_];

    switch (memoryRepresentation_) {
        case E57_INT8:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            //??? fault if get special value: NaN, NegInf...  (all other ints below too)
            if (value < E57_INT8_MIN || E57_INT8_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int8_t*>(p) = static_cast<int8_t>(value);
            break;
        case E57_UINT8:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_UINT8_MIN || E57_UINT8_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint8_t*>(p) = static_cast<uint8_t>(value);
            break;
        case E57_INT16:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_INT16_MIN || E57_INT16_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int16_t*>(p) = static_cast<int16_t>(value);
            break;
        case E57_UINT16:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_UINT16_MIN || E57_UINT16_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint16_t*>(p) = static_cast<uint16_t>(value);
            break;
        case E57_INT32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_INT32_MIN || E57_INT32_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int32_t*>(p) = static_cast<int32_t>(value);
            break;
        case E57_UINT32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_UINT32_MIN || E57_UINT32_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint32_t*>(p) = static_cast<uint32_t>(value);
            break;
        case E57_INT64:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_INT64_MIN || E57_INT64_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int64_t*>(p) = static_cast<int64_t>(value);
            break;
        case E57_BOOL:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            *reinterpret_cast<bool*>(p) = (value ? false : true);
            break;
        case E57_REAL32:
            *reinterpret_cast<float*>(p) = value;
            break;
        case E57_REAL64:
            //??? does this count as a conversion?
            *reinterpret_cast<double*>(p) = static_cast<double>(value);
            break;
        case E57_USTRING:
            throw E57_EXCEPTION2(E57_ERROR_EXPECTING_NUMERIC, "pathName=" + pathName_);
    }

    nextIndex_++;
}

void SourceDestBufferImpl::setNextDouble(double value)
{
    /// don't checkImageFileOpen

    /// Verify have room
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Calc start of memory location, index into buffer using stride_ (the distance between elements).
    char* p = &base_[nextIndex_*stride_];

    switch (memoryRepresentation_) {
        case E57_INT8:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            //??? fault if get special value: NaN, NegInf...  (all other ints below too)
            if (value < E57_INT8_MIN || E57_INT8_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int8_t*>(p) = static_cast<int8_t>(value);
            break;
        case E57_UINT8:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_UINT8_MIN || E57_UINT8_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint8_t*>(p) = static_cast<uint8_t>(value);
            break;
        case E57_INT16:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_INT16_MIN || E57_INT16_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int16_t*>(p) = static_cast<int16_t>(value);
            break;
        case E57_UINT16:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_UINT16_MIN || E57_UINT16_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint16_t*>(p) = static_cast<uint16_t>(value);
            break;
        case E57_INT32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_INT32_MIN || E57_INT32_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int32_t*>(p) = static_cast<int32_t>(value);
            break;
        case E57_UINT32:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_UINT32_MIN || E57_UINT32_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<uint32_t*>(p) = static_cast<uint32_t>(value);
            break;
        case E57_INT64:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            if (value < E57_INT64_MIN || E57_INT64_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<int64_t*>(p) = static_cast<int64_t>(value);
            break;
        case E57_BOOL:
            if (!doConversion_)
                throw E57_EXCEPTION2(E57_ERROR_CONVERSION_REQUIRED, "pathName=" + pathName_);
            *reinterpret_cast<bool*>(p) = (value ? false : true);
            break;
        case E57_REAL32:
            /// Does this count as conversion?  It loses information.
            /// Check for really large exponents that can't fit in a single precision
            if (value < E57_DOUBLE_MIN || E57_DOUBLE_MAX < value)
                throw E57_EXCEPTION2(E57_ERROR_VALUE_NOT_REPRESENTABLE, "pathName=" + pathName_ + " value=" + toString(value));
            *reinterpret_cast<float*>(p) = static_cast<float>(value);
            break;
        case E57_REAL64:
            *reinterpret_cast<double*>(p) = value;
            break;
        case E57_USTRING:
            throw E57_EXCEPTION2(E57_ERROR_EXPECTING_NUMERIC, "pathName=" + pathName_);
    }

    nextIndex_++;
}

void SourceDestBufferImpl::setNextString(const ustring& value)
{
    /// don't checkImageFileOpen

    if (memoryRepresentation_ != E57_USTRING)
        throw E57_EXCEPTION2(E57_ERROR_EXPECTING_USTRING, "pathName=" + pathName_);

    /// Verify have room.
    if (nextIndex_ >= capacity_)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "pathName=" + pathName_);

    /// Assign to already initialized element in vector
    (*ustrings_)[nextIndex_] = value;
    nextIndex_++;
}

void SourceDestBufferImpl::checkCompatible(shared_ptr<SourceDestBufferImpl> newBuf) const
{
    if (pathName_ != newBuf->pathName()) {
        throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                             "pathName=" + pathName_
                             + " newPathName=" + newBuf->pathName());
    }
    if (memoryRepresentation_ != newBuf->memoryRepresentation()) {
        throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                             "memoryRepresentation=" + toString(memoryRepresentation_)
                             + " newMemoryType=" + toString(newBuf->memoryRepresentation()));
    }
    if (capacity_ != newBuf->capacity()) {
        throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                             "capacity=" + toString(capacity_)
                             + " newCapacity=" + toString(newBuf->capacity()));
    }
    if (doConversion_ != newBuf->doConversion()) {
        throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                             "doConversion=" + toString(doConversion_)
                             + "newDoConversion=" + toString(newBuf->doConversion()));
    }
    if (doConversion_ != newBuf->doConversion()) {
        throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                             "doConversion=" + toString(doConversion_)
                             + " newDoConversion=" + toString(newBuf->doConversion()));
    }
    if (stride_ != newBuf->stride()) {
        throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                             "stride=" + toString(stride_)
                             + " newStride=" + toString(newBuf->stride()));
    }
}

#ifdef E57_DEBUG
void SourceDestBufferImpl::dump(int indent, ostream& os)
{
    /// don't checkImageFileOpen

    os << space(indent) << "pathName:             " << pathName_ << endl;
    os << space(indent) << "memoryRepresentation: ";
    switch (memoryRepresentation_) {
        case E57_INT8:      os << "int8_t" << endl;    break;
        case E57_UINT8:     os << "uint8_t" << endl;   break;
        case E57_INT16:     os << "int16_t" << endl;    break;
        case E57_UINT16:    os << "uint16_t" << endl;  break;
        case E57_INT32:     os << "int32_t" << endl;   break;
        case E57_UINT32:    os << "uint32_t" << endl;  break;
        case E57_INT64:     os << "int64_t" << endl;   break;
        case E57_BOOL:      os << "bool" << endl;      break;
        case E57_REAL32:    os << "float" << endl;     break;
        case E57_REAL64:    os << "double" << endl;    break;
        case E57_USTRING:   os << "ustring" << endl;   break;
        default:            os << "<unknown>" << endl; break;
    }
    os << space(indent) << "base:                 " << static_cast<const void*>(base_) << endl;
    os << space(indent) << "ustrings:             " << static_cast<const void*>(ustrings_) << endl;
    os << space(indent) << "capacity:             " << capacity_ << endl;
    os << space(indent) << "doConversion:         " << doConversion_ << endl;
    os << space(indent) << "doScaling:            " << doScaling_ << endl;
    os << space(indent) << "stride:               " << stride_ << endl;
    os << space(indent) << "nextIndex:            " << nextIndex_ << endl;
}
#endif

//=============================================================================
CompressedVectorNodeImpl::CompressedVectorNodeImpl(weak_ptr<ImageFileImpl> destImageFile)
: NodeImpl(destImageFile)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    recordCount_                = 0;
    binarySectionLogicalStart_  = 0;
}

NodeType CompressedVectorNodeImpl::type() const
{
    // don't checkImageFileOpen
    return E57_COMPRESSED_VECTOR;
}

void CompressedVectorNodeImpl::setPrototype(shared_ptr<NodeImpl> prototype)
{
    // don't checkImageFileOpen, ctor did it

    //??? check ok for proto, no Blob CompressedVector, empty?
    //??? throw E57_EXCEPTION2(E57_ERROR_BAD_PROTOTYPE)

    /// Can't set prototype twice.
    if (prototype_)
        throw E57_EXCEPTION2(E57_ERROR_SET_TWICE, "this->pathName=" + this->pathName());

    /// prototype can't have a parent (must be a root node)
    if (!prototype->isRoot()) {
        throw E57_EXCEPTION2(E57_ERROR_ALREADY_HAS_PARENT,
                             "this->pathName=" + this->pathName() +
                             " prototype->pathName=" + prototype->pathName());
    }

    /// Verify that prototype is destined for same ImageFile as this is
    shared_ptr<ImageFileImpl> thisDest(destImageFile());
    shared_ptr<ImageFileImpl> prototypeDest(prototype->destImageFile());
    if (thisDest != prototypeDest) {
        throw E57_EXCEPTION2(E57_ERROR_DIFFERENT_DEST_IMAGEFILE,
                             "this->destImageFile" + thisDest->fileName()
                             + " prototype->destImageFile" + prototypeDest->fileName());
    }

    //!!! check for incomplete CompressedVectors when closing file
    prototype_ = prototype;

    /// Note that prototype is not attached to CompressedVector in a parent/child relationship.
    /// This means that prototype is a root node (has no parent).
}

shared_ptr<NodeImpl> CompressedVectorNodeImpl::getPrototype()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(prototype_);  //??? check defined
}

void CompressedVectorNodeImpl::setCodecs(shared_ptr<VectorNodeImpl> codecs)
{
    // don't checkImageFileOpen, ctor did it

    //??? check ok for codecs, empty vector, or each element has "inputs" vector of strings, codec substruct

    /// Can't set codecs twice.
    if (codecs_)
        throw E57_EXCEPTION2(E57_ERROR_SET_TWICE, "this->pathName=" + this->pathName());

    /// codecs can't have a parent (must be a root node)
    if (!codecs->isRoot()) {
        throw E57_EXCEPTION2(E57_ERROR_ALREADY_HAS_PARENT,
                             "this->pathName=" + this->pathName() +
                             " codecs->pathName=" + codecs->pathName());
    }

    /// Verify that codecs is destined for same ImageFile as this is
    shared_ptr<ImageFileImpl> thisDest(destImageFile());
    shared_ptr<ImageFileImpl> codecsDest(codecs->destImageFile());
    if (thisDest != codecsDest) {
        throw E57_EXCEPTION2(E57_ERROR_DIFFERENT_DEST_IMAGEFILE,
                             "this->destImageFile" + thisDest->fileName()
                             + " codecs->destImageFile" + codecsDest->fileName());
    }

    codecs_ = codecs;

    /// Note that codecs is not attached to CompressedVector in a parent/child relationship.
    /// This means that codecs is a root node (has no parent).
}

shared_ptr<VectorNodeImpl> CompressedVectorNodeImpl::getCodecs()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(codecs_);  //??? check defined
}

bool CompressedVectorNodeImpl::isTypeEquivalent(shared_ptr<NodeImpl> ni)
{
    // don't checkImageFileOpen

    //??? is this test a good idea?

    /// Same node type?
    if (ni->type() != E57_COMPRESSED_VECTOR)
        return(false);

    /// Downcast to shared_ptr<CompressedVectorNodeImpl>
    shared_ptr<CompressedVectorNodeImpl> cvi(dynamic_pointer_cast<CompressedVectorNodeImpl>(ni));
    if (!cvi)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// recordCount must match
    if (recordCount_ != cvi->recordCount_)
        return(false);

    /// Prototypes and codecs must match ???
    if (!prototype_->isTypeEquivalent(cvi->prototype_))
        return(false);
    if (!codecs_->isTypeEquivalent(cvi->codecs_))
        return(false);

    return(true);
}

bool CompressedVectorNodeImpl::isDefined(const ustring& pathName)
{
    throw E57_EXCEPTION2(E57_ERROR_NOT_IMPLEMENTED, "this->pathName=" + this->pathName() + " pathName=" + pathName);
    return(false);
}

void CompressedVectorNodeImpl::setAttachedRecursive()
{
    /// Mark this node as attached to an ImageFile
    isAttached_ = true;

    /// Mark nodes in prototype tree, if defined
    if (prototype_)
        prototype_->setAttachedRecursive();

    /// Mark nodes in codecs tree if defined
    if (codecs_)
        codecs_->setAttachedRecursive();
}

int64_t CompressedVectorNodeImpl::childCount()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(recordCount_);
}

void CompressedVectorNodeImpl::checkLeavesInSet(const std::set<ustring>& /*pathNames*/, shared_ptr<NodeImpl> /*origin*/)
{
    // don't checkImageFileOpen

    /// Since only called for prototype nodes, should't be able to get here since CompressedVectors can't be in prototypes
    throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->pathName=" + this->pathName());
}

void CompressedVectorNodeImpl::writeXml(std::shared_ptr<ImageFileImpl> imf, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    uint64_t physicalStart = cf.logicalToPhysical(binarySectionLogicalStart_);

    cf << space(indent) << "<" << fieldName << " type=\"CompressedVector\"";
    cf << " fileOffset=\"" << physicalStart;
    cf << "\" recordCount=\"" << recordCount_ << "\">\n";

    if (prototype_)
        prototype_->writeXml(imf, cf, indent+2, "prototype");
    if (codecs_)
        codecs_->writeXml(imf, cf, indent+2, "codecs");
    cf << space(indent) << "</"<< fieldName << ">\n";
}

#ifdef E57_DEBUG
void CompressedVectorNodeImpl::dump(int indent, ostream& os)
{
    os << space(indent) << "type:        CompressedVector" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    if (prototype_) {
        os << space(indent) << "prototype:" << endl;
        prototype_->dump(indent+2, os);
    } else
        os << space(indent) << "prototype: <empty>" << endl;
    if (codecs_) {
        os << space(indent) << "codecs:" << endl;
        codecs_->dump(indent+2, os);
    } else
        os << space(indent) << "codecs: <empty>" << endl;
    os << space(indent) << "recordCount:                " << recordCount_ << endl;
    os << space(indent) << "binarySectionLogicalStart:  " << binarySectionLogicalStart_ << endl;
}
#endif

shared_ptr<CompressedVectorWriterImpl> CompressedVectorNodeImpl::writer(vector<SourceDestBuffer> sbufs)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    shared_ptr<ImageFileImpl> destImageFile(destImageFile_);

    /// Check don't have any writers/readers open for this ImageFile
    if (destImageFile->writerCount() > 0) {
        throw E57_EXCEPTION2(E57_ERROR_TOO_MANY_WRITERS,
                             "fileName=" + destImageFile->fileName()
                             + " writerCount=" + toString(destImageFile->writerCount())
                             + " readerCount=" + toString(destImageFile->readerCount()));
    }
    if (destImageFile->readerCount() > 0) {
        throw E57_EXCEPTION2(E57_ERROR_TOO_MANY_READERS,
                             "fileName=" + destImageFile->fileName()
                             + " writerCount=" + toString(destImageFile->writerCount())
                             + " readerCount=" + toString(destImageFile->readerCount()));
    }

    /// sbufs can't be empty
    if (sbufs.size() == 0)
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT, "fileName=" + destImageFile->fileName());

    if (!destImageFile->isWriter())
        throw E57_EXCEPTION2(E57_ERROR_FILE_IS_READ_ONLY, "fileName=" + destImageFile->fileName());

    if (!isAttached())
        throw E57_EXCEPTION2(E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName());

    /// Get pointer to me (really shared_ptr<CompressedVectorNodeImpl>)
    shared_ptr<NodeImpl> ni(shared_from_this());

    /// Downcast pointer to right type
    shared_ptr<CompressedVectorNodeImpl> cai(dynamic_pointer_cast<CompressedVectorNodeImpl>(ni));
    if (!cai)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// Return a shared_ptr to new object
    shared_ptr<CompressedVectorWriterImpl> cvwi(new CompressedVectorWriterImpl(cai, sbufs));
    return(cvwi);
}

shared_ptr<CompressedVectorReaderImpl> CompressedVectorNodeImpl::reader(vector<SourceDestBuffer> dbufs)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    shared_ptr<ImageFileImpl> destImageFile(destImageFile_);

    /// Check don't have any writers/readers open for this ImageFile
    if (destImageFile->writerCount() > 0) {
        throw E57_EXCEPTION2(E57_ERROR_TOO_MANY_WRITERS,
                             "fileName=" + destImageFile->fileName()
                             + " writerCount=" + toString(destImageFile->writerCount())
                             + " readerCount=" + toString(destImageFile->readerCount()));
    }
    if (destImageFile->readerCount() > 0) {
        throw E57_EXCEPTION2(E57_ERROR_TOO_MANY_READERS,
                             "fileName=" + destImageFile->fileName()
                             + " writerCount=" + toString(destImageFile->writerCount())
                             + " readerCount=" + toString(destImageFile->readerCount()));
    }

    /// dbufs can't be empty
    if (dbufs.size() == 0)
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT, "fileName=" + destImageFile->fileName());

    /// Can be read or write mode, but must be attached
    if (!isAttached())
        throw E57_EXCEPTION2(E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName());

    /// Get pointer to me (really shared_ptr<CompressedVectorNodeImpl>)
    shared_ptr<NodeImpl> ni(shared_from_this());
#ifdef E57_MAX_VERBOSE
    //cout << "constructing CAReader, ni:" << endl;
    //ni->dump(4);
#endif

    /// Downcast pointer to right type
    shared_ptr<CompressedVectorNodeImpl> cai(dynamic_pointer_cast<CompressedVectorNodeImpl>(ni));
    if (!cai)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());
#ifdef E57_MAX_VERBOSE
    //cout<<"constructing CAReader, cai:"<<endl;
    //cai->dump(4);
#endif
    /// Return a shared_ptr to new object
    shared_ptr<CompressedVectorReaderImpl> cvri(new CompressedVectorReaderImpl(cai, dbufs));
    return(cvri);
}

//=====================================================================
IntegerNodeImpl::IntegerNodeImpl(weak_ptr<ImageFileImpl> destImageFile, int64_t value, int64_t minimum, int64_t maximum)
: NodeImpl(destImageFile),
  value_(value),
  minimum_(minimum),
  maximum_(maximum)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Enforce the given bounds
    if (value < minimum || maximum < value) {
        throw E57_EXCEPTION2(E57_ERROR_VALUE_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " value=" + toString(value)
                             + " minimum=" + toString(minimum)
                             + " maximum=" + toString(maximum));
    }
}

NodeType IntegerNodeImpl::type()  const
{
    // don't checkImageFileOpen
    return E57_INTEGER;
}

bool IntegerNodeImpl::isTypeEquivalent(shared_ptr<NodeImpl> ni)
{
    // don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_INTEGER)
        return(false);

    /// Downcast to shared_ptr<IntegerNodeImpl>
    shared_ptr<IntegerNodeImpl> ii(dynamic_pointer_cast<IntegerNodeImpl>(ni));
    if (!ii)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// minimum must match
    if (minimum_ != ii->minimum_)
        return(false);

    /// maximum must match
    if (maximum_ != ii->maximum_)
        return(false);

    /// ignore value_, doesn't have to match

    /// Types match
    return(true);
}

bool IntegerNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen

    /// We have no sub-structure, so if path not empty return false
    return(pathName == "");
}

int64_t IntegerNodeImpl::value()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(value_);
}

int64_t IntegerNodeImpl::minimum()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(minimum_);
}

int64_t IntegerNodeImpl::maximum()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(maximum_);
}

void IntegerNodeImpl::checkLeavesInSet(const std::set<ustring>& pathNames, shared_ptr<NodeImpl> origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set.
    if (pathNames.find(relativePathName(origin)) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void IntegerNodeImpl::writeXml(std::shared_ptr<ImageFileImpl> /*imf???*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"Integer\"";

    /// Don't need to write if are default values
    if (minimum_ != E57_INT64_MIN)
        cf << " minimum=\"" << minimum_ << "\"";
    if (maximum_ != E57_INT64_MAX)
        cf << " maximum=\"" << maximum_ << "\"";

    /// Write value as child text, unless it is the default value
    if (value_ != 0)
        cf << ">" << value_ << "</" << fieldName << ">\n";
    else
        cf << "/>\n";
}

#ifdef E57_DEBUG
void IntegerNodeImpl::dump(int indent, ostream& os)
{
    // don't checkImageFileOpen
    os << space(indent) << "type:        Integer" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "value:       " << value_ << endl;
    os << space(indent) << "minimum:     " << minimum_ << endl;
    os << space(indent) << "maximum:     " << maximum_ << endl;
}
#endif

//=============================================================================
ScaledIntegerNodeImpl::ScaledIntegerNodeImpl(weak_ptr<ImageFileImpl> destImageFile, int64_t rawValue, int64_t minimum, int64_t maximum, double scale, double offset)
: NodeImpl(destImageFile),
  value_(rawValue),
  minimum_(minimum),
  maximum_(maximum),
  scale_(scale),
  offset_(offset)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Enforce the given bounds on raw value
    if (rawValue < minimum || maximum < rawValue) {
        throw E57_EXCEPTION2(E57_ERROR_VALUE_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " rawValue=" + toString(rawValue)
                             + " minimum=" + toString(minimum)
                             + " maximum=" + toString(maximum));
    }
}
//=============================================================================
ScaledIntegerNodeImpl::ScaledIntegerNodeImpl(weak_ptr<ImageFileImpl> destImageFile, double scaledValue, double scaledMinimum, double scaledMaximum, double scale, double offset)
: NodeImpl(destImageFile),
  value_(static_cast<int64_t>(floor((scaledValue - offset)/scale +.5))),
  minimum_(static_cast<int64_t>(floor((scaledMinimum - offset)/scale +.5))),
  maximum_(static_cast<int64_t>(floor((scaledMaximum - offset)/scale +.5))),
  scale_(scale),
  offset_(offset)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Enforce the given bounds on raw value
    if (scaledValue < scaledMinimum || scaledMaximum < scaledValue) {
        throw E57_EXCEPTION2(E57_ERROR_VALUE_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " scaledValue=" + toString(scaledValue)
                             + " scaledMinimum=" + toString(scaledMinimum)
                             + " scaledMaximum=" + toString(scaledMaximum));
    }
}
NodeType ScaledIntegerNodeImpl::type() const
{
    // don't checkImageFileOpen
    return E57_SCALED_INTEGER;
}

bool ScaledIntegerNodeImpl::isTypeEquivalent(shared_ptr<NodeImpl> ni)
{
    // don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_SCALED_INTEGER)
        return(false);

    /// Downcast to shared_ptr<ScaledIntegerNodeImpl>
    shared_ptr<ScaledIntegerNodeImpl> ii(dynamic_pointer_cast<ScaledIntegerNodeImpl>(ni));
    if (!ii)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// minimum must match
    if (minimum_ != ii->minimum_)
        return(false);

    /// maximum must match
    if (maximum_ != ii->maximum_)
        return(false);

    /// scale must match
    if (scale_ != ii->scale_)
        return(false);

    /// offset must match
    if (offset_ != ii->offset_)
        return(false);

    /// ignore value_, doesn't have to match

    /// Types match
    return(true);
}

bool ScaledIntegerNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen

    /// We have no sub-structure, so if path not empty return false
    return(pathName == "");
}

int64_t ScaledIntegerNodeImpl::rawValue()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(value_);
}

double ScaledIntegerNodeImpl::scaledValue()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(value_ * scale_ + offset_);
}

int64_t ScaledIntegerNodeImpl::minimum()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(minimum_);
}
double ScaledIntegerNodeImpl::scaledMinimum()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(minimum_ * scale_ + offset_);
}

int64_t ScaledIntegerNodeImpl::maximum()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(maximum_);
}
double ScaledIntegerNodeImpl::scaledMaximum()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(maximum_ * scale_ + offset_);
}

double ScaledIntegerNodeImpl::scale()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(scale_);
}

double ScaledIntegerNodeImpl::offset()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(offset_);
}

void ScaledIntegerNodeImpl::checkLeavesInSet(const std::set<ustring>& pathNames, shared_ptr<NodeImpl> origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set.
    if (pathNames.find(relativePathName(origin)) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void ScaledIntegerNodeImpl::writeXml(std::shared_ptr<ImageFileImpl> /*imf*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"ScaledInteger\"";

    /// Don't need to write if are default values
    if (minimum_ != E57_INT64_MIN)
        cf << " minimum=\"" << minimum_ << "\"";
    if (maximum_ != E57_INT64_MAX)
        cf << " maximum=\"" << maximum_ << "\"";
    if (scale_ != 1.0)
        cf << " scale=\""  << scale_  << "\"";
    if (offset_ != 0.0)
        cf << " offset=\"" << offset_ << "\"";

    /// Write value as child text, unless it is the default value
    if (value_ != 0)
        cf << ">" << value_ << "</" << fieldName << ">\n";
    else
        cf << "/>\n";
}

#ifdef E57_DEBUG
void ScaledIntegerNodeImpl::dump(int indent, ostream& os)
{
    // don't checkImageFileOpen
    os << space(indent) << "type:        ScaledInteger" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "rawValue:    " << value_ << endl;
    os << space(indent) << "minimum:     " << minimum_ << endl;
    os << space(indent) << "maximum:     " << maximum_ << endl;
    os << space(indent) << "scale:       " << scale_ << endl;
    os << space(indent) << "offset:      " << offset_ << endl;
}
#endif

//=============================================================================

FloatNodeImpl::FloatNodeImpl(weak_ptr<ImageFileImpl> destImageFile, double value, FloatPrecision precision, double minimum, double maximum)
: NodeImpl(destImageFile),
  value_(value),
  precision_(precision),
  minimum_(minimum),
  maximum_(maximum)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Since this ctor also used to construct single precision, and defaults for minimum/maximum are for double precision,
    /// adjust bounds smaller if single.
    if (precision_ == E57_SINGLE) {
        if (minimum_ < E57_FLOAT_MIN)
            minimum_ = E57_FLOAT_MIN;
        if (maximum_ > E57_FLOAT_MAX)
            maximum_ = E57_FLOAT_MAX;
    }

    /// Enforce the given bounds on raw value
    if (value < minimum || maximum < value) {
        throw E57_EXCEPTION2(E57_ERROR_VALUE_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " value=" + toString(value)
                             + " minimum=" + toString(minimum)
                             + " maximum=" + toString(maximum));
    }
}

NodeType FloatNodeImpl::type() const
{
    /// don't checkImageFileOpen
    return E57_FLOAT;
}

bool FloatNodeImpl::isTypeEquivalent(shared_ptr<NodeImpl> ni)
{
    // don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_FLOAT)
        return(false);

    /// Downcast to shared_ptr<FloatNodeImpl>
    shared_ptr<FloatNodeImpl> fi(dynamic_pointer_cast<FloatNodeImpl>(ni));
    if (!fi)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// precision must match
    if (precision_ != fi->precision_)
        return(false);

    /// minimum must match
    if (minimum_ != fi->minimum_)
        return(false);

    /// maximum must match
    if (maximum_ != fi->maximum_)
        return(false);

    /// ignore value_, doesn't have to match

    /// Types match
    return(true);
}

bool FloatNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen

    /// We have no sub-structure, so if path not empty return false
    return(pathName == "");
}

double FloatNodeImpl::value()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(value_);
}

FloatPrecision FloatNodeImpl::precision()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(precision_);
}

double FloatNodeImpl::minimum()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(minimum_);
}

double FloatNodeImpl::maximum()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(maximum_);
}

void FloatNodeImpl::checkLeavesInSet(const std::set<ustring>& pathNames, shared_ptr<NodeImpl> origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set (either relative or absolute form)
    if (pathNames.find(relativePathName(origin)) == pathNames.end() && pathNames.find(pathName()) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void FloatNodeImpl::writeXml(std::shared_ptr<ImageFileImpl> /*imf*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"Float\"";
    if (precision_ == E57_SINGLE) {
        cf << " precision=\"single\"";

        /// Don't need to write if are default values
        if (minimum_ > E57_FLOAT_MIN)
            cf << " minimum=\"" << static_cast<float>(minimum_) << "\"";
        if (maximum_ < E57_FLOAT_MAX)
            cf << " maximum=\"" << static_cast<float>(maximum_) << "\"";

        /// Write value as child text, unless it is the default value
        if (value_ != 0.0)
            cf << ">" << static_cast<float>(value_) << "</" << fieldName << ">\n";
        else
            cf << "/>\n";
    } else {
        /// Don't need to write precision="double", because that's the default

        /// Don't need to write if are default values
        if (minimum_ > E57_DOUBLE_MIN)
            cf << " minimum=\"" << minimum_ << "\"";
        if (maximum_ < E57_DOUBLE_MAX)
            cf << " maximum=\"" << maximum_ << "\"";

        /// Write value as child text, unless it is the default value
        if (value_ != 0.0)
            cf << ">" << value_ << "</" << fieldName << ">\n";
        else
            cf << "/>\n";
    }
}

#ifdef E57_DEBUG
void FloatNodeImpl::dump(int indent, ostream& os)
{
    // don't checkImageFileOpen
    os << space(indent) << "type:        Float" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "precision:   ";
    if (precision() == E57_SINGLE)
        os << "single" << endl;
    else
        os << "double" << endl;

    /// Save old stream config
    const streamsize oldPrecision = os.precision();
    const ios_base::fmtflags oldFlags = os.flags();

    os << space(indent) << scientific << setprecision(17) << "value:       " << value_ << endl;
    os << space(indent) << "minimum:     " << minimum_ << endl;
    os << space(indent) << "maximum:     " << maximum_ << endl;

    /// Restore old stream config
    os.precision(oldPrecision);
    os.flags(oldFlags);
}
#endif

//=============================================================================

StringNodeImpl::StringNodeImpl(weak_ptr<ImageFileImpl> destImageFile, const ustring value)
: NodeImpl(destImageFile),
  value_(value)
{
    // don't checkImageFileOpen, NodeImpl() will do it
}

NodeType StringNodeImpl::type() const
{
    // don't checkImageFileOpen
    return E57_STRING;
}

bool StringNodeImpl::isTypeEquivalent(shared_ptr<NodeImpl> ni)
{
    // don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_STRING)
        return(false);

    /// ignore value_, doesn't have to match

    /// Types match
    return(true);
}

bool StringNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen

    /// We have no sub-structure, so if path not empty return false
    return(pathName == "");
}

ustring StringNodeImpl::value()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(value_);
}

void StringNodeImpl::checkLeavesInSet(const std::set<ustring>& pathNames, shared_ptr<NodeImpl> origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set.
    if (pathNames.find(relativePathName(origin)) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void StringNodeImpl::writeXml(std::shared_ptr<ImageFileImpl> /*imf*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"String\"";

    /// Write value as child text, unless it is the default value
    if (value_ == "") {
        cf << "/>\n";
    } else {
        cf << "><![CDATA[";

        size_t currentPosition = 0;
        size_t len = value_.length();

        /// Loop, searching for occurences of "]]>", which will be split across two CDATA directives
        while (currentPosition < len) {
            size_t found = value_.find("]]>", currentPosition);
            if (found == string::npos) {
                /// Didn't find any more "]]>", so can send the rest.
                cf << value_.substr(currentPosition);
                break;
            } else {
                /// Must output in two pieces, first send upto end of "]]"  (don't send the following ">").
                cf << value_.substr(currentPosition, found-currentPosition+2);

                /// Then start a new CDATA
                cf << "]]><![CDATA[";

                /// Keep looping to send the ">" plus the remaining part of the string
                currentPosition = found+2;
            }
        }
        cf << "]]></" << fieldName << ">\n";
    }
}

#ifdef E57_DEBUG
void StringNodeImpl::dump(int indent, ostream& os)
{
    os << space(indent) << "type:        String" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "value:       '" << value_ << "'" << endl;
}
#endif

//=============================================================================

BlobNodeImpl::BlobNodeImpl(weak_ptr<ImageFileImpl> destImageFile, int64_t byteCount)
: NodeImpl(destImageFile)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    shared_ptr<ImageFileImpl> imf(destImageFile);

    /// This what caller thinks blob length is
    blobLogicalLength_ = byteCount;

    /// Round segment length up to multiple of 4 bytes
    binarySectionLogicalLength_ = sizeof(BlobSectionHeader) + blobLogicalLength_;
    unsigned remainder = binarySectionLogicalLength_ % 4;
    if (remainder > 0)
        binarySectionLogicalLength_ += 4 - remainder;

    /// Reserve space for blob in file, extend with zeros since writes will happen at later time by caller
    binarySectionLogicalStart_ = imf->allocateSpace(binarySectionLogicalLength_, true);

    /// Prepare BlobSectionHeader
    BlobSectionHeader header;
    memset(&header, 0, sizeof(header));  /// need to init to zero, ok since no constructor
    header.sectionId = E57_BLOB_SECTION;
    header.sectionLogicalLength = binarySectionLogicalLength_;
#ifdef E57_MAX_VERBOSE
    header.dump(); //???
#endif
    header.swab();  /// swab if neccesary

    /// Write header at beginning of section
    imf->file_->seek(binarySectionLogicalStart_);
    imf->file_->write(reinterpret_cast<char*>(&header), sizeof(header));
}

BlobNodeImpl::BlobNodeImpl(weak_ptr<ImageFileImpl> destImageFile, int64_t fileOffset, int64_t length)
: NodeImpl(destImageFile)
{
    /// Init blob object that already exists in E57 file currently reading.

    // don't checkImageFileOpen, NodeImpl() will do it

    shared_ptr<ImageFileImpl> imf(destImageFile);

    /// Init state from values read from XML
    blobLogicalLength_ = length;
    binarySectionLogicalStart_ = imf->file_->physicalToLogical(fileOffset);
    binarySectionLogicalLength_ = sizeof(BlobSectionHeader) + blobLogicalLength_;
}

NodeType BlobNodeImpl::type() const
{
    /// don't checkImageFileOpen
    return E57_BLOB;
}

bool BlobNodeImpl::isTypeEquivalent(shared_ptr<NodeImpl> ni)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Same node type?
    if (ni->type() != E57_BLOB)
        return(false);

    /// Downcast to shared_ptr<BlobNodeImpl>
    shared_ptr<BlobNodeImpl> bi(dynamic_pointer_cast<BlobNodeImpl>(ni));
    if (!bi)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// blob lengths must match
    if (blobLogicalLength_ != bi->blobLogicalLength_)
        return(false);

    /// ignore blob contents, doesn't have to match

    /// Types match
    return(true);
}

bool BlobNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// We have no sub-structure, so if path not empty return false
    return(pathName == "");
}

int64_t BlobNodeImpl::byteCount()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(blobLogicalLength_);
}

void BlobNodeImpl::read(uint8_t* buf, int64_t start, size_t count)
{
    //??? check start not negative

    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    if (static_cast<uint64_t>(start)+count > blobLogicalLength_) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "this->pathName=" + this->pathName()
                             + " start=" + toString(start)
                             + " count=" + toString(count)
                             + " length=" + toString(blobLogicalLength_));
    }
    shared_ptr<ImageFileImpl> imf(destImageFile_);
    imf->file_->seek(binarySectionLogicalStart_ + sizeof(BlobSectionHeader) + start);
    imf->file_->read(reinterpret_cast<char*>(buf), static_cast<size_t>(count));  //??? arg1 void* ?
}

void BlobNodeImpl::write(uint8_t* buf, int64_t start, size_t count)
{
    //??? check start not negative
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    shared_ptr<ImageFileImpl> destImageFile(destImageFile_);

    if (!destImageFile->isWriter())
        throw E57_EXCEPTION2(E57_ERROR_FILE_IS_READ_ONLY, "fileName=" + destImageFile->fileName());
    if (!isAttached())
        throw E57_EXCEPTION2(E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName());

    if (static_cast<uint64_t>(start)+count > blobLogicalLength_) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "this->pathName=" + this->pathName()
                             + " start=" + toString(start)
                             + " count=" + toString(count)
                             + " length=" + toString(blobLogicalLength_));
    }

    shared_ptr<ImageFileImpl> imf(destImageFile_);
    imf->file_->seek(binarySectionLogicalStart_ + sizeof(BlobSectionHeader) + start);
    imf->file_->write(reinterpret_cast<char*>(buf), static_cast<size_t>(count));  //??? arg1 void* ?
}

void BlobNodeImpl::checkLeavesInSet(const std::set<ustring>& pathNames, shared_ptr<NodeImpl> origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set. ???true for blobs? what exception get if try blob in compressedvector?
    if (pathNames.find(relativePathName(origin)) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void BlobNodeImpl::writeXml(std::shared_ptr<ImageFileImpl> /*imf*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    //??? need to implement
    //??? Type --> type
    //??? need to have length?, check same as in section header?
    uint64_t physicalOffset = cf.logicalToPhysical(binarySectionLogicalStart_);
    cf << space(indent) << "<" << fieldName << " type=\"Blob\" fileOffset=\"" << physicalOffset << "\" length=\"" << blobLogicalLength_ << "\"/>\n";
}

#ifdef E57_DEBUG
void BlobNodeImpl::dump(int indent, ostream& os)
{
    // don't checkImageFileOpen
    os << space(indent) << "type:        Blob" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "blobLogicalLength_:           " << blobLogicalLength_ << endl;
    os << space(indent) << "binarySectionLogicalStart:    " << binarySectionLogicalStart_ << endl;
    os << space(indent) << "binarySectionLogicalLength:   " << binarySectionLogicalLength_ << endl;
    size_t i;
    for (i = 0; i < blobLogicalLength_ && i < 10; i++) {
        uint8_t b;
        read(&b, i, 1);
        os << space(indent) << "data[" << i << "]: "<< static_cast<int>(b) << endl;
    }
    if (i < blobLogicalLength_)
        os << space(indent) << "more data unprinted..." << endl;
}
#endif

//=============================================================================
//=============================================================================
//=============================================================================

ImageFileImpl::ImageFileImpl( ReadChecksumPolicy policy )
: isWriter_( false ),
  writerCount_(0),
  readerCount_(0),
  checksumPolicy( std::max( 0, std::min( policy, 100 ) ) ),
  file_(nullptr),
  xmlLogicalOffset_( 0 ),
  xmlLogicalLength_( 0 ),
  unusedLogicalStart_( 0 )
{
    /// First phase of construction, can't do much until have the ImageFile object.
    /// See ImageFileImpl::construct2() for second phase.
}

void ImageFileImpl::construct2(const ustring& fileName, const ustring& mode)
{
    /// Second phase of construction, now we have a well-formed ImageFile object.

#ifdef E57_MAX_VERBOSE
    cout << "ImageFileImpl() called, fileName=" << fileName << " mode=" << mode << endl;
#endif
    unusedLogicalStart_ = sizeof(E57FileHeader);
    fileName_ = fileName;

    /// Get shared_ptr to this object
    shared_ptr<ImageFileImpl> imf=shared_from_this();

    //??? allow "rw" or "a"?
    if (mode == "w")
        isWriter_ = true;
    else if (mode == "r")
        isWriter_ = false;
    else
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT, "mode=" + ustring(mode));

    /// If mode is read, do it
    file_ = nullptr;
    if (!isWriter_) {
        try { //??? should one try block cover whole function?
            /// Open file for reading.
            file_ = new CheckedFile( fileName_, CheckedFile::ReadOnly, checksumPolicy );

            shared_ptr<StructureNodeImpl> root(new StructureNodeImpl(imf));
            root_ = root;
            root_->setAttachedRecursive();

            E57FileHeader header;
            readFileHeader(file_, header);

            ///!!! stash major,minor numbers for API?
            xmlLogicalOffset_ = file_->physicalToLogical(header.xmlPhysicalOffset);
            xmlLogicalLength_ = header.xmlLogicalLength;
        } catch (...) {
            /// Remember to close file if got any exception
            if (file_ != nullptr) {
                delete file_;
                file_ = nullptr;
            }
            throw;  // rethrow
        }

        SAX2XMLReader* xmlReader = nullptr;

        // Initialize the XML4C2 system
        try {
             XMLPlatformUtils::Initialize();
        } catch (const XMLException& ex) {
            /// Turn parser exception into E57Exception
             throw E57_EXCEPTION2(E57_ERROR_XML_PARSER_INIT, "parserMessage=" + ustring(XMLString::transcode(ex.getMessage())));
        }

        xmlReader = XMLReaderFactory::createXMLReader(); //??? auto_ptr?

        if ( xmlReader == nullptr )
        {
           throw E57_EXCEPTION2( E57_ERROR_XML_PARSER_INIT, "could not create the xml reader" );
        }

        //??? check these are right
        xmlReader->setFeature(XMLUni::fgSAX2CoreValidation,        true);
        xmlReader->setFeature(XMLUni::fgXercesDynamic,             true);
        xmlReader->setFeature(XMLUni::fgSAX2CoreNameSpaces,        true);
        xmlReader->setFeature(XMLUni::fgXercesSchema,              true);
        xmlReader->setFeature(XMLUni::fgXercesSchemaFullChecking,  true);
        xmlReader->setFeature(XMLUni::fgSAX2CoreNameSpacePrefixes, true);

        try {
            /// Create parser state, attach its event handers to the SAX2 reader
            E57XmlParser parser(imf);
            xmlReader->setContentHandler(&parser);
            xmlReader->setErrorHandler(&parser);

            /// Create input source (XML section of E57 file turned into a stream).
            E57FileInputSource xmlSection(file_, xmlLogicalOffset_, xmlLogicalLength_);

            unusedLogicalStart_ = sizeof(E57FileHeader);

            /// Do the parse, building up the node tree
            xmlReader->parse(xmlSection);

        } catch (...) {
            if (xmlReader != nullptr) {
                delete xmlReader;
                xmlReader = nullptr;
            }
            if (file_ != nullptr) {
                delete file_;
                file_ = nullptr;
            }
            throw;  // rethrow
        }
        delete xmlReader;

        XMLPlatformUtils::Terminate();

    } else { /// open for writing (start empty)
        try {
            /// Open file for writing, truncate if already exists.
            file_ = new CheckedFile( fileName_, CheckedFile::WriteCreate, checksumPolicy );

            shared_ptr<StructureNodeImpl> root(new StructureNodeImpl(imf));
            root_ = root;
            root_->setAttachedRecursive();

            unusedLogicalStart_ = sizeof(E57FileHeader);
            xmlLogicalOffset_ = 0;
            xmlLogicalLength_ = 0;

        } catch (...) {
            /// Remember to close file if got any exception
            if (file_ != nullptr) {
                delete file_;
                file_ = nullptr;
            }
            throw;  // rethrow
        }
    }
}

void ImageFileImpl::readFileHeader(CheckedFile* file, E57FileHeader& header)
{
#ifdef E57_DEBUG
    /// Double check that compiler thinks sizeof header is what it is supposed to be
    if (sizeof(E57FileHeader) != 48)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "headerSize=" + toString(sizeof(E57FileHeader)));
#endif

    /// Fetch the file header
    file->read(reinterpret_cast<char*>(&header), sizeof(header));
#ifdef E57_MAX_VERBOSE
    header.dump(); //???
#endif
    header.swab();  /// swab if neccesary

    /// Check signature
    if (strncmp(header.fileSignature, "ASTM-E57", 8) != 0)
        throw E57_EXCEPTION2(E57_ERROR_BAD_FILE_SIGNATURE, "fileName="+file->fileName());

    /// Check file version compatibility
    if (header.majorVersion > E57_FORMAT_MAJOR) {
        throw E57_EXCEPTION2(E57_ERROR_UNKNOWN_FILE_VERSION,
                             "fileName=" + file->fileName()
                             + " header.majorVersion=" + toString(header.majorVersion)
                             + " header.minorVersion=" + toString(header.minorVersion));
    }

    /// If is a prototype version (majorVersion==0), then minorVersion has to match too.
    /// In production versions (majorVersion==E57_FORMAT_MAJOR), should be able to handle any minor version.
    if (header.majorVersion == E57_FORMAT_MAJOR &&
        header.minorVersion > E57_FORMAT_MINOR) {
        throw E57_EXCEPTION2(E57_ERROR_UNKNOWN_FILE_VERSION,
                             "fileName=" + file->fileName()
                             + " header.majorVersion=" + toString(header.majorVersion)
                             + " header.minorVersion=" + toString(header.minorVersion));
    }

    /// Check if file length matches actual physical length
    if (header.filePhysicalLength != file->length(CheckedFile::Physical)) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_FILE_LENGTH,
                             "fileName=" + file->fileName()
                             + " header.filePhysicalLength=" + toString(header.filePhysicalLength)
                             + " file->length=" + toString(file->length(CheckedFile::Physical)));
    }

    /// Check that page size is correct constant
    if (header.majorVersion != 0 &&
        header.pageSize != CheckedFile::physicalPageSize)
        throw E57_EXCEPTION2(E57_ERROR_BAD_FILE_LENGTH, "fileName=" + file->fileName());
}

void ImageFileImpl::incrWriterCount()
{
    writerCount_++;
}

void ImageFileImpl::decrWriterCount()
{
    writerCount_--;
#ifdef E57_MAX_DEBUG
    if (writerCount_ < 0) {
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "fileName=" + fileName_
                             + " writerCount=" + toString(writerCount_)
                             + " readerCount=" + toString(readerCount_));
    }
#endif
}

void ImageFileImpl::incrReaderCount()
{
    readerCount_++;
}

void ImageFileImpl::decrReaderCount()
{
    readerCount_--;
#ifdef E57_MAX_DEBUG
    if (readerCount_ < 0) {
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "fileName=" + fileName_
                             + " writerCount=" + toString(writerCount_)
                             + " readerCount=" + toString(readerCount_));
    }
#endif
}

shared_ptr<StructureNodeImpl> ImageFileImpl::root()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(root_);
}

void ImageFileImpl::close()
{
    //??? check if already closed
    //??? flush, close

    /// If file already closed, have nothing to do
    if (file_ == nullptr)
        return;

    if (isWriter_) {
        /// Go to end of file, note physical position
        xmlLogicalOffset_ = unusedLogicalStart_;
        file_->seek(xmlLogicalOffset_, CheckedFile::Logical);
        uint64_t xmlPhysicalOffset = file_->position(CheckedFile::Physical);
        *file_ << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
#ifdef E57_OXYGEN_SUPPORT //???
//???        *file_ << "<?oxygen RNGSchema=\"file:/C:/kevin/astm/DataFormat/xif/las_v0_05.rnc\" type=\"compact\"?>\n";
#endif

        //??? need to add name space attributes to e57Root
        root_->writeXml(shared_from_this(), *file_, 0, "e57Root");

        /// Pad XML section so length is multiple of 4
        while ((file_->position(CheckedFile::Logical) - xmlLogicalOffset_) % 4 != 0)
            *file_ << " ";

        /// Note logical length
        xmlLogicalLength_ = file_->position(CheckedFile::Logical) - xmlLogicalOffset_;

        /// Init header contents
        E57FileHeader header;
        memset(&header, 0, sizeof(header));  /// need to init to zero, ok since no constructor
        memcpy(&header.fileSignature, "ASTM-E57", 8);
        header.majorVersion       = E57_FORMAT_MAJOR;
        header.minorVersion       = E57_FORMAT_MINOR;
        header.filePhysicalLength = file_->length(CheckedFile::Physical);
        header.xmlPhysicalOffset  = xmlPhysicalOffset;
        header.xmlLogicalLength   = xmlLogicalLength_;
        header.pageSize           = CheckedFile::physicalPageSize;
#ifdef E57_MAX_VERBOSE
        header.dump(); //???
#endif
        header.swab();  /// swab if neccesary

        /// Write header at beginning of file
        file_->seek(0);
        file_->write(reinterpret_cast<char*>(&header), sizeof(header));

        file_->close();
    }

    delete file_;
    file_ = nullptr;
}

void ImageFileImpl::cancel()
{
    /// If file already closed, have nothing to do
    if (file_ == nullptr)
        return;

    /// Close the file and ulink (delete) it.
    /// It is legal to cancel a read file, but file isn't deleted.
    if (isWriter_)
        file_->unlink();
    else
        file_->close();

    delete file_;
    file_ = nullptr;
}

bool ImageFileImpl::isOpen()
{
    return(file_ != nullptr);
}

bool ImageFileImpl::isWriter()
{
    return(isWriter_);
}

int ImageFileImpl::writerCount()
{
    return(writerCount_);
}

int ImageFileImpl::readerCount()
{
    return(readerCount_);
}

ImageFileImpl::~ImageFileImpl()
{
    /// Try to cancel if not already closed, but don't allow any exceptions to propogate to caller (because in dtor).
    /// If writing, this will unlink the file, so make sure call ImageFileImpl::close explicitly before dtor runs.
    try {
        cancel();
    } catch (...) {};

    /// Just in case cancel failed without freeing file_, do free here.
    if (file_ != nullptr) {
        delete file_;
        file_ = nullptr;
    }
}

uint64_t ImageFileImpl::allocateSpace(uint64_t byteCount, bool doExtendNow)
{
    uint64_t oldLogicalStart = unusedLogicalStart_;

    /// Reserve space at end of file
    unusedLogicalStart_ += byteCount;

    /// If caller won't write to file immediately, it should request that the file be extended with zeros here
    if (doExtendNow)
        file_->extend(unusedLogicalStart_);

    return(oldLogicalStart);
}

CheckedFile* ImageFileImpl::file()
{
    return(file_);
}

ustring ImageFileImpl::fileName()
{
    // don't checkImageFileOpen, since need to get fileName to report not open
    return(fileName_);
}

void ImageFileImpl::extensionsAdd(const ustring& prefix, const ustring& uri)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    //??? check if prefix characters ok, check if uri has a double quote char (others?)

    /// Check to make sure that neither prefix or uri is already defined.
    ustring dummy;
    if (extensionsLookupPrefix(prefix, dummy))
        throw E57_EXCEPTION2(E57_ERROR_DUPLICATE_NAMESPACE_PREFIX, "prefix=" + prefix + " uri=" + uri);
    if (extensionsLookupUri(uri, dummy))
        throw E57_EXCEPTION2(E57_ERROR_DUPLICATE_NAMESPACE_URI, "prefix=" + prefix + " uri=" + uri);;

    /// Append at end of list
    nameSpaces_.push_back(NameSpace(prefix, uri));
}

bool ImageFileImpl::extensionsLookupPrefix(const ustring& prefix, ustring& uri)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    /// Linear search for matching prefix
    vector<NameSpace>::iterator it;
    for (it = nameSpaces_.begin(); it < nameSpaces_.end(); ++it) {
        if (it->prefix == prefix) {
            uri = it->uri;
            return(true);
        }
    }
    return(false);
}

bool ImageFileImpl::extensionsLookupUri(const ustring& uri, ustring& prefix)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    /// Linear search for matching URI
    vector<NameSpace>::iterator it;
    for (it = nameSpaces_.begin(); it < nameSpaces_.end(); ++it) {
        if (it->uri == uri) {
            prefix = it->prefix;
            return(true);
        }
    }
    return(false);
}

size_t ImageFileImpl::extensionsCount()
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(nameSpaces_.size());
}

ustring ImageFileImpl::extensionsPrefix(const size_t index)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(nameSpaces_[index].prefix);  //??? throw e57 exception here if out of bounds?
}

ustring ImageFileImpl::extensionsUri(const size_t index)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    return(nameSpaces_[index].uri);  //??? throw e57 exception here if out of bounds?
}

bool ImageFileImpl::isElementNameExtended(const ustring& elementName)
{
    /// don't checkImageFileOpen

    /// Make sure doesn't have any "/" in it
    size_t found = elementName.find_first_of('/');
    if (found != string::npos)
        return(false);

    ustring prefix, localPart;
    try {
        /// Throws if elementName bad
        elementNameParse(elementName, prefix, localPart);
    } catch(E57Exception& /*ex*/) {
        return(false);
    }

    /// If get here, the name was good, so test if found a prefix part
    return(prefix.length() > 0);
}

bool ImageFileImpl::isElementNameLegal(const ustring& elementName, bool allowNumber)
{
#ifdef E57_MAX_VERBOSE
    //cout << "isElementNameLegal elementName=""" << elementName << """" << endl;
#endif
    try {
        checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

        /// Throws if elementName bad
        checkElementNameLegal(elementName, allowNumber);
    } catch(E57Exception& /*ex*/) {
        return(false);
    }

    /// If get here, the name was good
    return(true);
}

bool ImageFileImpl::isPathNameLegal(const ustring& pathName)
{
#ifdef E57_MAX_VERBOSE
    //cout << "isPathNameLegal elementName=""" << pathName << """" << endl;
#endif
    try {
        checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

        /// Throws if pathName bad
        pathNameCheckWellFormed(pathName);
    } catch(E57Exception& /*ex*/) {
        return(false);
    }

    /// If get here, the name was good
    return(true);
}

void ImageFileImpl::checkElementNameLegal(const ustring& elementName, bool allowNumber)
{
    /// no checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__)

    ustring prefix, localPart;

    /// Throws if bad elementName
    elementNameParse(elementName, prefix, localPart, allowNumber);

    /// If has prefix, it must be registered
    ustring uri;
    if (prefix.length() > 0 && !extensionsLookupPrefix(prefix, uri))
        throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "elementName=" + elementName + " prefix=" + prefix);
}

void ImageFileImpl::elementNameParse(const ustring& elementName, ustring& prefix, ustring& localPart, bool allowNumber)
{
    /// no checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__)

    //??? check if elementName is good UTF-8?

    size_t len = elementName.length();

    /// Empty name is bad
    if (len == 0)
        throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "elementName=" + elementName);
    unsigned char c = elementName[0];

    /// If allowing numeric element name, check if first char is digit
    if (allowNumber && '0'<=c && c<='9') {
        /// All remaining characters must be digits
        for (size_t i = 1; i < len; i++) {
            c = elementName[i];
            if (!('0'<=c && c<='9'))
                throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "elementName=" + elementName);
        }
        return;
    }

    /// If first char is ASCII (< 128), check for legality
    /// Don't test any part of a multi-byte code point sequence (c >= 128).
    /// Don't allow ':' as first char.
    if (c<128 && !(('a'<=c && c<='z') || ('A'<=c && c<='Z') || c=='_'))
        throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "elementName=" + elementName);

    /// If each following char is ASCII (<128), check for legality
    /// Don't test any part of a multi-byte code point sequence (c >= 128).
    for (size_t i = 1; i < len; i++) {
        c = elementName[i];
        if (c<128 && !(('a'<=c && c<='z') || ('A'<=c && c<='Z') || c=='_' || c==':' || ('0'<=c && c<='9') || c=='-' || c=='.'))
            throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "elementName=" + elementName);
    }

    /// Check if has at least one colon, try to split it into prefix & localPart
    size_t found = elementName.find_first_of(':');
    if (found != string::npos) {
        /// Check doesn't have two colons
        if (elementName.find_first_of(':', found+1) != string::npos)
            throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "elementName=" + elementName);

        /// Split element name at the colon
        /// ??? split before check first/subsequent char legal?
        prefix    = elementName.substr(0, found);
        localPart = elementName.substr(found+1);

        if (prefix.length() == 0 || localPart.length() == 0) {
            throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME,
                                 "elementName=" + elementName +
                                 " prefix=" + prefix +
                                 " localPart=" + localPart);
        }
    } else {
        prefix = "";
        localPart = elementName;
    }
}

void ImageFileImpl::pathNameCheckWellFormed(const ustring& pathName)
{
    /// no checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__)

    /// Just call pathNameParse() which throws if not well formed
    bool isRelative;
    vector<ustring> fields;
    pathNameParse(pathName, isRelative, fields);
}

void ImageFileImpl::pathNameParse(const ustring& pathName, bool& isRelative, vector<ustring>& fields)
{
#ifdef E57_MAX_VERBOSE
    cout << "pathNameParse pathname=""" << pathName << """" << endl;
#endif
    /// no checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__)

    /// Clear previous contents of fields vector
    fields.clear();

    size_t start = 0;

    /// Check if absolute path
    if (pathName[start] == '/') {
        isRelative = false;
        start = 1;
    } else
        isRelative = true;

    /// Save strings in between each forward slash '/'
    /// Don't ignore whitespace
    while (start < pathName.size()) {
        size_t slash = pathName.find_first_of('/', start);

        /// Get element name from in between '/', check valid
        ustring elementName = pathName.substr(start, slash-start);
        if (!isElementNameLegal(elementName))
            throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "pathName=" + pathName + " elementName=" + elementName);

        /// Add to list
        fields.push_back(elementName);

        if (slash == string::npos)
            break;

        /// Handle case when pathname ends in /, e.g. "/foo/", add empty field at end of list
        if (slash == pathName.size()-1) {
            fields.push_back("");
            break;
        }

        /// Skip over the slash and keep going
        start = slash + 1;
    }

    /// Empty relative path is not allowed
    if (isRelative && fields.size() == 0)
        throw E57_EXCEPTION2(E57_ERROR_BAD_PATH_NAME, "pathName=" + pathName);

#ifdef E57_MAX_VERBOSE
    cout << "pathNameParse returning: isRelative=" << isRelative << " fields.size()=" << fields.size() << " fields=";
    for (int i = 0; i < fields.size(); i++)
        cout << fields[i] << ",";
    cout << endl;
#endif
}

ustring ImageFileImpl::pathNameUnparse(bool isRelative, const vector<ustring>& fields)
{
    /// no checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__)

    ustring path;

    if (!isRelative)
        path.push_back('/');
    for (unsigned i = 0; i < fields.size(); i++) {
        path.append(fields.at(i));
        if (i < fields.size()-1)
            path.push_back('/');
    }
    return(path);
}

void ImageFileImpl::checkImageFileOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName)
{
    if (!isOpen()) {
        throw E57Exception(E57_ERROR_IMAGEFILE_NOT_OPEN,
                           "fileName=" + fileName(),
                           srcFileName,
                           srcLineNumber,
                           srcFunctionName);
    }
}

void ImageFileImpl::dump(int indent, ostream& os)
{
    /// no checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__)
    os << space(indent) << "fileName:    " << fileName_ << endl;
    os << space(indent) << "writerCount: " << writerCount_ << endl;
    os << space(indent) << "readerCount: " << readerCount_ << endl;
    os << space(indent) << "isWriter:    " << isWriter_ << endl;
    for (size_t i=0; i < extensionsCount(); i++)
        os << space(indent) << "nameSpace[" << i << "]: prefix=" << extensionsPrefix(i) << " uri=" << extensionsUri(i) << endl;
    os << space(indent) << "root:      " << endl;
    root_->dump(indent+2, os);
}

unsigned ImageFileImpl::bitsNeeded(int64_t minimum, int64_t maximum)
{
    /// Relatively quick way to compute ceil(log2(maximum - minimum + 1)));
    /// Uses only integer operations and is machine independent (no assembly code).
    /// Find the bit position of the first 1 (from left) in the binary form of stateCountMinus1.
    ///??? move to E57Utility?

    uint64_t stateCountMinus1 = maximum - minimum;
    unsigned log2 = 0;
    if (stateCountMinus1 & 0xFFFFFFFF00000000LL) {
        stateCountMinus1 >>= 32;
        log2 += 32;
    }
    if (stateCountMinus1 & 0xFFFF0000LL) {
        stateCountMinus1 >>= 16;
        log2 += 16;
    }
    if (stateCountMinus1 & 0xFF00LL) {
        stateCountMinus1 >>= 8;
        log2 += 8;
    }
    if (stateCountMinus1 & 0xF0LL) {
        stateCountMinus1 >>= 4;
        log2 += 4;
    }
    if (stateCountMinus1 & 0xCLL) {
        stateCountMinus1 >>= 2;
        log2 += 2;
    }
    if (stateCountMinus1 & 0x2LL) {
        stateCountMinus1 >>= 1;
        log2 += 1;
    }
    if (stateCountMinus1 & 1LL)
        log2++;
    return(log2);
}

#ifdef BITSNEEDED_UNIT_TEST

void test1(int64_t minimum, int64_t maximum)
{
#ifdef E57_MAX_VERBOSE
    cout << "bitsNeeded(" << minimum << "," << maximum << ") = " << bitsNeeded(minimum, maximum) << endl;
#endif
}

void main()
{
    test1(0, 0);
    test1(0, 1);
    test1(0, 2);
    test1(0, 3);
    test1(0, 4);
    test1(0, 5);
    test1(0, 6);
    test1(0, 7);
    test1(0, 8);
    cout << endl;

    test1(1, 1);
    test1(1, 2);
    test1(-128, 127);
    cout << endl;

    test1(E57_INT8_MIN, E57_INT8_MAX);
    test1(E57_INT16_MIN, E57_INT16_MAX);
    test1(E57_INT32_MIN, E57_INT32_MAX);
    test1(E57_INT64_MIN, E57_INT64_MAX);
    cout << endl;

    for (int i=0; i < 64; i++) {
        if (bitsNeeded(0, 1LL<<i) != i+1) {
            cout << "OOPS: i=" << i << endl;
            exit(-1);
        }
    }

    cout << endl;
    for (int i=0; i < 64; i++)
        test1(0, 3LL<<i);
    cout << endl;
    for (int i=0; i < 64; i++)
        test1(0, 5LL<<i);
}

#endif

//================================================================

//=============================================================
#ifdef UNIT_TEST

using namespace e57;

void printState(CheckedFile& cf)
{
    cout << "  Current logical file position:  " << cf.position(CheckedFile::logical) << endl;
    cout << "  Current physical file position: " << cf.position(CheckedFile::physical)<< endl;
    cout << "  Current logical file length:    " << cf.length(CheckedFile::logical) << endl;
    cout << "  Current physical file length:   " << cf.length(CheckedFile::physical) << endl;
}

int main()
{
    try {
        CheckedFile cf("checked_file_test._e57", CheckedFile::writeCreate);

        printState(cf);

        cout << "Writing 'hey'" << endl;
        cf.write("hey", 3);

        printState(cf);

        cout << "Writing ' you'" << endl;
        cf.write(" you", 4);

        printState(cf);

#if 0
        for (int i=0; i < 11; i++) {
            cout << "Writing ' yada yada...'" << endl;
            cf.write(" yada yada yada yada yada yada yada yada yada yada yada yada yada yada yada yada yada yada yada yada", 100);
        }

        printState(cf);

        uint64_t n = 2035;
        for (int i=0; i < 10; i++) {
            cout << "Extending to " << n << " bytes" << endl;
            cf.extend(n);
            printState(cf);
            n++;
        }
#else
        cout << "Extending to " << 1016 << " bytes" << endl;
        cf.extend(1016);

        for (int i=0; i < 6; i++) {
            cout << "Writing 'a'" << endl;
            char c = 'a' + i;
            cf.write(&c, 1);
            printState(cf);
        }
#endif
        cf.seek(0);
        char buf[8];
        cf.read(buf, sizeof(buf)-1);
        buf[sizeof(buf)-1] = '\0';
        cout << "Read: '" << buf << "'" << endl;
    } catch(E57Exception& ex) {
        ex.report(__FILE__, __LINE__, __FUNCTION__);
    } catch (std::exception& ex) {
        cerr << "Got an std::exception, what=" << ex.what() << endl;
    } catch (...) {
        cerr << "Got an unknown exception" << endl;
    }

    return(0);
}

#endif

//================================================================

CompressedVectorSectionHeader::CompressedVectorSectionHeader()
{
    /// Double check that header is correct length.  Watch out for RTTI increasing the size.
    if (sizeof(*this) != 32)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "size=" + toString(sizeof(*this)));

    /// Now confident we have correct size, zero header.
    /// This guarantees that headers are always completely initialized to zero.
    memset(this, 0, sizeof(*this));
}

void CompressedVectorSectionHeader::verify(uint64_t filePhysicalSize)
{
    /// Verify that section is correct type
    if (sectionId != E57_COMPRESSED_VECTOR_SECTION)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER, "sectionId=" + toString(sectionId));

    /// Verify reserved fields are zero. ???  if fileversion==1.0 ???
    for (unsigned i=0; i < sizeof(reserved1); i++) {
        if (reserved1[i] != 0)
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER, "i=" + toString(i) + " reserved=" + toString(reserved1[i]));
    }

    /// Check section length is multiple of 4
    if (sectionLogicalLength % 4)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER, "sectionLogicalLength=" + toString(sectionLogicalLength));

    /// Check sectionLogicalLength is in bounds
    if (filePhysicalSize > 0 && sectionLogicalLength >= filePhysicalSize) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER,
                             "sectionLogicalLength=" + toString(sectionLogicalLength)
                             + " filePhysicalSize=" + toString(filePhysicalSize));
    }

    /// Check dataPhysicalOffset is in bounds
    if (filePhysicalSize > 0 && dataPhysicalOffset >= filePhysicalSize) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER,
                             "dataPhysicalOffset=" + toString(dataPhysicalOffset)
                             + " filePhysicalSize=" + toString(filePhysicalSize));
    }

    /// Check indexPhysicalOffset is in bounds
    if (filePhysicalSize > 0 && indexPhysicalOffset >= filePhysicalSize) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER,
                             "indexPhysicalOffset=" + toString(indexPhysicalOffset)
                             + " filePhysicalSize=" + toString(filePhysicalSize));
    }
}

#ifdef E57_BIGENDIAN
void CompressedVectorSectionHeader::swab()
{
    /// Byte swap fields in-place, if CPU is BIG_ENDIAN
    swab(&sectionLogicalLength);
    swab(&dataPhysicalOffset);
    swab(&indexPhysicalOffset);
};
#endif

#ifdef E57_DEBUG
void CompressedVectorSectionHeader::dump(int indent, std::ostream& os)
{
    os << space(indent) << "sectionId:            " << static_cast<unsigned>(sectionId) << endl;
    os << space(indent) << "sectionLogicalLength: " << sectionLogicalLength << endl;
    os << space(indent) << "dataPhysicalOffset:   " << dataPhysicalOffset << endl;
    os << space(indent) << "indexPhysicalOffset:  " << indexPhysicalOffset << endl;
}
#endif

///================================================================

DataPacketHeader::DataPacketHeader()
{
    /// Double check that packet struct is correct length.  Watch out for RTTI increasing the size.
    if (sizeof(*this) != 6)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "size=" + toString(sizeof(*this)));

    /// Now confident we have correct size, zero packet.
    /// This guarantees that data packet headers are always completely initialized to zero.
    memset(this, 0, sizeof(*this));
}

void DataPacketHeader::verify(unsigned bufferLength) const
{
    /// Verify that packet is correct type
    if (packetType != E57_DATA_PACKET)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetType=" + toString(packetType));

    /// ??? check reserved flags zero?

    /// Check packetLength is at least large enough to hold header
    unsigned packetLength = packetLogicalLengthMinus1+1;
    if (packetLength < sizeof(*this))
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetLength=" + toString(packetLength));

    /// Check packet length is multiple of 4
    if (packetLength % 4)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetLength=" + toString(packetLength));

    /// Check actual packet length is large enough.
    if (bufferLength > 0 && packetLength > bufferLength) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                             "packetLength=" + toString(packetLength)
                             + " bufferLength=" + toString(bufferLength));
    }

    /// Make sure there is at least one entry in packet  ??? 0 record cvect allowed?
    if (bytestreamCount == 0)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "bytestreamCount=" + toString(bytestreamCount));

    /// Check packet is at least long enough to hold bytestreamBufferLength array
    if (sizeof(DataPacketHeader) + 2*bytestreamCount > packetLength) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                             "packetLength=" + toString(packetLength)
                             + " bytestreamCount=" + toString(bytestreamCount));
    }
}

#ifdef E57_BIGENDIAN
void DataPacketHeader::swab()
{
    /// Byte swap fields in-place, if CPU is BIG_ENDIAN
    swab(&packetLogicalLengthMinus1);
    swab(&bytestreamCount);
};
#endif

#ifdef E57_DEBUG
void DataPacketHeader::dump(int indent, std::ostream& os) const
{
    os << space(indent) << "packetType:                " << static_cast<unsigned>(packetType) << endl;
    os << space(indent) << "packetFlags:               " << static_cast<unsigned>(packetFlags) << endl;
    os << space(indent) << "packetLogicalLengthMinus1: " << packetLogicalLengthMinus1 << endl;
    os << space(indent) << "bytestreamCount:           " << bytestreamCount << endl;
}
#endif

//================================================================

DataPacket::DataPacket()
{
    /// Double check that packet struct is correct length.  Watch out for RTTI increasing the size.
    if (sizeof(*this) != 64*1024)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "size=" + toString(sizeof(*this)));

    /// Now confident we have correct size, zero packet.
    /// This guarantees that data packets are always completely initialized to zero.
    memset(this, 0, sizeof(*this));
}

void DataPacket::verify(unsigned bufferLength) const
{
    //??? do all packets need versions?  how extend without breaking older checking?  need to check file version#?

    /// Verify header is good
    const DataPacketHeader* hp = reinterpret_cast<const DataPacketHeader*>(this);

    hp->verify(bufferLength);

    /// Calc sum of lengths of each bytestream buffer in this packet
    const uint16_t* bsbLength = reinterpret_cast<const uint16_t*>(&payload[0]);
    unsigned totalStreamByteCount = 0;

    for (unsigned i=0; i < bytestreamCount; i++)
    {
        totalStreamByteCount += bsbLength[i];
    }

    /// Calc size of packet needed
    const unsigned packetLength = packetLogicalLengthMinus1+1;
    const unsigned needed = sizeof(DataPacketHeader) + 2*bytestreamCount + totalStreamByteCount;
#ifdef E57_MAX_VERBOSE
    cout << "needed=" << needed << " actual=" << packetLength << endl; //???
#endif

    /// If needed is not with 3 bytes of actual packet size, have an error
    if (needed > packetLength || needed+3 < packetLength)
    {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                             "needed=" + toString(needed)
                             + "packetLength=" + toString(packetLength));
    }

    /// Verify that padding at end of packet is zero
    for (unsigned i=needed; i < packetLength; i++)
    {
        if (reinterpret_cast<const char*>(this)[i] != 0)
        {
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "i=" + toString(i));
        }
    }
}

char* DataPacket::getBytestream(unsigned bytestreamNumber, unsigned& byteCount)
{
#ifdef E57_MAX_VERBOSE
    cout << "getBytestream called, bytestreamNumber=" << bytestreamNumber << endl;
#endif

    /// Verify that packet is correct type
    if (packetType != E57_DATA_PACKET)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetType=" + toString(packetType));

    /// Check bytestreamNumber in bounds
    if (bytestreamNumber >= bytestreamCount) {
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "bytestreamNumber=" + toString(bytestreamNumber)
                             + "bytestreamCount=" + toString(bytestreamCount));
    }

    /// Calc positions in packet
    uint16_t* bsbLength = reinterpret_cast<uint16_t*>(&payload[0]);
    char* streamBase = reinterpret_cast<char*>(&bsbLength[bytestreamCount]);

    /// Sum size of preceeding stream buffers to get position
    unsigned totalPreceeding = 0;
    for (unsigned i=0; i < bytestreamNumber; i++)
        totalPreceeding += bsbLength[i];

    byteCount = bsbLength[bytestreamNumber];

    /// Double check buffer is completely within packet
    if (sizeof(DataPacketHeader) + 2*bytestreamCount + totalPreceeding + byteCount > packetLogicalLengthMinus1 + 1U) {
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "bytestreamCount=" + toString(bytestreamCount)
                             + " totalPreceeding=" + toString(totalPreceeding)
                             + " byteCount=" + toString(byteCount)
                             + " packetLogicalLengthMinus1=" + toString(packetLogicalLengthMinus1));
    }

    /// Return start of buffer
    return(&streamBase[totalPreceeding]);
}

unsigned DataPacket::getBytestreamBufferLength(unsigned bytestreamNumber)
{
    //??? for now:
    unsigned byteCount;
    (void) getBytestream(bytestreamNumber, byteCount);
    return(byteCount);
}

#ifdef E57_BIGENDIAN
DataPacket::swab(bool toLittleEndian)
{
    /// Be a little paranoid
    if (packetType != E57_INDEX_PACKET)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "packetType=" + toString(packetType));

    swab(packetLogicalLengthMinus1);

    /// Need to watch out if packet starts out in natural CPU ordering or not
    unsigned goodEntryCount;
    if (toLittleEndian) {
        /// entryCount starts out in correct order, save it before trashing
        goodEntryCount = entryCount;
        swab(entryCount);
    } else {
        /// Have to fix entryCount before can use.
        swab(entryCount);
        goodEntryCount = entryCount;
    }

    /// Make sure we wont go off end of buffer (e.g. if we accidentally swab)
    if (goodEntryCount > MAX_ENTRIES)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "goodEntryCount=" + toString(goodEntryCount));

    for (unsigned i=0; i < goodEntryCount; i++) {
        swab(entries[i].chunkRecordNumber);
        swab(entries[i].chunkPhysicalOffset);
    }
}
#endif

#ifdef E57_DEBUG
void DataPacket::dump(int indent, std::ostream& os) const
{
    if (packetType != E57_DATA_PACKET)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "packetType=" + toString(packetType));
    reinterpret_cast<const DataPacketHeader*>(this)->dump(indent, os);

    const uint16_t* bsbLength = reinterpret_cast<const uint16_t*>(&payload[0]);
    const uint8_t* p = reinterpret_cast<const uint8_t*>(&bsbLength[bytestreamCount]);

    for (unsigned i=0; i < bytestreamCount; i++)
    {
        os << space(indent) << "bytestream[" << i << "]:" << endl;
        os << space(indent+4) << "length: " << bsbLength[i] << endl;
/*====
        unsigned j;
        for (j=0; j < bsbLength[i] && j < 10; j++)
            os << space(indent+4) << "byte[" << j << "]=" << (unsigned)p[j] << endl;
        if (j < bsbLength[i])
            os << space(indent+4) << bsbLength[i]-j << " more unprinted..." << endl;
====*/
        p += bsbLength[i];
        if (p - reinterpret_cast<const uint8_t*>(this) > E57_DATA_PACKET_MAX)
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "size=" + toString(p - reinterpret_cast<const uint8_t*>(this)));
    }
}
#endif

//================================================================

IndexPacket::IndexPacket()
{
    /// Double check that packet struct is correct length.  Watch out for RTTI increasing the size.
    if (sizeof(*this) != 16+16*MAX_ENTRIES)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "size=" + toString(sizeof(*this)));

    /// Now confident we have correct size, zero packet.
    /// This guarantees that index packets are always completely initialized to zero.
    memset(this, 0, sizeof(*this));
}

void IndexPacket::verify(unsigned bufferLength, uint64_t totalRecordCount, uint64_t fileSize) const
{
    //??? do all packets need versions?  how extend without breaking older checking?  need to check file version#?

    /// Verify that packet is correct type
    if (packetType != E57_INDEX_PACKET)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetType=" + toString(packetType));

    /// Check packetLength is at least large enough to hold header
    unsigned packetLength = packetLogicalLengthMinus1+1;
    if (packetLength < sizeof(*this))
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetLength=" + toString(packetLength));

    /// Check packet length is multiple of 4
    if (packetLength % 4)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetLength=" + toString(packetLength));

    /// Make sure there is at least one entry in packet  ??? 0 record cvect allowed?
    if (entryCount == 0)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "entryCount=" + toString(entryCount));

    /// Have to have <= 2048 entries
    if (entryCount > MAX_ENTRIES)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "entryCount=" + toString(entryCount));

    /// Index level should be <= 5.  Because (5+1)* 11 bits = 66 bits, which will cover largest number of chunks possible.
    if (indexLevel > 5)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "indexLevel=" + toString(indexLevel));

    /// Index packets above level 0 must have at least two entries (otherwise no point to existing).
    ///??? check that this is in spec
    if (indexLevel > 0 && entryCount < 2)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "indexLevel=" + toString(indexLevel) + " entryCount=" + toString(entryCount));

    /// If not later version, verify reserved fields are zero. ??? test file version
    /// if (version <= E57_FORMAT_MAJOR) { //???
    for (unsigned i=0; i < sizeof(reserved1); i++) {
        if (reserved1[i] != 0)
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "i=" + toString(i));
    }

    /// Check actual packet length is large enough.
    if (bufferLength > 0 && packetLength > bufferLength) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                             "packetLength=" + toString(packetLength)
                             + " bufferLength=" + toString(bufferLength));
    }

    /// Check if entries will fit in space provided
    unsigned neededLength = 16 + 8*entryCount;
    if (packetLength < neededLength) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                             "packetLength=" + toString(packetLength)
                             + " neededLength=" + toString(neededLength));
    }

#ifdef E57_MAX_DEBUG
    /// Verify padding at end is zero.
    const char* p = reinterpret_cast<const char*>(this);
    for (unsigned i=neededLength; i < packetLength; i++) {
        if (p[i] != 0)
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "i=" + toString(i));
    }

    /// Verify records and offsets are in sorted order
    for (unsigned i=0; i < entryCount; i++) {
        /// Check chunkRecordNumber is in bounds
        if (totalRecordCount > 0 && entries[i].chunkRecordNumber >= totalRecordCount) {
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                                 "i=" + toString(i)
                                 + " chunkRecordNumber=" + toString(entries[i].chunkRecordNumber)
                                 + " totalRecordCount=" + toString(totalRecordCount));
        }

        /// Check record numbers are strictly increasing
        if (i > 0 && entries[i-1].chunkRecordNumber >= entries[i].chunkRecordNumber) {
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                                 "i=" + toString(i)
                                 + " prevChunkRecordNumber=" + toString(entries[i-1].chunkRecordNumber)
                                 + " currentChunkRecordNumber=" + toString(entries[i].chunkRecordNumber));
        }

        /// Check chunkPhysicalOffset is in bounds
        if (fileSize > 0 && entries[i].chunkPhysicalOffset >= fileSize) {
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                                 "i=" + toString(i)
                                 + " chunkPhysicalOffset=" + toString(entries[i].chunkPhysicalOffset)
                                 + " fileSize=" + toString(fileSize));
        }

        /// Check chunk offsets are strictly increasing
        if (i > 0 && entries[i-1].chunkPhysicalOffset >= entries[i].chunkPhysicalOffset) {
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                                 "i=" + toString(i)
                                 + " prevChunkPhysicalOffset=" + toString(entries[i-1].chunkPhysicalOffset)
                                 + " currentChunkPhysicalOffset=" + toString(entries[i].chunkPhysicalOffset));
        }
    }
#endif
}

#ifdef E57_BIGENDIAN
IndexPacket::swab(bool toLittleEndian)
{
    /// Be a little paranoid
    if (packetType != E57_INDEX_PACKET)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "packetType=" + toString(packetType));

    swab(packetLogicalLengthMinus1);

    /// Need to watch out if packet starts out in natural CPU ordering or not
    unsigned goodEntryCount;
    if (toLittleEndian) {
        /// entryCount starts out in correct order, save it before trashing
        goodEntryCount = entryCount;
        swab(entryCount);
    } else {
        /// Have to fix entryCount before can use.
        swab(entryCount);
        goodEntryCount = entryCount;
    }

    /// Make sure we wont go off end of buffer (e.g. if we accidentally swab)
    if (goodEntryCount > MAX_ENTRIES)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "goodEntryCount=" + toString(goodEntryCount));

    for (unsigned i=0; i < goodEntryCount; i++) {
        swab(entries[i].chunkRecordNumber);
        swab(entries[i].chunkPhysicalOffset);
    }
}
#endif

#ifdef E57_DEBUG
void IndexPacket::dump(int indent, std::ostream& os) const
{
    os << space(indent) << "packetType:                " << static_cast<unsigned>(packetType) << endl;
    os << space(indent) << "packetFlags:               " << static_cast<unsigned>(packetFlags) << endl;
    os << space(indent) << "packetLogicalLengthMinus1: " << packetLogicalLengthMinus1 << endl;
    os << space(indent) << "entryCount:                " << entryCount << endl;
    os << space(indent) << "indexLevel:                " << indexLevel << endl;
    unsigned i;
    for (i=0; i < entryCount && i < 10; i++) {
        os << space(indent) << "entry[" << i << "]:" << endl;
        os << space(indent+4) << "chunkRecordNumber:    " << entries[i].chunkRecordNumber << endl;
        os << space(indent+4) << "chunkPhysicalOffset:  " << entries[i].chunkPhysicalOffset << endl;
    }
    if (i < entryCount)
        os << space(indent) << entryCount-i << "more entries unprinted..." << endl;
}
#endif

///================================================================

EmptyPacketHeader::EmptyPacketHeader()
{
    /// Double check that packet struct is correct length.  Watch out for RTTI increasing the size.
    if (sizeof(*this) != 4)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "size=" + toString(sizeof(*this)));

    /// Now confident we have correct size, zero packet.
    /// This guarantees that EmptyPacket headers are always completely initialized to zero.
    memset(this, 0, sizeof(*this));
}

void EmptyPacketHeader::verify(unsigned bufferLength) const
{
    /// Verify that packet is correct type
    if (packetType != E57_EMPTY_PACKET)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetType=" + toString(packetType));

    /// Check packetLength is at least large enough to hold header
    unsigned packetLength = packetLogicalLengthMinus1+1;
    if (packetLength < sizeof(*this))
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetLength=" + toString(packetLength));

    /// Check packet length is multiple of 4
    if (packetLength % 4)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetLength=" + toString(packetLength));

    /// Check actual packet length is large enough.
    if (bufferLength > 0 && packetLength > bufferLength) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET,
                             "packetLength=" + toString(packetLength)
                             + " bufferLength=" + toString(bufferLength));
    }
}

#ifdef E57_BIGENDIAN
void EmptyPacketHeader::swab()
{
    /// Byte swap fields in-place, if CPU is BIG_ENDIAN
    SWAB(&packetLogicalLengthMinus1);
};
#endif

#ifdef E57_DEBUG
void EmptyPacketHeader::dump(int indent, std::ostream& os) const
{
    os << space(indent) << "packetType:                " << static_cast<unsigned>(packetType) << endl;
    os << space(indent) << "packetLogicalLengthMinus1: " << packetLogicalLengthMinus1 << endl;
}
#endif

///================================================================

struct SortByBytestreamNumber {
    bool operator () (shared_ptr<Encoder> lhs , shared_ptr<Encoder> rhs) const {
        return(lhs->bytestreamNumber() < rhs->bytestreamNumber());
    }
};

CompressedVectorWriterImpl::CompressedVectorWriterImpl(shared_ptr<CompressedVectorNodeImpl> ni, vector<SourceDestBuffer>& sbufs)
: cVector_(ni),
  isOpen_(false)  // set to true when succeed below
{
    //???  check if cvector already been written (can't write twice)

    /// Empty sbufs is an error
    if (sbufs.size() == 0) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "imageFileName=" + cVector_->imageFileName()
                             + " cvPathName=" + cVector_->pathName());
    }

    /// Get CompressedArray's prototype node (all array elements must match this type)
    proto_ = cVector_->getPrototype();

    /// Check sbufs well formed (matches proto exactly)
    setBuffers(sbufs); //??? copy code here?

    /// Zero dataPacket_ at start
    memset(&dataPacket_, 0, sizeof(dataPacket_));

    /// For each individual sbuf, create an appropriate Encoder based on the cVector_ attributes
    for (unsigned i=0; i < sbufs_.size(); i++) {
        /// Create vector of single sbuf  ??? for now, may have groups later
        vector<SourceDestBuffer> vTemp;
        vTemp.push_back(sbufs_.at(i));

        ustring codecPath = sbufs_.at(i).pathName();

        /// Calc which stream the given path belongs to.  This depends on position of the node in the proto tree.
        shared_ptr<NodeImpl> readNode = proto_->get(sbufs.at(i).pathName());
        uint64_t bytestreamNumber = 0;
        if (!proto_->findTerminalPosition(readNode, bytestreamNumber))
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "sbufIndex=" + toString(i));

        /// EncoderFactory picks the appropriate encoder to match type declared in prototype
        bytestreams_.push_back(Encoder::EncoderFactory(static_cast<unsigned>(bytestreamNumber), cVector_, vTemp, codecPath));
    }

    /// The bytestreams_ vector must be ordered by bytestreamNumber, not by order called specified sbufs, so sort it.
    sort(bytestreams_.begin(), bytestreams_.end(), SortByBytestreamNumber());
#ifdef E57_MAX_DEBUG
    /// Double check that all bytestreams are specified
    for (unsigned i=0; i < bytestreams_.size(); i++) {
        if (bytestreams_.at(i)->bytestreamNumber() != i){
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                                 "bytestreamIndex=" + toString(i)
                                 + " bytestreamNumber=" + toString(bytestreams_.at(i)->bytestreamNumber()));
        }
    }
#endif

    shared_ptr<ImageFileImpl> imf(ni->destImageFile_);

    /// Reserve space for CompressedVector binary section header, record location so can save to when writer closes.
    /// Request that file be extended with zeros since we will write to it at a later time (when writer closes).
    sectionHeaderLogicalStart_ = imf->allocateSpace(sizeof(CompressedVectorSectionHeader), true);

    sectionLogicalLength_   = 0;
    dataPhysicalOffset_     = 0;
    topIndexPhysicalOffset_ = 0;
    recordCount_            = 0;
    dataPacketsCount_       = 0;
    indexPacketsCount_      = 0;

    /// Just before return (and can't throw) increment writer count  ??? safer way to assure don't miss close?
    imf->incrWriterCount();

    /// If get here, the writer is open
    isOpen_ = true;
}

CompressedVectorWriterImpl::~CompressedVectorWriterImpl()
{
#ifdef E57_MAX_VERBOSE
    cout << "~CompressedVectorWriterImpl() called" << endl; //???
#endif

    try {
        if (isOpen_)
            close();
    } catch (...) {
        //??? report?
    }
}

void CompressedVectorWriterImpl::close()
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorWriterImpl::close() called" << endl; //???
#endif
    shared_ptr<ImageFileImpl> imf(cVector_->destImageFile_);

    /// Before anything that can throw, decrement writer count
    imf->decrWriterCount();

    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    /// don't call checkWriterOpen();

    if (!isOpen_)
        return;

    /// Set closed before do anything, so if get fault and start unwinding, don't try to close again.
    isOpen_ = false;

    /// If have any data, write packet
    /// Write all remaining ioBuffers and internal encoder register cache into file.
    /// Know we are done when totalOutputAvailable() returns 0 after a flush().
    flush();
    while (totalOutputAvailable() > 0) {
        packetWrite();
        flush();
    }

    /// Compute length of whole section we just wrote (from section start to current start of free space).
    sectionLogicalLength_ = imf->unusedLogicalStart_ - sectionHeaderLogicalStart_;
#ifdef E57_MAX_VERBOSE
    cout << "  sectionLogicalLength_=" << sectionLogicalLength_ << endl; //???
#endif

    /// Prepare CompressedVectorSectionHeader
    CompressedVectorSectionHeader header;
    header.sectionId            = E57_COMPRESSED_VECTOR_SECTION;
    header.sectionLogicalLength = sectionLogicalLength_;
    header.dataPhysicalOffset   = dataPhysicalOffset_;   ///??? can be zero, if no data written ???not set yet
    header.indexPhysicalOffset  = topIndexPhysicalOffset_;  ///??? can be zero, if no data written ???not set yet
#ifdef E57_MAX_VERBOSE
    cout << "  CompressedVectorSectionHeader:" << endl;
    header.dump(4); //???
#endif
#ifdef E57_DEBUG
    /// Verify OK before write it.
    header.verify(imf->file_->length(CheckedFile::Physical));
#endif
    header.swab();  /// swab if neccesary

    /// Write header at beginning of section, previously allocated
    imf->file_->seek(sectionHeaderLogicalStart_);
    imf->file_->write(reinterpret_cast<char*>(&header), sizeof(header));

    /// Set address and size of associated CompressedVector
    cVector_->setRecordCount(recordCount_);
    cVector_->setBinarySectionLogicalStart(sectionHeaderLogicalStart_);

    /// Free channels
    bytestreams_.clear();

#ifdef E57_MAX_VERBOSE
    cout << "  CompressedVectorWriter:" << endl;
    dump(4);
#endif
}

bool CompressedVectorWriterImpl::isOpen() const
{
    /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), or checkWriterOpen()
    return isOpen_;
}

std::shared_ptr<CompressedVectorNodeImpl> CompressedVectorWriterImpl::compressedVectorNode() const
{
    return cVector_;
}

void CompressedVectorWriterImpl::setBuffers(vector<SourceDestBuffer>& sbufs)
{
    /// don't checkImageFileOpen

    /// If had previous sbufs_, check to see if new ones have changed in incompatible way
    if (sbufs_.size() > 0) {
        if (sbufs_.size() != sbufs.size()) {
            throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                                 "oldSize=" + toString(sbufs_.size())
                                 + " newSize=" + toString(sbufs.size()));
        }
        for (size_t i = 0; i < sbufs_.size(); i++) {
            shared_ptr<SourceDestBufferImpl> oldbuf = sbufs_[i].impl();
            shared_ptr<SourceDestBufferImpl> newBuf = sbufs[i].impl();

            /// Throw exception if old and new not compatible
            oldbuf->checkCompatible(newBuf);
        }
    }

    /// Check sbufs well formed: no dups, no missing, no extra
    /// For writing, all data fields in prototype must be presented for writing at same time.
    proto_->checkBuffers(sbufs, false);

    sbufs_ = sbufs;
}

void CompressedVectorWriterImpl::write(vector<SourceDestBuffer>& sbufs, const size_t requestedRecordCount)
{
    /// don't checkImageFileOpen, write(unsigned) will do it
    /// don't checkWriterOpen(), write(unsigned) will do it

    setBuffers(sbufs);
    write(requestedRecordCount);
}

void CompressedVectorWriterImpl::write(const size_t requestedRecordCount)
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorWriterImpl::write() called" << endl; //???
#endif
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    checkWriterOpen(__FILE__, __LINE__, __FUNCTION__);

    /// Check that requestedRecordCount is not larger than the sbufs
    if (requestedRecordCount > sbufs_.at(0).impl()->capacity()) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "requested=" + toString(requestedRecordCount)
                             + " capacity=" + toString(sbufs_.at(0).impl()->capacity())
                             + " imageFileName=" + cVector_->imageFileName()
                             + " cvPathName=" + cVector_->pathName());
    }

    /// Rewind all sbufs so start reading from beginning
    for (unsigned i=0; i < sbufs_.size(); i++)
        sbufs_.at(i).impl()->rewind();

    /// Loop until all channels have completed requestedRecordCount transfers
    uint64_t endRecordIndex = recordCount_ + requestedRecordCount;
    for (;;) {
        /// Calc remaining record counts for all channels
        uint64_t totalRecordCount = 0;
        for (unsigned i=0; i < bytestreams_.size(); i++)
            totalRecordCount += endRecordIndex - bytestreams_.at(i)->currentRecordIndex();
#ifdef E57_MAX_VERBOSE
        cout << "  totalRecordCount=" << totalRecordCount << endl; //???
#endif

        /// We are done if have no more work, break out of loop
        if (totalRecordCount == 0)
            break;

        /// Estimate how many records can write before have enough data to fill data packet to efficient length
        /// Efficient packet length is >= 75% of maximum packet length.
        /// It is OK if get too much data (more than one packet) in an iteration.
        /// Reader will be able to handle packets whose streams are not exactly synchronized to the record boundaries.
        /// But try to do a good job of keeping the stream synchronization "close enough" (so a reader that can cache only two packets is efficient).

#ifdef E57_MAX_VERBOSE
        cout << "  currentPacketSize()=" << currentPacketSize() << endl; //???
#endif

#ifdef E57_WRITE_CRAZY_PACKET_MODE
///??? depends on number of streams
#  define E57_TARGET_PACKET_SIZE    500
#else
#  define E57_TARGET_PACKET_SIZE    (E57_DATA_PACKET_MAX*3/4)
#endif
        /// If have more than target fraction of packet, send it now
        if (currentPacketSize() >= E57_TARGET_PACKET_SIZE) {  //???
            packetWrite();
            continue;  /// restart loop so recalc statistics (packet size may not be zero after write, if have too much data)
        }

        ///??? useful?
        /// Get approximation of number of bytes per record of CompressedVector and total of bytes used
        float totalBitsPerRecord = 0;  // an estimate of future performance
        for (unsigned i=0; i < bytestreams_.size(); i++)
            totalBitsPerRecord += bytestreams_.at(i)->bitsPerRecord();

#ifdef E57_MAX_VERBOSE
        float totalBytesPerRecord = max(totalBitsPerRecord/8, 0.1F); //??? trust

        cout << "  totalBytesPerRecord=" << totalBytesPerRecord << endl; //???
#endif

//!!!        unsigned spaceRemaining = E57_DATA_PACKET_MAX - currentPacketSize();
//!!!        unsigned appoxRecordsNeeded = static_cast<unsigned>(floor(spaceRemaining / totalBytesPerRecord)); //??? divide by zero if all constants


        /// Don't allow straggler to get too far behind. ???
        /// Don't allow a single channel to get too far ahead ???
        /// Process channels that are furthest behind first. ???

        ///!!!! For now just process one record per loop until packet is full enough, or completed request
        for (unsigned i=0; i < bytestreams_.size(); i++) {
             if (bytestreams_.at(i)->currentRecordIndex() < endRecordIndex) {
#if 0
                bytestreams_.at(i)->processRecords(1);
#else
                //!!! For now, process up to 50 records at a time
                uint64_t recordCount = endRecordIndex - bytestreams_.at(i)->currentRecordIndex();
                recordCount = (recordCount<50ULL)?recordCount:50ULL; //min(recordCount, 50ULL);
                bytestreams_.at(i)->processRecords(static_cast<unsigned>(recordCount));
#endif
            }
        }
    }

    recordCount_ += requestedRecordCount;

    /// When we leave this function, will likely still have data in channel ioBuffers as well as partial words in Encoder registers.
}

size_t CompressedVectorWriterImpl::totalOutputAvailable() const
{
    size_t total = 0;
    for (size_t i=0; i < bytestreams_.size(); i++) {
        total += bytestreams_.at(i)->outputAvailable();
    }
    return(total);
}

size_t CompressedVectorWriterImpl::currentPacketSize() const
{
    /// Calc current packet size
    return(sizeof(DataPacketHeader) + bytestreams_.size()*sizeof(uint16_t) + totalOutputAvailable());
}

uint64_t CompressedVectorWriterImpl::packetWrite()
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorWriterImpl::packetWrite() called" << endl; //???
#endif

    /// Double check that we have work to do
    size_t totalOutput = totalOutputAvailable();
    if (totalOutput == 0)
        return(0);
#ifdef E57_MAX_VERBOSE
    cout << "  totalOutput=" << totalOutput << endl; //???
#endif

    /// Calc maximum number of bytestream values can put in data packet.
    size_t packetMaxPayloadBytes = E57_DATA_PACKET_MAX - sizeof(DataPacketHeader) - bytestreams_.size()*sizeof(uint16_t);
#ifdef E57_MAX_VERBOSE
    cout << "  packetMaxPayloadBytes=" << packetMaxPayloadBytes << endl; //???
#endif

    /// Allocate vector for number of bytes that each bytestream will write to file.
    vector<size_t> count(bytestreams_.size());

    /// See if we can fit into a single data packet
    if (totalOutput < packetMaxPayloadBytes) {
        /// We can fit everything in one packet
        for (unsigned i=0; i < bytestreams_.size(); i++)
            count.at(i) = bytestreams_.at(i)->outputAvailable();
    } else {
        /// We have too much data for one packet.  Send proportional amounts from each bytestream.
        /// Adjust packetMaxPayloadBytes down by one so have a little slack for floating point weirdness.
        float fractionToSend =  (packetMaxPayloadBytes-1) / static_cast<float>(totalOutput);
        for (unsigned i=0; i < bytestreams_.size(); i++) {
            /// Round down here so sum <= packetMaxPayloadBytes
            count.at(i) = static_cast<unsigned>(floor(fractionToSend * bytestreams_.at(i)->outputAvailable()));
        }
    }
#ifdef E57_MAX_VERBOSE
    for (unsigned i=0; i < bytestreams_.size(); i++)
        cout << "  count[" << i << "]=" << count.at(i) << endl; //???
#endif

#ifdef E57_DEBUG
    /// Double check sum of count is <= packetMaxPayloadBytes
    size_t totalByteCount = 0;
    for (unsigned i=0; i < count.size(); i++)
        totalByteCount += count.at(i);
    if (totalByteCount > packetMaxPayloadBytes) {
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "totalByteCount=" + toString(totalByteCount)
                             + " packetMaxPayloadBytes=" + toString(packetMaxPayloadBytes));
    }
#endif

    /// Get smart pointer to ImageFileImpl from associated CompressedVector
    shared_ptr<ImageFileImpl> imf(cVector_->destImageFile_);

    /// Use temp buf in object (is 64KBytes long) instead of allocating each time here
    char* packet = reinterpret_cast<char*>(&dataPacket_);
#ifdef E57_MAX_VERBOSE
    cout << "  packet=" << (unsigned)packet << endl; //???
#endif

    /// To be safe, clear header part of packet
    memset(packet, 0, sizeof(DataPacketHeader));

    /// Write bytestreamBufferLength[bytestreamCount] after header, in dataPacket_
    uint16_t* bsbLength = reinterpret_cast<uint16_t*>(&packet[sizeof(DataPacketHeader)]);
#ifdef E57_MAX_VERBOSE
    cout << "  bsbLength=" << (unsigned)bsbLength << endl; //???
#endif
    for (unsigned i=0; i < bytestreams_.size(); i++) {
        bsbLength[i] = static_cast<uint16_t>(count.at(i));      // %%% Truncation
#ifdef E57_MAX_VERBOSE
        cout << "  Writing " << bsbLength[i] << " bytes into bytestream " << i << endl; //???
#endif
    }

    /// Get pointer to end of data so far
    char* p = reinterpret_cast<char*>(&bsbLength[bytestreams_.size()]);
#ifdef E57_MAX_VERBOSE
    cout << "  after bsbLength, p=" << (unsigned)p << endl; //???
#endif

    /// Write contents of each bytestream in dataPacket_
    for (size_t i=0; i < bytestreams_.size(); i++) {
        size_t n = count.at(i);

#ifdef E57_DEBUG
        /// Double check we aren't accidentally going to write off end of vector<char>
        if (&p[n] > &packet[E57_DATA_PACKET_MAX])
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "n=" + toString(n));
#endif

        /// Read from encoder output into packet
        bytestreams_.at(i)->outputRead(p, n);

        /// Move pointer to end of current data
        p += n;
    }

    /// Length of packet is difference in beginning pointer and ending pointer
    unsigned packetLength = static_cast<unsigned>(p - packet);  ///??? pointer diff portable?
#ifdef E57_MAX_VERBOSE
    cout << "  packetLength=" << packetLength << endl; //???
#endif

#ifdef E57_DEBUG
    /// Double check that packetLength is what we expect
    if (packetLength != sizeof(DataPacketHeader) + bytestreams_.size()*sizeof(uint16_t) + totalByteCount) {
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "packetLength=" + toString(packetLength)
                             + " bytestreamSize=" + toString(bytestreams_.size()*sizeof(uint16_t))
                             + " totalByteCount=" + toString(totalByteCount));
    }
#endif

    /// packetLength must be multiple of 4, if not, add some zero padding
    while (packetLength % 4) {
        /// Double check we aren't accidentally going to write off end of vector<char>
        if (p >= &packet[E57_DATA_PACKET_MAX-1])
            throw E57_EXCEPTION1(E57_ERROR_INTERNAL);
        *p++ = 0;
        packetLength++;
#ifdef E57_MAX_VERBOSE
        cout << "  padding with zero byte, new packetLength=" << packetLength << endl; //???
#endif
    }

    /// Prepare header in dataPacket_, now that we are sure of packetLength
    dataPacket_.packetType = E57_DATA_PACKET;
    dataPacket_.packetFlags = 0;
    dataPacket_.packetLogicalLengthMinus1 = static_cast<uint16_t>(packetLength-1);          // %%% Truncation
    dataPacket_.bytestreamCount = static_cast<uint16_t>(bytestreams_.size());       // %%% Truncation

    /// Double check that data packet is well formed
    dataPacket_.verify(packetLength);

#ifdef E57_BIGENDIAN
    /// On bigendian CPUs, swab packet to little-endian byte order before writing.
    dataPacket_.swab(true);
#endif

    /// Write whole data packet at beginning of free space in file
    uint64_t packetLogicalOffset = imf->allocateSpace(packetLength, false);
    uint64_t packetPhysicalOffset = imf->file_->logicalToPhysical(packetLogicalOffset);
    imf->file_->seek(packetLogicalOffset);  //??? have seekLogical and seekPhysical instead? more explicit
    imf->file_->write(packet, packetLength);

#ifdef E57_MAX_VERBOSE
//  cout << "data packet:" << endl;
//  dataPacket_.dump(4);
#endif

    /// If first data packet written for this CompressedVector binary section, save address to put in section header
    ///??? what if no data packets?
    ///??? what if have exceptions while write, what is state of file?  will close report file good/bad?
    if (dataPacketsCount_ == 0)
        dataPhysicalOffset_ = packetPhysicalOffset;
    dataPacketsCount_++;

    ///!!! update seekIndex here? if started new chunk?

    /// Return physical offset of data packet for potential use in seekIndex
    return(packetPhysicalOffset); //??? needed
}

void CompressedVectorWriterImpl::flush()
{
    for (unsigned i=0; i < bytestreams_.size(); i++)
        bytestreams_.at(i)->registerFlushToOutput();
}

void CompressedVectorWriterImpl::checkImageFileOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName)
{
#if 0
!!! how get destImageFile?
    /// Throw an exception if destImageFile (destImageFile_) isn't open
    shared_ptr<CompressedVectorNodeImpl> cv(cVector_);

!!! how get destImageFile?
    shared_ptr<ImageFileImpl> destImageFile(destImageFile_);
    if (!destImageFile->isOpen()) {
        throw E57Exception(E57_ERROR_IMAGEFILE_NOT_OPEN,
                           "fileName=" + destImageFile->fileName(),
                           srcFileName,
                           srcLineNumber,
                           srcFunctionName);
    }
XXX
#endif
}

void CompressedVectorWriterImpl::checkWriterOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName) const
{
    if (!isOpen_)
    {
        throw E57Exception(E57_ERROR_WRITER_NOT_OPEN,
                           "imageFileName=" + cVector_->imageFileName() + " cvPathName=" + cVector_->pathName(),
                           srcFileName,
                           srcLineNumber,
                           srcFunctionName);
    }
}

void CompressedVectorWriterImpl::dump(int indent, std::ostream& os)
{
    os << space(indent) << "isOpen:" << isOpen_ << endl;

    for (unsigned i = 0; i < sbufs_.size(); i++) {
        os << space(indent) << "sbufs[" << i << "]:" << endl;
        sbufs_.at(i).dump(indent+4, os);
    }

    os << space(indent) << "cVector:" << endl;
    cVector_->dump(indent+4, os);

    os << space(indent) << "proto:" << endl;
    proto_->dump(indent+4, os);

    for (unsigned i = 0; i < bytestreams_.size(); i++) {
        os << space(indent) << "bytestreams[" << i << "]:" << endl;
        bytestreams_.at(i)->dump(indent+4, os);
    }

    os << space(indent) << "seekIndex:" << endl;
    seekIndex_.dump(indent+4, os);

    /// Don't call dump() for DataPacket, since it may contain junk when debugging.  Just print a few byte values.
    os << space(indent) << "dataPacket:" << endl;
    uint8_t* p = reinterpret_cast<uint8_t*>(&dataPacket_);
    for (unsigned i = 0; i < 40; ++i) {
        os << space(indent+4) << "dataPacket[" << i << "]: " << static_cast<unsigned>(p[i]) << endl;
    }
    os << space(indent+4) << "more unprinted..." << endl;

    os << space(indent) << "sectionHeaderLogicalStart: " << sectionHeaderLogicalStart_ << endl;
    os << space(indent) << "sectionLogicalLength:      " << sectionLogicalLength_ << endl;
    os << space(indent) << "dataPhysicalOffset:        " << dataPhysicalOffset_ << endl;
    os << space(indent) << "topIndexPhysicalOffset:    " << topIndexPhysicalOffset_ << endl;
    os << space(indent) << "recordCount:               " << recordCount_ << endl;
    os << space(indent) << "dataPacketsCount:          " << dataPacketsCount_ << endl;
    os << space(indent) << "indexPacketsCount:         " << indexPacketsCount_ << endl;
}

///================================================================
///================================================================
///================================================================

CompressedVectorReaderImpl::CompressedVectorReaderImpl(shared_ptr<CompressedVectorNodeImpl> cvi, vector<SourceDestBuffer>& dbufs)
: isOpen_(false),  // set to true when succeed below
  cVector_(cvi)
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorReaderImpl() called" << endl; //???
#endif
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    /// Allow reading of a completed CompressedVector, whether file is being read or currently being written.
    ///??? what other situations need checking for?
    ///??? check if CV not yet written to?
    ///??? file in error state?

    /// Empty dbufs is an error
    if (dbufs.size() == 0) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "imageFileName=" + cVector_->imageFileName()
                             + " cvPathName=" + cVector_->pathName());
    }

    /// Get CompressedArray's prototype node (all array elements must match this type)
    proto_ = cVector_->getPrototype();

    /// Check dbufs well formed (matches proto exactly)
    setBuffers(dbufs);

    /// For each dbuf, create an appropriate Decoder based on the cVector_ attributes
    for (unsigned i=0; i < dbufs_.size(); i++) {
        vector<SourceDestBuffer> theDbuf;
        theDbuf.push_back(dbufs.at(i));

        shared_ptr<Decoder> decoder =  Decoder::DecoderFactory(i, cVector_, theDbuf, ustring());

        /// Calc which stream the given path belongs to.  This depends on position of the node in the proto tree.
        shared_ptr<NodeImpl> readNode = proto_->get(dbufs.at(i).pathName());
        uint64_t bytestreamNumber = 0;
        if (!proto_->findTerminalPosition(readNode, bytestreamNumber))
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "dbufIndex=" + toString(i));

        channels_.push_back(DecodeChannel(dbufs.at(i), decoder, static_cast<unsigned>(bytestreamNumber), cVector_->childCount()));
    }

    recordCount_ = 0;

    /// Get how many records are actually defined
    maxRecordCount_ = cvi->childCount();

    shared_ptr<ImageFileImpl> imf(cVector_->destImageFile_);

    //??? what if fault in this constructor?
    cache_ = new PacketReadCache(imf->file_, 32);

    /// Read CompressedVector section header
    CompressedVectorSectionHeader sectionHeader;
    uint64_t sectionLogicalStart = cVector_->getBinarySectionLogicalStart();
    if (sectionLogicalStart == 0) {
        //??? should have caught this before got here, in XML read, get this if CV wasn't written to by writer.
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "imageFileName=" + cVector_->imageFileName()
                             + " cvPathName=" + cVector_->pathName());
    }
    imf->file_->seek(sectionLogicalStart, CheckedFile::Logical);
    imf->file_->read(reinterpret_cast<char*>(&sectionHeader), sizeof(sectionHeader));
    sectionHeader.swab();  /// swab if neccesary

#ifdef E57_DEBUG
    sectionHeader.verify(imf->file_->length(CheckedFile::Physical));
#endif

    /// Pre-calc end of section, so can tell when we are out of packets.
    sectionEndLogicalOffset_ = sectionLogicalStart + sectionHeader.sectionLogicalLength;

    /// Convert physical offset to first data packet to logical
    uint64_t dataLogicalOffset = imf->file_->physicalToLogical(sectionHeader.dataPhysicalOffset);

    /// Verify that packet given by dataPhysicalOffset is actually a data packet, init channels
    {
        char* anyPacket = nullptr;
        unique_ptr<PacketLock> packetLock = cache_->lock(dataLogicalOffset, anyPacket);

        DataPacket* dpkt = reinterpret_cast<DataPacket*>(anyPacket);

        /// Double check that have a data packet
        if (dpkt->packetType != E57_DATA_PACKET)
            throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetType=" + toString(dpkt->packetType));

        /// Have good packet, initialize channels
        for (unsigned i = 0; i < channels_.size(); i++) {
            DecodeChannel* chan = &channels_.at(i);
            chan->currentPacketLogicalOffset    = dataLogicalOffset;
            chan->currentBytestreamBufferIndex  = 0;
            chan->currentBytestreamBufferLength = dpkt->getBytestreamBufferLength(chan->bytestreamNumber);
        }
    }

    /// Just before return (and can't throw) increment reader count  ??? safer way to assure don't miss close?
    imf->incrReaderCount();

    /// If get here, the reader is open
    isOpen_ = true;
}

CompressedVectorReaderImpl::~CompressedVectorReaderImpl()
{
#ifdef E57_MAX_VERBOSE
    cout << "~CompressedVectorReaderImpl() called" << endl; //???
    //dump(4);
#endif

    if (isOpen_) {
        try {
            close();  ///??? what if already closed?
        } catch (...) {
                //??? report?
        }
    }
}

void CompressedVectorReaderImpl::setBuffers(vector<SourceDestBuffer>& dbufs)
{
    /// don't checkImageFileOpen
    /// don't checkReaderOpen

    /// Check dbufs well formed: no dups, no extra, missing is ok
    proto_->checkBuffers(dbufs, true);

    /// If had previous dbufs_, check to see if new ones have changed in incompatible way
    if (dbufs_.size() > 0) {
        if (dbufs_.size() != dbufs.size()) {
            throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                                 "oldSize=" + toString(dbufs_.size())
                                 + " newSize=" + toString(dbufs.size()));
        }
        for (size_t i = 0; i < dbufs_.size(); i++) {
            shared_ptr<SourceDestBufferImpl> oldBuf = dbufs_[i].impl();
            shared_ptr<SourceDestBufferImpl> newBuf = dbufs[i].impl();

            /// Throw exception if old and new not compatible
            oldBuf->checkCompatible(newBuf);
        }
    }

    dbufs_ = dbufs;
}

unsigned CompressedVectorReaderImpl::read(vector<SourceDestBuffer>& dbufs)
{
    /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), read() will do it

    checkReaderOpen(__FILE__, __LINE__, __FUNCTION__);

    /// Check compatible with current dbufs
    setBuffers(dbufs);

    return(read());
}

unsigned CompressedVectorReaderImpl::read()
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorReaderImpl::read() called" << endl; //???
#endif
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);
    checkReaderOpen(__FILE__, __LINE__, __FUNCTION__);

    /// Rewind all dbufs so start writing to them at beginning
    for (unsigned i=0; i < dbufs_.size(); i++)
        dbufs_[i].impl()->rewind();

    /// Allow decoders to use data they already have in their queue to fill newly empty dbufs
    /// This helps to keep decoder input queues smaller, which reduces backtracking in the packet cache.
    for (unsigned i = 0; i < channels_.size(); i++)
        channels_[i].decoder->inputProcess(NULL, 0);

    /// Loop until every dbuf is full or we have reached end of the binary section.
    while (1) {
        /// Find the earliest packet position for channels that are still hungry
        /// It's important to call inputProcess of the decoders before this call, so current hungriness level is reflected.
        uint64_t earliestPacketLogicalOffset = earliestPacketNeededForInput();

        /// If nobody's hungry, we are done with the read
        if (earliestPacketLogicalOffset == E57_UINT64_MAX)
            break;

        /// Feed packet to the hungry decoders
        feedPacketToDecoders(earliestPacketLogicalOffset);
    }

    /// Verify that each channel produced the same number of records
    unsigned outputCount = 0;
    for (unsigned i = 0; i < channels_.size(); i++) {
        DecodeChannel* chan = &channels_[i];
        if (i == 0)
            outputCount = chan->dbuf.impl()->nextIndex();
        else {
            if (outputCount != chan->dbuf.impl()->nextIndex()){
                throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                                     "outputCount=" + toString(outputCount)
                                     + " nextIndex=" + toString(chan->dbuf.impl()->nextIndex()));
            }
        }
    }

    /// Return number of records transferred to each dbuf.
    return(outputCount);
}

uint64_t CompressedVectorReaderImpl::earliestPacketNeededForInput() const
{
    uint64_t earliestPacketLogicalOffset = E57_UINT64_MAX;
    unsigned earliestChannel = 0;

    for (unsigned i = 0; i < channels_.size(); i++)
    {
        const DecodeChannel* chan = &channels_[i];

        /// Test if channel needs more input.
        /// Important to call inputProcess just before this, so these tests work.
        if (!chan->isOutputBlocked() && !chan->inputFinished)
        {
            /// Check if earliest so far
            if (chan->currentPacketLogicalOffset < earliestPacketLogicalOffset)
            {
                earliestPacketLogicalOffset = chan->currentPacketLogicalOffset;
                earliestChannel = i;
            }
        }
    }
#ifdef E57_MAX_VERBOSE
    if (earliestPacketLogicalOffset == E57_UINT64_MAX)
        cout << "earliestPacketNeededForInput returning none found" << endl;
    else
        cout << "earliestPacketNeededForInput returning " << earliestPacketLogicalOffset << " for channel[" << earliestChannel << "]" << endl;
#endif
    return earliestPacketLogicalOffset;
}

void CompressedVectorReaderImpl::feedPacketToDecoders(uint64_t currentPacketLogicalOffset)
{
    /// Read earliest packet into cache and send data to decoders with unblocked output
    bool channelHasExhaustedPacket = false;

    uint64_t nextPacketLogicalOffset = E57_UINT64_MAX;
    {
        /// Get packet at currentPacketLogicalOffset into memory.
        char* anyPacket = nullptr;
        unique_ptr<PacketLock> packetLock = cache_->lock(currentPacketLogicalOffset, anyPacket);
        DataPacket* dpkt = reinterpret_cast<DataPacket*>(anyPacket);

        /// Double check that have a data packet.  Should have already determined this.
        if (dpkt->packetType != E57_DATA_PACKET)
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "packetType=" + toString(dpkt->packetType));

        /// Feed bytestreams to channels with unblocked output that are reading from this packet
        for ( DecodeChannel &channel : channels_ )
        {
            /// Skip channels that have already read this packet.
            if (channel.currentPacketLogicalOffset != currentPacketLogicalOffset || channel.isOutputBlocked())
                continue;

            /// Get bytestream buffer for this channel from packet
            unsigned bsbLength;
            char* bsbStart = dpkt->getBytestream(channel.bytestreamNumber, bsbLength);

            /// Calc where we are in the buffer
            char* uneatenStart = &bsbStart[channel.currentBytestreamBufferIndex];
            size_t uneatenLength = bsbLength - channel.currentBytestreamBufferIndex;

            /// Double check we are not off end of buffer
            if (channel.currentBytestreamBufferIndex > bsbLength)
            {
                throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                                     "currentBytestreamBufferIndex =" + toString(channel.currentBytestreamBufferIndex)
                                     + " bsbLength=" + toString(bsbLength));
            }

            if (&uneatenStart[uneatenLength] > &bsbStart[bsbLength])
            {
                throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                                     "uneatenLength=" + toString(uneatenLength)
                                     + " bsbLength=" + toString(bsbLength));
            }
#ifdef E57_MAX_VERBOSE
            cout << "  stream[" << channel.bytestreamNumber << "]: feeding decoder " << uneatenLength << " bytes" << endl;
            if (uneatenLength == 0)
                chan->dump(8);
#endif
            /// Feed into decoder
            size_t bytesProcessed = channel.decoder->inputProcess(uneatenStart, uneatenLength);
#ifdef E57_MAX_VERBOSE
            cout << "  stream[" << channel.bytestreamNumber << "]: bytesProcessed=" << bytesProcessed << endl;
#endif
            /// Adjust counts of bytestream location
            channel.currentBytestreamBufferIndex += bytesProcessed;

            /// Check if this channel has exhausted its bytestream buffer in this packet
            if (channel.isInputBlocked())
            {
#ifdef E57_MAX_VERBOSE
                cout << "  stream[" << channel.bytestreamNumber << "] has exhausted its input in current packet" << endl;
#endif
                channelHasExhaustedPacket = true;
                nextPacketLogicalOffset = currentPacketLogicalOffset + dpkt->packetLogicalLengthMinus1 + 1;
            }
        }
    }

    /// Skip over any index or empty packets to next data packet.
    nextPacketLogicalOffset = findNextDataPacket(nextPacketLogicalOffset);

    /// If some channel has exhausted this packet, find next data packet and update currentPacketLogicalOffset for all interested channels.
    if (channelHasExhaustedPacket)
    {
        if (nextPacketLogicalOffset < E57_UINT64_MAX)
        { //??? huh?
            /// Get packet at nextPacketLogicalOffset into memory.
            char* anyPacket = nullptr;
            unique_ptr<PacketLock> packetLock = cache_->lock(nextPacketLogicalOffset, anyPacket);
            DataPacket* dpkt = reinterpret_cast<DataPacket*>(anyPacket);

#ifdef E57_MAX_VERBOSE
            unsigned int   i = 0;
#endif
            /// Got a data packet, update the channels with exhausted input
            for ( DecodeChannel &channel : channels_ )
            {
               if (channel.currentPacketLogicalOffset == currentPacketLogicalOffset && channel.isInputBlocked())
               {
                    channel.currentPacketLogicalOffset = nextPacketLogicalOffset;
                    channel.currentBytestreamBufferIndex = 0;

                    /// It is OK if the next packet doesn't contain any data for this channel, will skip packet on next iter of loop
                    channel.currentBytestreamBufferLength = dpkt->getBytestreamBufferLength(channel.bytestreamNumber);

#ifdef E57_MAX_VERBOSE
                    cout << "  set new stream buffer for channel[" << i++ << "], length=" << channel.currentBytestreamBufferLength << endl;
#endif
                    /// ??? perform flush if new packet flag set?
                }
            }
        }
        else
        {
            /// Reached end without finding data packet, mark exhausted channels as finished
#ifdef E57_MAX_VERBOSE
            cout << "  at end of data packets" << endl;
#endif
            if (nextPacketLogicalOffset >= sectionEndLogicalOffset_)
            {
                for ( DecodeChannel &channel : channels_ )
                {
                    if (channel.currentPacketLogicalOffset == currentPacketLogicalOffset && channel.isInputBlocked()) {
#ifdef E57_MAX_VERBOSE
                        cout << "  Marking channel[" << i++ << "] as finished" << endl;
#endif
                        channel.inputFinished = true;
                    }
                }
            }
        }
    }
}

uint64_t CompressedVectorReaderImpl::findNextDataPacket(uint64_t nextPacketLogicalOffset)
{
#ifdef E57_MAX_VERBOSE
    cout << "  searching for next data packet, nextPacketLogicalOffset=" << nextPacketLogicalOffset
         << " sectionEndLogicalOffset=" << sectionEndLogicalOffset_ << endl;
#endif

    /// Starting at nextPacketLogicalOffset, search for next data packet until hit end of binary section.
    while (nextPacketLogicalOffset < sectionEndLogicalOffset_)
    {
        char* anyPacket = nullptr;

        unique_ptr<PacketLock> packetLock = cache_->lock(nextPacketLogicalOffset, anyPacket);

        /// Guess it's a data packet, if not continue to next packet
        const DataPacket* dpkt = reinterpret_cast<const DataPacket*>(anyPacket);

        if (dpkt->packetType == E57_DATA_PACKET)
        {
#ifdef E57_MAX_VERBOSE
            cout << "  Found next data packet at nextPacketLogicalOffset=" << nextPacketLogicalOffset << endl;
#endif
            return nextPacketLogicalOffset;
        }

        /// All packets have length in same place, so can use the field to skip to next packet.
        nextPacketLogicalOffset += dpkt->packetLogicalLengthMinus1 + 1;
    }

    /// Ran off end of section, so return failure code.
    return E57_UINT64_MAX;
}

void CompressedVectorReaderImpl::seek(uint64_t /*recordNumber*/)
{
    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    ///!!! implement
    throw E57_EXCEPTION1(E57_ERROR_NOT_IMPLEMENTED);
}

bool CompressedVectorReaderImpl::isOpen() const
{
    /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), or checkReaderOpen()
    return(isOpen_);
}

std::shared_ptr<CompressedVectorNodeImpl> CompressedVectorReaderImpl::compressedVectorNode() const
{
    return(cVector_);
}

void CompressedVectorReaderImpl::close()
{
    /// Before anything that can throw, decrement reader count
    shared_ptr<ImageFileImpl> imf(cVector_->destImageFile_);
    imf->decrReaderCount();

    checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__);

    /// No error if reader not open
    if (!isOpen_)
        return;

    /// Destroy decoders
    channels_.clear();

    delete cache_;
    cache_ = nullptr;

    isOpen_ = false;
}

void CompressedVectorReaderImpl::checkImageFileOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName)
{
#if 0
!!! how get destImageFile?
    /// Throw an exception if destImageFile (destImageFile_) isn't open
    shared_ptr<CompressedVectorNodeImpl> cv(cVector_);

!!! how get destImageFile?
    shared_ptr<ImageFileImpl> destImageFile(destImageFile_);
    if (!destImageFile->isOpen()) {
        throw E57Exception(E57_ERROR_IMAGEFILE_NOT_OPEN,
                           "fileName=" + destImageFile->fileName());
                           srcFileName,
                           srcLineNumber,
                           srcFunctionName);
    }
XXX
#endif
}

void CompressedVectorReaderImpl::checkReaderOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName) const
{
    if (!isOpen_) {
        throw E57Exception(E57_ERROR_READER_NOT_OPEN,
                           "imageFileName=" + cVector_->imageFileName()
                           + " cvPathName=" + cVector_->pathName(),
                           srcFileName,
                           srcLineNumber,
                           srcFunctionName);
    }
}

void CompressedVectorReaderImpl::dump(int indent, std::ostream& os)
{
    os << space(indent) << "isOpen:" << isOpen_ << endl;

    for (unsigned i = 0; i < dbufs_.size(); i++) {
        os << space(indent) << "dbufs[" << i << "]:" << endl;
        dbufs_[i].dump(indent+4, os);
    }

    os << space(indent) << "cVector:" << endl;
    cVector_->dump(indent+4, os);

    os << space(indent) << "proto:" << endl;
    proto_->dump(indent+4, os);

    for (unsigned i = 0; i < channels_.size(); i++) {
        os << space(indent) << "channels[" << i << "]:" << endl;
        channels_[i].dump(indent+4, os);
    }

    os << space(indent) << "recordCount:             " << recordCount_ << endl;
    os << space(indent) << "maxRecordCount:          " << maxRecordCount_ << endl;
    os << space(indent) << "sectionEndLogicalOffset: " << sectionEndLogicalOffset_ << endl;
}

//================================================================

PacketLock::PacketLock(PacketReadCache* cache, unsigned cacheIndex)
: cache_(cache),
  cacheIndex_(cacheIndex)
{
#ifdef E57_MAX_VERBOSE
    cout << "PacketLock() called" << endl;
#endif
}

PacketLock::~PacketLock()
{
#ifdef E57_MAX_VERBOSE
    cout << "~PacketLock() called" << endl;
#endif
    try {
        /// Note cache must live longer than lock, this is reasonable assumption.
        cache_->unlock(cacheIndex_);
    } catch (...) {
        //??? report?
    }
}

PacketReadCache::PacketReadCache(CheckedFile* cFile, unsigned packetCount)
: lockCount_(0),
  useCount_(0),
  cFile_(cFile),
  entries_(packetCount)
{
    if (packetCount == 0)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "packetCount=" + toString(packetCount));

    /// Allocate requested number of maximum sized data packets buffers for holding data read from file
    for (unsigned i=0; i < entries_.size(); i++) {
        entries_.at(i).logicalOffset_ = 0;
        entries_.at(i).buffer_        = new char[E57_DATA_PACKET_MAX];
        entries_.at(i).lastUsed_      = 0;
    }
}

PacketReadCache::~PacketReadCache()
{
    /// Free allocated packet buffers
    for (unsigned i=0; i < entries_.size(); i++) {
        delete [] entries_.at(i).buffer_;
        entries_.at(i).buffer_ = NULL;
    }
}

unique_ptr<PacketLock> PacketReadCache::lock(uint64_t packetLogicalOffset, char* &pkt)
{
#ifdef E57_MAX_VERBOSE
    cout << "PacketReadCache::lock() called, packetLogicalOffset=" << packetLogicalOffset << endl;
#endif

    /// Only allow one locked packet at a time.
    if (lockCount_ > 0)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "lockCount=" + toString(lockCount_));

    /// Offset can't be 0
    if (packetLogicalOffset == 0)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "packetLogicalOffset=" + toString(packetLogicalOffset));

    /// Linear scan for matching packet offset in cache
    for (unsigned i = 0; i < entries_.size(); i++)
    {
        if (packetLogicalOffset == entries_[i].logicalOffset_)
        {
            /// Found a match, so don't have to read anything
#ifdef E57_MAX_VERBOSE
            cout << "  Found matching cache entry, index=" << i << endl;
#endif
            /// Mark entry with current useCount (keeps track of age of entry).
            entries_[i].lastUsed_ = ++useCount_;

            /// Publish buffer address to caller
            pkt = entries_[i].buffer_;

            /// Create lock so we are sure that we will be unlocked when use is finished.
            unique_ptr<PacketLock> plock(new PacketLock(this, i));

            /// Increment cache lock just before return
            lockCount_++;

            return plock;
        }
    }
    /// Get here if didn't find a match already in cache.

    /// Find least recently used (LRU) packet buffer
    unsigned oldestEntry = 0;
    unsigned oldestUsed = entries_.at(0).lastUsed_;

    for (unsigned i = 0; i < entries_.size(); i++)
    {
        if (entries_[i].lastUsed_ < oldestUsed)
        {
            oldestEntry = i;
            oldestUsed = entries_[i].lastUsed_;
        }
    }
#ifdef E57_MAX_VERBOSE
    cout << "  Oldest entry=" << oldestEntry << " lastUsed=" << oldestUsed << endl;
#endif

    readPacket(oldestEntry, packetLogicalOffset);

    /// Publish buffer address to caller
    pkt = entries_[oldestEntry].buffer_;

    /// Create lock so we are sure we will be unlocked when use is finished.
    unique_ptr<PacketLock> plock(new PacketLock(this, oldestEntry));

    /// Increment cache lock just before return
    lockCount_++;

    return plock;
}

void PacketReadCache::unlock(unsigned lockedEntry)
{
//??? why lockedEntry not used?
#ifdef E57_MAX_VERBOSE
    cout << "PacketReadCache::unlock() called, lockedEntry=" << lockedEntry << endl;
#endif

    if (lockCount_ != 1)
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "lockCount=" + toString(lockCount_));

    lockCount_--;
}

void PacketReadCache::readPacket(unsigned oldestEntry, uint64_t packetLogicalOffset)
{
#ifdef E57_MAX_VERBOSE
    cout << "PacketReadCache::readPacket() called, oldestEntry=" << oldestEntry << " packetLogicalOffset=" << packetLogicalOffset << endl;
#endif

    /// Read header of packet first to get length.  Use EmptyPacketHeader since it has the commom fields to all packets.
    EmptyPacketHeader header;
    cFile_->seek(packetLogicalOffset, CheckedFile::Logical);
    cFile_->read(reinterpret_cast<char*>(&header), sizeof(header));
    header.swab();
    /// Can't verify packet header here, because it is not really an EmptyPacketHeader.
    unsigned packetLength = header.packetLogicalLengthMinus1+1;

    /// Be paranoid about packetLength before read
    if (packetLength > E57_DATA_PACKET_MAX)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetLength=" + toString(packetLength));

    /// Now read in whole packet into preallocated buffer_.  Note buffer is
    cFile_->seek(packetLogicalOffset, CheckedFile::Logical);
    cFile_->read(entries_.at(oldestEntry).buffer_, packetLength);

    /// Swab if necessary, then verify that packet is good.
    switch (header.packetType)
    {
        case E57_DATA_PACKET: {
                DataPacket* dpkt = reinterpret_cast<DataPacket*>(entries_.at(oldestEntry).buffer_);
#ifdef E57_BIGENDIAN
                dpkt->swab(false);
#endif
                dpkt->verify(packetLength);
#ifdef E57_MAX_VERBOSE
                cout << "  data packet:" << endl;
                dpkt->dump(4); //???
#endif
            }
            break;
        case E57_INDEX_PACKET: {
                IndexPacket* ipkt = reinterpret_cast<IndexPacket*>(entries_.at(oldestEntry).buffer_);
#ifdef E57_BIGENDIAN
                ipkt->swab(false);
#endif
                ipkt->verify(packetLength);
#ifdef E57_MAX_VERBOSE
                cout << "  index packet:" << endl;
                ipkt->dump(4); //???
#endif
            }
            break;
        case E57_EMPTY_PACKET: {
                EmptyPacketHeader* hp = reinterpret_cast<EmptyPacketHeader*>(entries_.at(oldestEntry).buffer_);
                hp->swab();
                hp->verify(packetLength);
#ifdef E57_MAX_VERBOSE
                cout << "  empty packet:" << endl;
                hp->dump(4); //???
#endif
            }
            break;
        default:
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "packetType=" + toString(header.packetType));
    }

    entries_[oldestEntry].logicalOffset_ = packetLogicalOffset;

    /// Mark entry with current useCount (keeps track of age of entry).
    /// This is a cache, so a small hiccup when useCount_ overflows won't hurt.
    entries_[oldestEntry].lastUsed_ = ++useCount_;
}

#ifdef E57_DEBUG
void PacketReadCache::dump(int indent, std::ostream& os)
{
    os << space(indent) << "lockCount: " << lockCount_ << endl;
    os << space(indent) << "useCount:  " << useCount_ << endl;
    os << space(indent) << "entries:" << endl;
    for (unsigned i=0; i < entries_.size(); i++) {
        os << space(indent) << "entry[" << i << "]:" << endl;
        os << space(indent+4) << "logicalOffset:  " << entries_[i].logicalOffset_ << endl;
        os << space(indent+4) << "lastUsed:        " << entries_[i].lastUsed_ << endl;
        if (entries_[i].logicalOffset_ != 0) {
            os << space(indent+4) << "packet:" << endl;
            switch (reinterpret_cast<EmptyPacketHeader*>(entries_.at(i).buffer_)->packetType) {
                case E57_DATA_PACKET: {
                        DataPacket* dpkt = reinterpret_cast<DataPacket*>(entries_.at(i).buffer_);
                        dpkt->dump(indent+6, os);
                    }
                    break;
                case E57_INDEX_PACKET: {
                        IndexPacket* ipkt = reinterpret_cast<IndexPacket*>(entries_.at(i).buffer_);
                        ipkt->dump(indent+6, os);
                    }
                    break;
                case E57_EMPTY_PACKET: {
                        EmptyPacketHeader* hp = reinterpret_cast<EmptyPacketHeader*>(entries_.at(i).buffer_);
                        hp->dump(indent+6, os);
                    }
                    break;
                default:
                    throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                                         "packetType=" + toString(reinterpret_cast<EmptyPacketHeader*>(entries_.at(i).buffer_)->packetType));
            }
        }
    }
}
#endif

//================================================================

DecodeChannel::DecodeChannel(SourceDestBuffer dbuf_arg, shared_ptr<Decoder> decoder_arg, unsigned bytestreamNumber_arg, uint64_t maxRecordCount_arg)
: dbuf(dbuf_arg),
  decoder(decoder_arg),
  bytestreamNumber(bytestreamNumber_arg)
{
    maxRecordCount = maxRecordCount_arg;
    currentPacketLogicalOffset = 0;
    currentBytestreamBufferIndex = 0;
    currentBytestreamBufferLength = 0;
    inputFinished = 0;
}

bool DecodeChannel::isOutputBlocked() const
{
    /// If we have completed the entire vector, we are done
    if (decoder->totalRecordsCompleted() >= maxRecordCount)
        return(true);

    /// If we have filled the dest buffer, we are blocked
    return(dbuf.impl()->nextIndex() == dbuf.impl()->capacity());
}

bool DecodeChannel::isInputBlocked() const
{
    /// If have read until the section end, we are done
    if (inputFinished)
        return(true);

    /// If have eaten all the input in the current packet, we are blocked.
    return(currentBytestreamBufferIndex == currentBytestreamBufferLength);
}

void DecodeChannel::dump(int indent, std::ostream& os)
{
    os << space(indent) << "dbuf" << endl;
    dbuf.dump(indent+4, os);

    os << space(indent) << "decoder:" << endl;
    decoder->dump(indent+4, os);

    os << space(indent) << "bytestreamNumber:              " << bytestreamNumber << endl;
    os << space(indent) << "maxRecordCount:                " << maxRecordCount << endl;
    os << space(indent) << "currentPacketLogicalOffset:    " << currentPacketLogicalOffset << endl;
    os << space(indent) << "currentBytestreamBufferIndex:  " << currentBytestreamBufferIndex << endl;
    os << space(indent) << "currentBytestreamBufferLength: " << currentBytestreamBufferLength << endl;
    os << space(indent) << "inputFinished:                 " << inputFinished << endl;
    os << space(indent) << "isInputBlocked():              " << isInputBlocked() << endl;
    os << space(indent) << "isOutputBlocked():             " << isOutputBlocked() << endl;
}
