//
//  Utils4.h
//  
//
//  Created by Yufeng Wu on 9/30/13.
//  More utilities that may be useful
//

#ifndef ____Utils4__
#define ____Utils4__

#include "Utils3.h"
#include <cmath>
#include <algorithm>
#include <sstream>
#include <string>
#include <queue>
#include <set>
#include <fstream>
#include <iostream>
#include <unordered_map>
#include <list>
#include <cstddef>
#include <stdexcept>

#define YW_VERY_SMALL_FRACTION   0.000000000001


// a list of templates
template<class TYPE1, class TYPE2>
void CreateMapFromTwoVec( const vector<TYPE1> &vecKey, const vector<TYPE2> &vecval, map<TYPE1,TYPE2> &mapCreated  )
{
    //
    YW_ASSERT_INFO( vecKey.size() == vecval.size(), "veckey has different size as vecval" );
    mapCreated.clear();
    for( int i=0; i<(int)vecKey.size(); ++i )
    {
        //
        mapCreated.insert(typename map<TYPE1,TYPE2> :: value_type(vecKey[i], vecval[i] ) );
    }
}

template<class TYPE1,class TYPE2>
void KeepCommonInMaps( map<TYPE1,TYPE2> &mapSubtracted, const map<TYPE1,TYPE2> &mapToSub )
{
    // only keep those that is also in the second map
    map<TYPE1, TYPE2> mapNew;
    for( typename map<TYPE1,TYPE2> :: iterator it = mapSubtracted.begin(); it != mapSubtracted.end(); ++it )
    {
        //
        if( mapToSub.find(it->first) != mapToSub.end() )
        {
            // appear in second map so keep
            mapNew.insert( typename map<TYPE1,TYPE2> :: value_type(it->first, it->second) );
        }
    }
    mapSubtracted = mapNew;
}

template<class TYPE1,class TYPE2>
void KeepCommonInMapsSet( map<TYPE1,TYPE2> &mapSubtracted, const set<TYPE1> &setKept )
{
    // only keep those that is also in the second map
    map<TYPE1, TYPE2> mapNew;
    for( typename map<TYPE1,TYPE2> :: iterator it = mapSubtracted.begin(); it != mapSubtracted.end(); ++it )
    {
        //
        if( setKept.find(it->first) != setKept.end() )
        {
            // appear in second map so keep
            mapNew.insert( typename map<TYPE1,TYPE2> :: value_type(it->first, it->second) );
        }
    }
    mapSubtracted = mapNew;
    
}

template<class TYPE1, class TYPE2>
void CreateTwoVecFromMap(const map<TYPE1,TYPE2> &mapIn, vector<TYPE1> &vecKey, vector<TYPE2> &vecval )
{
    vecKey.clear();
    vecval.clear();
    for( typename map<TYPE1,TYPE2> :: const_iterator it = mapIn.begin(); it != mapIn.end(); ++it )
    {
        vecKey.push_back( it->first );
        vecval.push_back( it->second );
    }
}

template<class TYPE>
TYPE GetSumOfVecElements(const vector<TYPE> &listVals)
{
    TYPE sum = 0;
    for(int i=0; i<(int)listVals.size(); ++i)
    {
        sum += listVals[i];
    }
    return sum;
}

template<class TYPE>
double MyCalcStdError(const vector<TYPE> &listVals, const vector<TYPE> &listValsRef)
{
    YW_ASSERT_INFO(listVals.size() == listValsRef.size(), "CalcStdError: Size mismatch");
    double sum = 0.0;
    if( listValsRef.size() == 0)
    {
        return 0.0;
    }
    for(int i=0; i<(int)listVals.size(); ++i)
    {
        double diff = listVals[i] - listValsRef[i];
        sum += diff*diff;
    }
    return sqrt(sum/listValsRef.size());
}

template<class TYPE>
void GetPositionsOverThres(const vector<TYPE> &listVals, const TYPE& val, int maxNum, set<int> &listPoses)
{
    // get positions that are either over or at, as long as the total number is not over
    listPoses.clear();
    for(int p=0; p<(int)listVals.size(); ++p)
    {
        //
        if(  val < listVals[p] && (int)listPoses.size() < maxNum  )
        {
            //
            listPoses.insert(p);
        }
    }
    // now also check for those equal
    for(int p=0; p<(int)listVals.size(); ++p)
    {
        //
        if(  val == listVals[p] && (int)listPoses.size() < maxNum  )
        {
            //
            listPoses.insert(p);
        }
    }
}

template<class TYPE>
void AddVecTo(vector<TYPE> &vecAdded, const vector<TYPE> &vecAdding)
{
    //
    YW_ASSERT_INFO(vecAdded.size() == vecAdding.size(), "Size mismatch");
    for(int i=0; i<(int)vecAdding.size(); ++i)
    {
        vecAdded[i] += vecAdding[i];
    }
}

template<class TYPE>
void ConcatVecTo(vector<TYPE> &vecAdded, const vector<TYPE> &vecAdding)
{
    //
    for(int i=0; i<(int)vecAdding.size(); ++i)
    {
        vecAdded.push_back( vecAdding[i] );
    }
}

template<class TYPE>
int FindMajorityElemVal(const vector<TYPE> &listItems, double valThres)
{
    // find out whether there is an item that is over some percentage of all the sum (say 50%)
    //TYPE sum = GetSumOfVecElements(listItems);
    //YW_ASSERT_INFO(sum > 0, "Can not only have zero");
    for(int i=0; i<(int)listItems.size(); ++i)
    {
        if( (double)(listItems[i]) > valThres )
           {
               return i;
           }
    }
    // no majority item
    return -1;
}

template<class TYPE>
int FindMajorityElem(const vector<TYPE> &listItems, double fracMaj)
{
    TYPE sum = GetSumOfVecElements(listItems);
    return FindMajorityElemVal(listItems, fracMaj*sum);
}

template<class TYPE>
void FindMajorityMultiElemVal(const vector<TYPE> &listItems, double valThres, int maxNum, set<int> &listChosenPos)
{
    // find out whether there is an item that is over some percentage of all the sum (say 50%)
    listChosenPos.clear();
    //TYPE sum = GetSumOfVecElements(listItems);
    //YW_ASSERT_INFO(sum > 0, "Can not only have zero");
    vector<TYPE> listItemsSort = listItems;
    YWSort(listItemsSort);
    TYPE sumSoFar = 0;
    int numAdd = 0;
    int indexPicked = -1;
    for(int i=(int)listItemsSort.size()-1; i>=0;--i)
    {
        ++numAdd;
        if( numAdd > maxNum )
        {
            break;
        }
        sumSoFar += listItemsSort[i];
        
        if( (double)(sumSoFar) > valThres )
        {
            indexPicked = i;
            break;
        }
    }
    // no majority item
    if( indexPicked < 0 )
    {
        return;
    }
    // find the set of items that is at least that much
    // get those over first
    for( int i=0; i<(int)listItems.size(); ++i)
    {
        if( listItems[i] > listItemsSort[indexPicked])
        {
            listChosenPos.insert(i);
        }
    }
    for( int i=0; i<(int)listItems.size(); ++i)
    {
        if( listItems[i] == listItemsSort[indexPicked])
        {
            if( (int)listChosenPos.size() < numAdd )
            {
                listChosenPos.insert(i);
            }
            else
            {
                break;
            }
        }
    }
}

template<class TYPE>
void FindMajorityMultiElem(const vector<TYPE> &listItems, double fracMaj, int maxNum, set<int> &listChosenPos)
{
    // find out whether there is an item that is over some percentage of all the sum (say 50%)
    listChosenPos.clear();
    TYPE sum = GetSumOfVecElements(listItems);
    YW_ASSERT_INFO(sum > 0, "Can not only have zero");
    FindMajorityMultiElemVal( listItems, fracMaj*sum, maxNum, listChosenPos );
}

template<class TYPE>
TYPE FindExtremeFreqElem(const multiset<TYPE> &setItems, bool fMin)
{
    // find out the least frequent (fMin=true) or most frequent (fMin=false) from a list of items
    std::set<TYPE> my_set(setItems.begin(), setItems.end());
    vector<int> listOcc;
    vector<pair<int, TYPE > > mapOcc;
    for(typename set<TYPE> :: iterator it = my_set.begin(); it != my_set.end(); ++it)
    {
        //
        int count = setItems.count(*it);
        listOcc.push_back( count );
        pair<int,TYPE> pp(count, *it);
        mapOcc.push_back( pp );
    }
    std::sort (listOcc.begin(), listOcc.end() );
    int occExt;
    if( fMin)
    {
        occExt = listOcc[0];
    }
    else
    {
        occExt = listOcc[ (int)listOcc.size()-1 ];
    }
    for(int i=0; i<(int)mapOcc.size(); ++i )
    {
        if( mapOcc[i].first == occExt )
        {
            return mapOcc[i].second;
        }
    }
    
    // if failed, just return the first item
    YW_ASSERT_INFO(false, "Fail");
    return mapOcc[0].second;
}



template<class TYPE>
bool YWSortCmpFunc (TYPE i,TYPE j) { return (i<j); }

template<class TYPE>
void YWSort(vector<TYPE> &vecIn)
{
    //
    typedef bool (*comparer_t)(const TYPE, const TYPE);
    comparer_t cmp = &YWSortCmpFunc;
    std::sort (vecIn.begin(), vecIn.end(), cmp);
}

template<class TYPE>
void DumpPair(const pair<TYPE,TYPE> &pp)
{
    cout << "[" << pp.first << "," << pp.second << "]";
}

template<class TYPE>
void GetSubsetItem( const vector<TYPE> &listItems, const vector<int> &vecpos, set<TYPE> &subsetItems )
{
    //
    subsetItems.clear();
    for(int i=0; i<(int)vecpos.size(); ++i)
    {
        YW_ASSERT_INFO( vecpos[i] <(int)listItems.size(), "Fail");
        subsetItems.insert( listItems[ vecpos[i] ]);
    }
}

template<class TYPE>
void GetSubsetItemVec( const vector<TYPE> &listItems, const vector<int> &vecpos, vector<TYPE> &vecSublist )
{
    //
    vecSublist.clear();
    for(int i=0; i<(int)vecpos.size(); ++i)
    {
        YW_ASSERT_INFO( vecpos[i] <(int)listItems.size(), "Fail");
        vecSublist.push_back( listItems[ vecpos[i] ]);
    }
}

template<class TYPE>
void GetSubsetSets( const vector<set<TYPE> > &listItems, const vector<int> &vecpos, set<TYPE> &subsetItems )
{
    //
    subsetItems.clear();
    for(int i=0; i<(int)vecpos.size(); ++i)
    {
        YW_ASSERT_INFO( vecpos[i] <(int)listItems.size(), "Fail");
        UnionSetsGen( subsetItems, listItems[ vecpos[i] ] );
    }
}

template<class TYPE>
int FindMaxValPositionFromList(const vector<TYPE> &listItems)
{
    // find out whether there is an item that is over some percentage of all the sum (say 50%)
    //TYPE sum = GetSumOfVecElements(listItems);
    //YW_ASSERT_INFO(sum > 0, "Can not only have zero");
    YW_ASSERT_INFO( listItems.size() > 0, "Must have at least one");
    TYPE valMaxCur = listItems[0];
    int posMaxCur = 0;
    for(int i=1; i<(int)listItems.size(); ++i)
    {
        if( valMaxCur < listItems[i] )
        {
            posMaxCur = i;
            valMaxCur = listItems[i];
        }
    }
    // no majority item
    return posMaxCur;
}

template<class TYPE>
int FindMaxValPositionFromListGap(const vector<TYPE> &listItems, const TYPE &gapMin)
{
    // find out whether there is an item that is over some percentage of all the sum (say 50%)
    //TYPE sum = GetSumOfVecElements(listItems);
    //YW_ASSERT_INFO(sum > 0, "Can not only have zero");
    YW_ASSERT_INFO( listItems.size() > 0, "Must have at least one");
    TYPE valMaxCur = listItems[0];
    int posMaxCur = 0;
    for(int i=1; i<(int)listItems.size(); ++i)
    {
        if( valMaxCur + gapMin < listItems[i] )
        {
            posMaxCur = i;
            valMaxCur = listItems[i];
        }
    }
    // no majority item
    return posMaxCur;
}

template<class TYPE>
bool IsSetContainerGen(const set<TYPE> &container, const set<TYPE> &contained)
{
    //
    for(typename set<TYPE> ::iterator it = contained.begin(); it != contained.end(); ++it)
    {
        if(  container.find( *it ) == container.end()  )
        {
            return false;
        }
    }
    return true;
}


template<class TYPE>
bool FindSmallestContainSetInMapGen(const set<TYPE> &setTest, const map<int, set<set<TYPE> > > &mapAllSets, set<TYPE> &setContainer)
{
    //
    for(typename map<int, set<set<TYPE> > > ::const_iterator it = mapAllSets.begin(); it != mapAllSets.end(); ++it)
    {
        for(typename set<set<TYPE> > :: const_iterator it2 = it->second.begin(); it2 != it->second.end(); ++it2)
        {
            if(  IsSetContainerGen( *it2, setTest ) == true  )
            {
                setContainer = *it2;
                return true;
            }
        }
    }
    return false;
}


template<class TYPE>
bool AreSetsIntersecting(const set<TYPE> &s1In, const set<TYPE> &s2In)
{
    //
    const set<TYPE> *ptrSet1 = &s1In;
    const set<TYPE> *ptrSet2 = &s2In;
    if( s1In.size() > s2In.size() )
    {
        ptrSet1 = &s2In;
        ptrSet2 = &s1In;
    }
    for(typename set<TYPE> ::iterator it = ptrSet1->begin(); it != ptrSet1->end(); ++it)
    {
        if(  ptrSet2->find( *it ) != ptrSet2->end()  )
        {
            return true;
        }
    }
    return false;
}

template<class TYPE>
string ConvToString(const TYPE &val)
{
    ostringstream convert;   // stream used for the conversion
    convert << val;      // insert the textual representation of 'Number' in the characters in the stream
    return convert.str();
}

double StrToDouble(const string &s);

template<class TYPE>
string ConsNewickTreeFromClades(const set<set<TYPE> > &setClades)
{
    // clade: a collection of taxa (int or string); output newick format
    // first, the set of taxa is always the outmost clade
    set<TYPE> setTaxa;
    map<set<TYPE>, set<TYPE>  > mapCladePars;
    for(typename set<set<TYPE> > :: const_iterator it = setClades.begin(); it!=setClades.end(); ++it)
    {
        // find it out the set of taxa
        for( typename set<TYPE> :: iterator itg=it->begin(); itg != it->end(); ++itg)
        {
            setTaxa.insert(*itg);
        }
    }
    set<set<TYPE> > setCladesUsed = setClades;
    setCladesUsed.insert(setTaxa);
    // also ensure single taxon is in
    for( typename set<TYPE> :: iterator it = setTaxa.begin(); it!= setTaxa.end(); ++it )
    {
        //
        set<TYPE> ss;
        ss.insert(*it);
        setCladesUsed.insert(ss);
    }
    // order the clades by size (YW: not the best implementation but hope it will work)
    map<int, set<set<TYPE> > > mapCladesSz;
    for( typename set<set<TYPE> > :: iterator it = setCladesUsed.begin(); it != setCladesUsed.end(); ++it)
    {
        if( mapCladesSz.find(it->size()) == mapCladesSz.end() )
        {
            set<set<TYPE> > ss;
            mapCladesSz.insert( typename map<int,set<set<TYPE> > > :: value_type(it->size(), ss) );
        }
        mapCladesSz[it->size()].insert(*it);
    }
    // find par of each clade
    for( typename set<set<TYPE> > :: iterator it = setCladesUsed.begin(); it != setCladesUsed.end(); ++it )
    {
        //
        for( typename set<set<TYPE> > :: iterator it2 = setCladesUsed.begin(); it2 != setCladesUsed.end(); ++it2 )
        {
            //
            if( it2 != it && IsSetContainerGen(*it2, *it) == true  )
            {
                //
                if( mapCladePars.find( *it) == mapCladePars.end() )   
                {
                    mapCladePars.insert( typename map<set<TYPE>,set<TYPE> > :: value_type(*it, *it2) );
                }
                else if(mapCladePars[*it].size() > it2->size())
                {
                    mapCladePars[*it] = *it2;
                }
            }
        }
    }
    // now assign each clade a string
    map<set<TYPE>, string> mapCladeToStr;
    queue<set<TYPE> > queueToProc;
    // init leaves
    for(typename set<TYPE> :: iterator it = setTaxa.begin(); it != setTaxa.end(); ++it)
    {
        set<TYPE> ss;
        ss.insert(*it);
        string strLbl = ConvToString(*it);
        mapCladeToStr.insert( typename map<set<TYPE>, string>  :: value_type(ss, strLbl) );
        queueToProc.push(ss);
    }
    // now proc from bottom up
    for( typename map<int, set<set<TYPE> > > :: iterator it = mapCladesSz.begin(); it != mapCladesSz.end(); ++it)
    {
        for( typename set<set<TYPE> > :: iterator itg = it->second.begin(); itg != it->second.end(); ++itg)
        {
            YW_ASSERT_INFO( mapCladeToStr.find( *itg) != mapCladeToStr.end(), "Fail to find string");
            // pass it to parent
            if( mapCladePars.find( *itg) != mapCladePars.end() )
            {
                string strBase = mapCladeToStr[*itg];
                if( itg->size() > 1 )
                {
                    // add parenthsis
                    strBase = "(" + mapCladeToStr[*itg]+")";
                }
                
                //
                set<TYPE> sPar = mapCladePars[*itg];
                if( mapCladeToStr.find(sPar) == mapCladeToStr.end() )
                {
                    mapCladeToStr.insert( typename map<set<TYPE>, string> :: value_type( sPar, strBase ) );
                }
                else
                {
                    mapCladeToStr[sPar] = mapCladeToStr[sPar] + "," + strBase;
                }
            }
        }
    }
    // finally
    YW_ASSERT_INFO( mapCladeToStr.find( setTaxa) != mapCladeToStr.end(), "Wrong");
    string res = "(" + mapCladeToStr[setTaxa] + ")";
    return res;
}

template<class TYPE>
void FindMaximalSets( set<set<TYPE> > &setsItems )
{
    // only keep those with no super set
    set<set<TYPE> > setsItemsRes;
    for( typename set<set<TYPE> > :: iterator it = setsItems.begin(); it != setsItems.end(); ++it)
    {
        bool fSuperSet = false;
        for( typename set<set<TYPE> > :: iterator itg = setsItems.begin(); itg != setsItems.end(); ++itg)
        {
            // is itg the super set?
            if( itg->size() > it->size() && IsSetContainerGen( *itg, *it) == true )
            {
                fSuperSet = true;
                break;
            }
        }
        if( fSuperSet == false )
        {
            setsItemsRes.insert(*it);
        }
    }
    setsItems = setsItemsRes;
}

template<class TYPE>
void InitVecWithVal( vector<TYPE> &listVec, TYPE valInit, int numItems)
{
    listVec.clear();
    for(int i=0; i<numItems; ++i)
    {
        listVec.push_back(valInit);
    }
}

template<class TYPE>
void PopulateVecBySetGen( vector<TYPE> &vec, const set<TYPE> &sset)
{
    //
    vec.clear();
    for( typename set<TYPE> :: const_iterator it = sset.begin(); it != sset.end(); ++it)
    {
        vec.push_back( *it );
    }
}

template<class TYPE>
void PopulateVecBySetPtrGen( vector<const TYPE *> &vec, const set<TYPE> &sset)
{
    //
    vec.clear();
    for( typename set<TYPE> :: const_iterator it = sset.begin(); it != sset.end(); ++it)
    {
        vec.push_back( &(*it) );
    }
}

template<class TYPE>
void PopulateSetPtrBySetGen( set<const TYPE *> &sptrs, const set<TYPE> &sset)
{
    //
    sptrs.clear();
    for( typename set<TYPE> :: const_iterator it = sset.begin(); it != sset.end(); ++it)
    {
        sptrs.insert( &(*it) );
    }
}


template<class TYPE>
void PopulateSetByVecGen( set<TYPE> &sset, const vector<TYPE> &vec)
{
    //
    sset.clear();
    for( typename vector<TYPE> :: const_iterator it = vec.begin(); it != vec.end(); ++it)
    {
        sset.insert( *it );
    }
}

template<class TYPE>
void PopulateSetBySetPtrGen( set<TYPE> &sset, const set<const TYPE *> &ssetPtr)
{
    //
    sset.clear();
    for( typename set<const TYPE *> :: const_iterator it = ssetPtr.begin(); it != ssetPtr.end(); ++it)
    {
        sset.push_back( *(*it) );
    }
}


template<class TYPE1, class TYPE2>
void MergeMapGen(map<TYPE1,TYPE2> &mapCombined, const map<TYPE1,TYPE2> &mapToAdd)
{
    for( typename map<TYPE1,TYPE2> :: const_iterator it = mapToAdd.begin(); it != mapToAdd.end(); ++it)
    {
        mapCombined.insert( typename map<TYPE1,TYPE2> :: value_type(it->first, it->second) );
    }
}

template<class TYPE>
void SplitItemsBySetOfPartition( const set<TYPE> &setItems, const set<set<TYPE> > &setPartitions, vector<set<TYPE> > &vecSplitParts )
{
    // setItems: a list of items; setpartitions: parition the space of items; vecSplitParts: split setItems into unit of those partitions
    // approach, take join repeatitively
    vecSplitParts.clear();
    set<TYPE> setItemsUse = setItems;
    while(setItemsUse.size() > 0)
    {
        bool fSub = false;
        for( typename set<set<TYPE> > :: iterator it = setPartitions.begin(); it != setPartitions.end(); ++it )
        {
            //
            set<TYPE> setItemSub;
            JoinSetsGen(*it, setItemsUse, setItemSub);
            YW_ASSERT_INFO( setItemSub.size() == 0 || setItemSub.size() == it->size(), "Not a partition");
            if( setItemSub.size() == it->size() && it->size() > 0 )
            {
                vecSplitParts.push_back(*it);
                SubtractSetsGen( setItemsUse, *it  );
                fSub = true;
            }
        }
        YW_ASSERT_INFO( fSub == true || setItemsUse.size() == 0, "FATAL ERROR: not progress made in SplitItemsBySetOfPartition");
    }
}

template<class TYPE>
bool SplitItemsBySetOfPartitionTF( const set<TYPE> &setItems, const set<set<TYPE> > &setPartitions, vector<set<TYPE> > &vecSplitParts )
{
    // setItems: a list of items; setpartitions: parition the space of items; vecSplitParts: split setItems into unit of those partitions
    // approach, take join repeatitively
    vecSplitParts.clear();
    set<TYPE> setItemsUse = setItems;
    while(setItemsUse.size() > 0)
    {
        bool fSub = false;
        for( typename set<set<TYPE> > :: iterator it = setPartitions.begin(); it != setPartitions.end(); ++it )
        {
            //
            set<TYPE> setItemSub;
            JoinSetsGen(*it, setItemsUse, setItemSub);
            if( setItemSub.size() > 0 && setItemSub.size() < it->size()  )
            {
                return false;
            }
            if( setItemSub.size() == it->size() && it->size() > 0 )
            {
                vecSplitParts.push_back(*it);
                SubtractSetsGen( setItemsUse, *it  );
                fSub = true;
            }
        }
        YW_ASSERT_INFO( fSub == true || setItemsUse.size() == 0, "FATAL ERROR: not progress made in SplitItemsBySetOfPartition");
    }
    return true;
}

template<class TYPE>
void SplitItemsofVecIntoTwoParts( const vector<TYPE> &vecItems, vector<TYPE> &vecFirstPart, vector<TYPE> &vecSecondPart, int posStartof2ndPart )
{
    // caution: position is 0 based
    vecFirstPart.clear();
    vecSecondPart.clear();
    for(int i=0; i<(int)vecItems.size() && i<posStartof2ndPart; ++i )
    {
        vecFirstPart.push_back(vecItems[i]);
    }
    for(int i=posStartof2ndPart; i<(int)vecItems.size(); ++i )
    {
        vecSecondPart.push_back(vecItems[i]);
    }
}

template<class TYPE>
void MergeTwoVectorsInto( vector<TYPE> &vecItems, const vector<TYPE> &vecFirstPart, const vector<TYPE> &vecSecondPart)
{
    //
    vecItems.clear();
    for(int i=0; i<(int)vecFirstPart.size(); ++i )
    {
        vecItems.push_back(vecFirstPart[i]);
    }
    for(int i=0; i<(int)vecSecondPart.size(); ++i )
    {
        vecItems.push_back(vecSecondPart[i]);
    }
}

template<class TYPE>
void ScaleVectorValBy(vector<TYPE> &vecItems, const TYPE &factor)
{
    for(int i=0; i<(int)vecItems.size(); ++i)
    {
        vecItems[i] *= factor;
    }
}

template<class TYPE>
void OffsetVectorValBy(vector<TYPE> &vecItems, const TYPE &factor)
{
    for(int i=0; i<(int)vecItems.size(); ++i)
    {
        vecItems[i] += factor;
    }
}

template<class TYPE>
void PointwiseMultiVectorBy(vector<TYPE> &vecItems, const vector<TYPE> &vecItemsFactors)
{
    YW_ASSERT_INFO( vecItems.size() == vecItemsFactors.size(), "PointwiseMultiVectorBy: size wrong" );
    for(int i=0; i<(int)vecItems.size(); ++i)
    {
        vecItems[i] *= vecItemsFactors[i];
    }
}

template<class TYPE>
void PointwiseAddVectorBy(vector<TYPE> &vecItemsAdded, const vector<TYPE> &vecItemsAdding)
{
    YW_ASSERT_INFO( vecItemsAdded.size() == vecItemsAdding.size(), "PointwiseMultiVectorBy: size wrong" );
    for(int i=0; i<(int)vecItemsAdded.size(); ++i)
    {
        vecItemsAdded[i] += vecItemsAdding[i];
    }
}

template<class TYPE>
void CopyVecToArray( const vector<TYPE> &vecItems, TYPE *parray)
{
    // CAUTION: the array must have adequate size to avoid buffer overrun
    for(int i=0; i<(int)vecItems.size(); ++i)
    {
        parray[i] = vecItems[i];
    }
}

template<class TYPE>
void CopyArrayToVec( TYPE *parray, int sz, vector<TYPE> &vecItems)
{
    // CAUTION: the array must have adequate size to avoid buffer overrun
    vecItems.clear();
    for(int i=0; i<sz; ++i)
    {
        vecItems.push_back( parray[i] );
    }
}

template<class TYPE>
void SwapItemsInVec(vector<TYPE> &vecItems, int pos1, int pos2)
{
    YW_ASSERT_INFO(pos1<(int)vecItems.size() && pos2 <(int)vecItems.size(), "Overflow");
    TYPE tmp = vecItems[pos1];
    vecItems[pos1] = vecItems[pos2];
    vecItems[pos2] = tmp;
}

template<class TYPE>
void SwapPairGen( pair<TYPE,TYPE> &pp )
{
    TYPE t = pp.first;
    pp.first = pp.second;
    pp.second = t;
}

template<class TYPE>
int GetClosestTo(const vector<TYPE> &listNums, TYPE &target)
{
    int pos = -1;
    TYPE absDistMin = HAP_MAX_INT*1.0;
    for( int i=0; i<(int)listNums.size(); ++i )
    {
        TYPE dist1 = listNums[i]-target;
        TYPE dist2 = target-listNums[i];
        if( dist1 >= 0 && dist1 < absDistMin )
        {
            absDistMin = dist1;
            pos = i;
        }
        else if( dist2 >= 0 && dist2 < absDistMin )
        {
            absDistMin = dist2;
            pos = i;
        }
    }
    
    return pos;
}

template<class TYPE>
int GetClosestTo2(const vector<TYPE> &listNums, TYPE &target)
{
    auto const it = std::lower_bound(listNums.begin(), listNums.end(), target);
    if (it == listNums.end()) { return listNums.size()-1; }
    return it - listNums.begin();
}

template<class TYPE>
int GetClosestTo3(const TYPE *listNums, int numElems, TYPE &target)
{
    auto const it = std::lower_bound(listNums, listNums+numElems, target);
    //if (it == listNums.end()) { return listNums.size()-1; }
    return it - listNums;
}

#if 0
// donot work; so DO NOT USE
// copy from a pointer to a one-dimensional array to a 2d vector
template<class TYPE>
void Copy2DArrayToVec(TYPE *parray, int nr, int nc, vector<vector<TYPE> > &vec2DArray)
{
    vec2DArray.clear();
    for(int i=0; i<nr; ++i)
    {
        vector<TYPE> row;
        for(int j=0; j<nc; ++j)
        {
            row[j] = parray[i*nc+j];
        }
cout << "Row " << i << " is done\n";
        vec2DArray.push_back(row);
    }
}
#endif

//#if 0
template<class TYPE>
void ReduceContainerSetsForSetsGen(vector<set<TYPE> > &listSets)
{
    // give a list of sets, if one set A contains another set B, then remove
    // the intersection between them from A (not B)
    // if there is non-empty intersection but neither contains one another, DO NOTHING!
    // note: there may be multiple ways for doing this; fornow, this procedure just finds a legal solution
    vector<set<TYPE> > listSetsNext;     // we ensure there is no container sets here
    // process each input set, if it contains any set in the new list, reduces it and add to the ist
    // if contained, reduce the one already in teh list (which still introduce no new container in the old list)
    for( int i=0; i<(int)listSets.size(); ++i )
    {
        //
        set<int> setToAdd = listSets[i];
        // loop until no more container is found
        bool fCont = true;
        while(fCont == true)
        {
            fCont = false;
            for(int j=0; j<(int)listSetsNext.size(); ++j)
            {
                // test whether the new set contains any of
                set<TYPE> setInt;
                JoinSetsGen(setToAdd, listSetsNext[j], setInt);
                if( setInt.size() == listSetsNext[j].size() )
                {
                    // reduce the one to add
                    SubtractSetsGen( setToAdd, setInt );
                    fCont = true;       // since we updated the one to add (so maybe new containment emerage), need to continue looping
                }
                else
                {
                    if( setInt.size() == setToAdd.size() )
                    {
                        SubtractSetsGen( listSetsNext[j], setInt );
                    }
                }
            }
        }
//cout << "Adding a new set to next set:";
//DumpIntSet(setToAdd);
        // add it
        listSetsNext.push_back(setToAdd);
    }
    // this is the updated sets that contains no containers
    listSets = listSetsNext;
//cout << "Resulting sets: ";
//for(int i=0; i<(int)listSets.size(); ++i)
//{
//DumpIntSet(listSets[i]);
//}
}
//#endif

template<class TYPE>
void RemoveVecElementAt(vector<TYPE> &listItems, int pos)
{
    // remove the item at the pos
    if( pos <(int) listItems.size() )
    {
        listItems.erase(listItems.begin() + pos);
    }
}

template<class TYPE>
void AppendItemToBoundedVec(const TYPE &item, vector<TYPE> &listItem, int posvecToAdd, int maxSize)
{
    // add an item to the position in a vector
    // if max capacity is reached, then drop the last one
    YW_ASSERT_INFO( posvecToAdd <=(int)listItem.size(), "Position: wrong" );
    if( (int)listItem.size() == maxSize && posvecToAdd == (int)listItem.size() )
    {
        // no room for it
        return;
    }
    else
    {
        // create a new list
        vector<TYPE> listItemNew;
        int pos = 0;
        for(; pos<posvecToAdd; ++pos)
        {
            listItemNew.push_back(listItem[pos]);
        }
        // add this item
        listItemNew.push_back(item);
        // add the rest if needed
        for(; pos<(int)listItem.size(); ++pos)
        {
            if( (int)listItemNew.size() >= maxSize )
            {
                // overflow, stop
                break;
            }
            else
            {
                listItemNew.push_back( listItem[pos] );
            }
        }
        listItem = listItemNew;
    }
}

// create a combined list by merging items (and then take average)
template<class TYPE>
void PutItemsInBuckets(int numBuckets, const vector<TYPE> &listItemsIn, vector<TYPE> &itemsInBuckets)
{
    // if list is empty, then dont do it
    if( listItemsIn.size() > 0 )
    {
        // here buckets contains the average items in the original list
        int stepNum = listItemsIn.size()/numBuckets;
        if( stepNum*numBuckets < (int)listItemsIn.size() )
        {
            stepNum += 1;
        }
        int pos = 0;
        for(int i=0; i<numBuckets; ++i)
        {
            //
            bool fStop = false;
            TYPE tot = 0;
            for(int j=0; j<stepNum; ++j)
            {
                if( pos >= (int)listItemsIn.size() )
                {
                    fStop = true;
                    break;
                }
                tot += listItemsIn[pos];
                ++pos;
            }
            if( fStop == false )
            {
                itemsInBuckets.push_back( tot/stepNum );
            }
        }
    }
    // fill in 0 if otherwise
    while( (int)itemsInBuckets.size() < numBuckets )
    {
        itemsInBuckets.push_back(0);
    }
}

template<class TYPE>
void ReverseVec( vector<TYPE> &vec)
{
    //cout << "Before switching: vec = ";
    //DumpIntVec( vec );
    // This function would reverse the integer vector, i.e. vec[0] = vec[n-1] and so on
    for(int i=0; i<(int)vec.size()/2; ++i)
    {
        TYPE tmp = vec[ (int)vec.size()- 1 -i ];
        vec[ (int)vec.size()- 1 -i ] = vec[i];
        vec[i] = tmp;
    }
    //cout << "After switching: vec = ";
    //DumpIntVec( vec );
}

// extract 1D array from 2D array
template<class TYPE>
void ExtractColFrom2DArray( const vector<vector<TYPE> > &array2D, int col, vector<TYPE> &vecCol )
{
    vecCol.clear();
    YW_ASSERT_INFO(  array2D.size() == 0 || col < (int)array2D[0].size(), "Overflow"  );
    for(int i=0; i<(int)array2D.size(); ++i )
    {
        vecCol.push_back( array2D[i][col] );
    }
}

// calc mean and variance
template<class TYPE>
void CalcMeanVarianceFor( const vector<TYPE> &listVals, double &valMean, double &valVar)
{
    YW_ASSERT_INFO( listVals.size() > 0, "Empty input" );
    
    //
    double valSum = 0.0;
    for(int i=0; i<(int)listVals.size(); ++i)
    {
        valSum += (double)listVals[i];
    }
    valMean = valSum/listVals.size();
    valVar = 0.0;
    for(int i=0; i<(int)listVals.size(); ++i)
    {
        double vdiff = listVals[i] - valMean;
        valVar += vdiff*vdiff;
    }
}

template<class TYPE1, class TYPE2>
void FindMinFromPairedListGen(const vector<pair<TYPE1,TYPE2> > &vecListInput, vector<pair<TYPE1,TYPE2> > &listMinItems)
{
    // TYPE1: value (key), TYPE2: can be anything (maybe a pointer for example)
    // there may be multiple items with value (type1) are minimum; listMinItems: contain all such items
    listMinItems.clear();
    if( vecListInput.size() == 0 )
    {
        return;
    }
    TYPE1 valMin = vecListInput[0].first;
    listMinItems.push_back( vecListInput[0] );
    for( int i=1; i<(int)vecListInput.size(); ++i )
    {
        //
        if( vecListInput[i].first < valMin )
        {
            valMin = vecListInput[i].first;
            listMinItems.clear();
            listMinItems.push_back( vecListInput[i] );
        }
        else if( vecListInput[i].first == valMin )
        {
            listMinItems.push_back( vecListInput[i] );
        }
    }
}

template<class TYPE>
void FindRangeInSortedVector( const vector<TYPE> &listSortVals, const TYPE &valLB, const TYPE &valUB, int &posLB, int &posUB)
{
    // given a sorted list, and a range [lb,ub]; want to find the range in the list that contain the list
    // if there is no such range, set as -1
    posLB = 0;
    posUB = (int)listSortVals.size()-1;
    while( listSortVals[posLB] < valLB  )
    {
        ++posLB;
    }
    while( listSortVals[posUB] > valUB  )
    {
        --posUB;
    }
    if( posLB > posUB)
    {
        posLB = -1;
        posUB = -1;
    }
}

template<class TYPE>
void DumpVecWithSpace(const vector<TYPE> &listItems)
{
    // remove the item at the pos
    for(int i=0; i <(int) listItems.size(); ++i )
    {
        cout << listItems[i];
        if( i<(int)listItems.size()-1 )
        {
            cout << "  ";
        }
    }
}


template<class TYPE1,class TYPE2>
void AddingMaps( map<TYPE1,TYPE2> &mapUnion, const map<TYPE1,TYPE2> &mapToUnion )
{
    // append two maps; for duplicates (i.e. in both maps), perform a adding
    for( typename map<TYPE1,TYPE2> :: const_iterator it = mapToUnion.begin(); it != mapToUnion.end(); ++it )
    {
        //
        if( mapUnion.find(it->first) != mapUnion.end() )
        {
            // add it in
            mapUnion[it->first] += it->second;
        }
        else
        {
            //
            mapUnion.insert( typename map<TYPE1,TYPE2> :: value_type(it->first, it->second) );
        }
    }

}


template<class TYPE1,class TYPE2>
void MaxMaps( map<TYPE1,TYPE2> &mapMax, const map<TYPE1,TYPE2> &mapCmp, bool fMax )
{
    // taking the maximum value of the two maps; if fMax = false, taking the min
    for( typename map<TYPE1,TYPE2> :: const_iterator it = mapCmp.begin(); it != mapCmp.end(); ++it )
    {
        //
        if( mapMax.find(it->first) != mapMax.end() )
        {
            //
            if( (mapMax[it->first] < it->second && fMax) || ( mapMax[it->first] > it->second && fMax ==false)  )
            {
                mapMax[it->first] = it->second;
            }
        }
        else
        {
            //
            mapMax.insert( typename map<TYPE1,TYPE2> :: value_type(it->first, it->second) );
        }
    }
    
}

template<class TYPE1,class TYPE2>
void MapIntSetTo( const set<TYPE1> &sint1, const map<TYPE1,TYPE2> &mapOneToOther, set<TYPE2> &sres  )
{
    // map items in sint1 to sres; CAUTION: duplicates may be lost
    sres.clear();
    for( typename set<TYPE1 > :: iterator it = sint1.begin(); it != sint1.end(); ++it )
    {
        typename map<TYPE1,TYPE2> :: const_iterator it2 = mapOneToOther.find( *it );
        if( it2 != mapOneToOther.end() )
        {
            sres.insert( it2->second  );
        }
        else
        {
            // something very wrong
            YW_ASSERT_INFO(false, "Mapping failed");
        }
    }
}

template<class TYPE1,class TYPE2>
void MapVecToGen( const vector<TYPE1> &sint1, const map<TYPE1,TYPE2> &mapOneToOther, vector<TYPE2> &sres  )
{
    // map items in sint1 to sres;
    // YW: if some items cannot find a record in map, store the original item
    sres.clear();
    for( typename vector<TYPE1 > :: iterator it = sint1.begin(); it != sint1.end(); ++it )
    {
        typename map<TYPE1,TYPE2> :: const_iterator it2 = mapOneToOther.find( *it );
        if( it2 != mapOneToOther.end() )
        {
            sres.push_back( it2->second  );
        }
        else
        {
            // something very wrong
            sres.push_back( *it );
        }
    }
}


template<class TYPE1,class TYPE2>
void InverseMap( const map<TYPE1,TYPE2> &map1, map<TYPE2,TYPE1> &mapInv )
{
    // append two maps; for duplicates (i.e. in both maps), perform a adding
    for( typename map<TYPE1,TYPE2> :: const_iterator it = map1.begin(); it != map1.end(); ++it )
    {
        mapInv.insert( typename map<TYPE2,TYPE1> :: value_type(it->second, it->first) );
    }
    
}

template<class TYPE>
int GetItemIndexInVecGen( const vector<TYPE> &vec, TYPE &item)
{
    //
    for(unsigned int i=0; i<vec.size(); ++i)
    {
        if( vec[i] == item )
        {
            return (int)i;
        }
    }
    return -1;
}

// if vec1 smaller than vec2 pointwise
template<class TYPE>
bool IsVecSmallerThan( const vector<TYPE> &vec1, const vector<TYPE> &vec2)
{
    //
    YW_ASSERT_INFO( vec1.size() == vec2.size(), "Size: mismatch" );
    for(unsigned int i=0; i<vec1.size(); ++i)
    {
        if( vec1[i] >= vec2[i] )
        {
            return false;
        }
    }
    return true;
}

// calc Jaccard index for two sets
template<class TYPE>
double CalcJaccrdIndexForTwoSets( const set<TYPE> &s1, const set<TYPE> &s2 )
{
    //
    set<TYPE> sunion = s1;
    UnionSetsGen(sunion, s2);
    set<TYPE> sjoin;
    JoinSetsGen(s1, s2, sjoin);
    return ((double) sjoin.size() )/sunion.size() ;
}

// find the best matched set
template<class TYPE>
double GetBestJaccrdMatchedSetIn( const set<TYPE> &s1, const set<set<TYPE> > &listSet2, set<TYPE> &bestMatch )
{
    // return negative if no match found
    double score = -1.0;
    for(typename set<set<TYPE> > :: const_iterator it = listSet2.begin(); it != listSet2.end(); ++it)
    {
        double scoreStep = CalcJaccrdIndexForTwoSets(s1, *it);
        if( scoreStep > score)
        {
            score = scoreStep;
            bestMatch = *it;
        }
    }
    return score;
}

// find the leftmost common item of the two lists
template<class TYPE>
bool FindLeftmostCommonItem( const vector<TYPE> &vec1, const vector<TYPE> &vec2, TYPE &res )
{
    // for now do a simple test;
    for( int i=0; i<(int)vec1.size(); ++i )
    {
        for(int j=0; j<(int)vec2.size(); ++j)
        {
            if( vec1[i] == vec2[j] )
            {
                res = vec1[i];
                return true;
            }
        }
    }
    return false;
}

// find different items in two sets
template<class TYPE>
void FindDiffOfTwoSets(const set<TYPE> &setItems1, const set<TYPE> &setItems2, set<TYPE> &set1Only, set<TYPE> &set2Only)
{
    // find items that are in set 1 and 2 only
    set1Only.clear();
    set2Only.clear();
    for( typename set<TYPE> :: const_iterator it = setItems1.begin(); it != setItems1.end(); ++it )
    {
        if( setItems2.find(*it) == setItems2.end() )
        {
            set1Only.insert(*it);
        }
    }
    for( typename set<TYPE> :: const_iterator it = setItems2.begin(); it != setItems2.end(); ++it )
    {
        if( setItems1.find(*it) == setItems1.end() )
        {
            set2Only.insert(*it);
        }
    }
}

// remove items that are too close
template<class TYPE>
void RemoveCloseNgbrs(const set<TYPE> &setItemsOrig, const TYPE &thresDist, set<TYPE> &setItemsTrimmed)
{
    // only keep items that are not too close to its predecessor
    for(typename set<TYPE> :: const_iterator it = setItemsOrig.begin(); it != setItemsOrig.end(); ++it )
    {
        if( setItemsTrimmed.size() == 0 || *setItemsTrimmed.rbegin() + thresDist < *it )
        {
            setItemsTrimmed.insert(*it);
        }
    }
}

// add set of item sets to map, based on their sizes
template<class TYPE>
void AddItemsToMapOnSizes(const set<set<TYPE> > &setItemSets, map<int, set<set<TYPE> > > &mapItemSetsOnSize)
{
    // only keep items that are not too close to its predecessor
    mapItemSetsOnSize.clear();
    for(typename set<set<TYPE> > :: const_iterator it = setItemSets.begin(); it != setItemSets.end(); ++it )
    {
        int sz = it->size();
        mapItemSetsOnSize[sz].insert( *it );
    }
}

// add set of item sets to map, based on their sizes
template<class TYPE>
void FindCommonItemsInVecs(const vector<vector<TYPE> > &listVecs, vector<TYPE> &itemsCommon)
{
    itemsCommon.clear();
    if( listVecs.size() == 0 )
    {
        return;
    }
    //
    set<TYPE> ssCommon;
    PopulateSetByVecGen(ssCommon, listVecs[0]);
    for(int i=1; i<(int)listVecs.size(); ++i)
    {
        set<TYPE> ssCurr;
        PopulateSetByVecGen(ssCurr, listVecs[i]);
        set<TYPE> ssJoin;
        JoinSetsGen( ssCommon, ssCurr, ssJoin );
        ssCommon = ssJoin;
    }
    PopulateVecBySetGen( itemsCommon, ssCommon );
}

template<class TYPE>
void SubtractMultisetsGen( multiset<TYPE> &setMain, const multiset<TYPE> &setSubtracted )
{
    for(typename multiset<TYPE>:: const_iterator it = setSubtracted.begin(); it != setSubtracted.end(); ++it )
    {
        typename multiset<TYPE>:: iterator it2 = setMain.find(*it);
        if( it2 != setMain.end() )
        {
            setMain.erase(it2);
        }
    }
}

template<class TYPE>
void CountMultisetsGen( const multiset<TYPE> &setMS, map<TYPE, int> &mapCounts )
{
    mapCounts.clear();
    for(typename multiset<TYPE>:: const_iterator it = setMS.begin(); it != setMS.end(); ++it )
    {
        if( mapCounts.find(*it) == mapCounts.end())
        {
            mapCounts[*it] = 0;
        }
        ++mapCounts[*it];
    }
}

template<class TYPE>
void SubtractMultisetsFreqGen( multiset<TYPE> &setMain, const multiset<TYPE> &setSubtracted )
{
    map<TYPE,int> mapFreq1, mapFreq2;
    CountMultisetsGen( setMain, mapFreq1 );
    CountMultisetsGen( setSubtracted, mapFreq2 );
    setMain.clear();
    
    for(typename map<TYPE,int>:: const_iterator it = mapFreq1.begin(); it != mapFreq1.end(); ++it )
    {
        typename map<TYPE,int>:: iterator it2 = mapFreq2.find(it->first);
        int numItemsOut = it->second;
        if( it2 != mapFreq2.end() )
        {
            numItemsOut -= it2->second;
        }
        for(int i=0; i<numItemsOut; ++i)
        {
            setMain.insert(it->first);
        }
    }
}

template<class TYPE>
void CreateMapForVecGen( const vector<TYPE> &vec, map<TYPE, int> &mapIndices )
{
    mapIndices.clear();
    for(int i=0; i<(int)vec.size(); ++i)
    {
        mapIndices[ vec[i] ] = i;
    }
}

template<class TYPE>
void SegmentVecGen( const vector<TYPE> &vec, vector<pair<pair<int,int>, TYPE> > &listSegs)
{
    //
    int beg = 0;
    for(unsigned int i=0; i<vec.size(); ++i)
    {
        if( vec[i] != vec[beg] || i == (int)vec.size()-1 )
        {
            // output one segment
            int epos = i-1;
            if( i == (int)vec.size()-1 )
            {
                epos = i;
            }
            pair<int,int> pp(beg, epos);
            pair<pair<int,int>, TYPE> pp2(pp, vec[beg]);
            listSegs.push_back(pp2);
            beg = i;
        }
    }
}

template<class TYPE>
void JoinMultiset(const multiset<TYPE> &set1, const multiset<TYPE> &set2, multiset<TYPE> &setInt)
{
    for( typename multiset<TYPE> :: iterator it=set1.begin(); it!=set1.end(); ++it )
    {
        if( set2.find(*it) != set2.end() )
        {
            setInt.insert(*it);
        }
    }
}

template<class TYPE>
void FindIntersectionOfSets(const set<set<TYPE> > &setSets, set<TYPE> &setInt)
{
    if( setSets.size() == 0)
    {
        setInt.clear();
        return;
    }
    setInt = *setSets.begin();
    for( typename set<set<TYPE> > :: iterator it=setSets.begin(); it!=setSets.end(); ++it )
    {
        set<TYPE> setIntNew;
        JoinSetsGen(*it,setInt, setIntNew);
        setInt = setIntNew;
        if( setInt.size() == 0 )
        {
            return;
        }
    }
}

template <class TYPE>
TYPE FindClosestItem(const vector<TYPE> &listItems, const TYPE &val)
{
    if( listItems.size() == 0 )
    {
        // throw exception
        throw string("Empty list");
    }
    double diffABSMin = fabs(val-listItems[0]);
    TYPE valBest = listItems[0];
    for(unsigned int i=1; i<listItems.size(); ++i)
    {
        double diffABSCur = fabs(listItems[i] - val );
        if( diffABSCur < diffABSMin )
        {
            diffABSMin = diffABSCur;
            valBest = listItems[i];
        }
    }
    return valBest;
}

template <class TYPE>
void FastVecDelElement( vector<TYPE> &listItems, int posDel )
{
    // we assume the position is in the right range
    // copy the last item (back) to the later del one
    listItems[posDel] = listItems.back();
    listItems.pop_back();
}

template <class TYPE>
void FastVecDelTwoElements( vector<TYPE> &listItems, int posDel1, int posDel2 )
{
    if( posDel1 == posDel2 || posDel1 >= (int)listItems.size() || posDel2 >= (int)listItems.size() )
    {
        throw std::string("FastVecDelTwoElements :: Out of bound");
    }
    OrderInt(posDel1, posDel2);
    FastVecDelElement(listItems, posDel2);
    FastVecDelElement(listItems, posDel1);
}

template <typename Iter>
Iter nextItem(Iter iter)
{
    return ++iter;
}

// get ranking of all items in a vector
template <class T>
void GetRankOfAllElementsInVec( const vector<T> &vecItems, vector<int> &listRanks )
{
    vector<pair<T,int> > vecItemsPos;
    for(unsigned int i=0; i<vecItems.size(); ++i)
    {
        vecItemsPos.push_back(std::make_pair(vecItems[i], i));
    }
    std::sort( vecItemsPos.begin(), vecItemsPos.end() );
    listRanks.clear();
    listRanks.resize( vecItems.size() );
    for(unsigned int i=0; i<vecItemsPos.size(); ++i)
    {
        listRanks[vecItemsPos[i].second] = i;
    }
}

// cache
template<typename key_t, typename value_t>
class lru_cache {
public:
    typedef typename std::pair<key_t, value_t> key_value_pair_t;
    typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;

    lru_cache() : _max_size(10000) {}
    
    lru_cache(size_t max_size) :
        _max_size(max_size) {
    }
    void setmaxsize(int sz) {
        _max_size = sz;
    }
    void put(const key_t& key, const value_t& value) {
        auto it = _cache_items_map.find(key);
        _cache_items_list.push_front(key_value_pair_t(key, value));
        if (it != _cache_items_map.end()) {
            _cache_items_list.erase(it->second);
            _cache_items_map.erase(it);
        }
        _cache_items_map[key] = _cache_items_list.begin();
        
        if (_cache_items_map.size() > _max_size) {
            auto last = _cache_items_list.end();
            last--;
            _cache_items_map.erase(last->first);
            _cache_items_list.pop_back();
        }
    }
    
    value_t& get(const key_t& key) {
        auto it = _cache_items_map.find(key);
        if (it == _cache_items_map.end()) {
            throw std::range_error("There is no such key in cache");
        } else {
            _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
            return it->second->second;
        }
    }
    
    bool exists(const key_t& key) const {
        return _cache_items_map.find(key) != _cache_items_map.end();
    }
    
    size_t size() const {
        return _cache_items_map.size();
    }
    
    // mainly for debugging
    void getall(std::vector<key_value_pair_t> &list_key_val_pairs) const {
        list_key_val_pairs.clear();
        for(auto x: _cache_items_list) {
            list_key_val_pairs.push_back(x);
        }
    }
    
private:
    std::list<key_value_pair_t> _cache_items_list;
    std::unordered_map<key_t, list_iterator_t> _cache_items_map;
    size_t _max_size;
};

// cache
template<typename key_t, typename value_t, typename hasher>
class lru_cache2 {
public:
    typedef typename std::pair<key_t, value_t> key_value_pair_t;
    typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;

    lru_cache2() : _max_size(10000) {}
    
    lru_cache2(size_t max_size) :
        _max_size(max_size) {
    }
    
    void setmaxsize(int sz) {
        _max_size = sz;
    }
    
    void put(const key_t& key, const value_t& value) {
        auto it = _cache_items_map.find(key);
        _cache_items_list.push_front(key_value_pair_t(key, value));
        if (it != _cache_items_map.end()) {
            _cache_items_list.erase(it->second);
            _cache_items_map.erase(it);
        }
        _cache_items_map[key] = _cache_items_list.begin();
        
        if (_cache_items_map.size() > _max_size) {
            auto last = _cache_items_list.end();
            last--;
            _cache_items_map.erase(last->first);
            _cache_items_list.pop_back();
        }
    }
    
    value_t& get(const key_t& key) {
        auto it = _cache_items_map.find(key);
        if (it == _cache_items_map.end()) {
            throw std::range_error("There is no such key in cache");
        } else {
            _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
            return it->second->second;
        }
    }
    
    bool exists(const key_t& key, value_t&res) {
        auto it = _cache_items_map.find(key);
        if( it != _cache_items_map.end() )
        {
            _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
            res = it->second->second;
            return true;
        }
        else
        {
            return false;
        }
    }
    
    bool exists(const key_t& key) const {
        return _cache_items_map.find(key) != _cache_items_map.end();
    }
    
    size_t size() const {
        return _cache_items_map.size();
    }
    
    // mainly for debugging
    void getall(std::vector<key_value_pair_t> &list_key_val_pairs) const {
        list_key_val_pairs.clear();
        for(auto x: _cache_items_list) {
            list_key_val_pairs.push_back(x);
        }
    }
    
private:
    std::list<key_value_pair_t> _cache_items_list;
    std::unordered_map<key_t, list_iterator_t, hasher> _cache_items_map;
    size_t _max_size;
};

// other utilities
int GetZeroOneDiff(int x, int y);
void GetMatchingPosIntVec(const int val, const vector<int> &listVals, vector<int> &listPos);
void FormUnitVector(int numItems, int posUnit, vector<int> &vecUnit);
void FormZeroVector(int numItems, vector<int> &vecUnit);
bool AreTwoSetsCompatible( const set<int> &set1, const set<int> &set2);
bool IsSetCompatibleWithSets( const set<int> &set1, const set< set<int> > &setSets);
bool AreTwoSetsCompatible( const set<int> &set1, const set<int> &set2, int numTotElem);
bool IsSetCompatibleWithSets( const set<int> &set1, const set< set<int> > &setSets, int numTotElem);
void GetSetsIntParts(const set<int> &set1, const set<int> &set2, const set<int> &setAll, set<int> &set1Only, set<int> &set2Only, set<int> &set12, set<int> &setNone );
bool IsSignificantFraction(int totNum, int numTypes, int numOneType, double minFrac = -1.0);
void OffsetAllNumInVecBy( vector<int> &vec, int diff);
void IncAllNumInSet( set<int> &sint );
void DecAllNumInSet( set<int> &sint );
void IncAllNumInSets( set< set<int> > &setInts);
void GetNonZeroPosofVec( const vector<int> &vec, set<int> &setpos);
void GetDiffPosOfTwoVec(const vector<int> &vec1, const vector<int> &vec2, set<int> &setpos);
int GetSegIndex(int val, const vector<int> &listSegSizes);
void ComplementBoolVec(vector<bool> &listVals);
void GetAllGridPoints( int gridLB, int gridUB, int dimGrid, set< vector<int> > &setGridPts );
//void ReduceContainerSetsForSets(vector<set<int> > &listSets);
void MapIntListToAnother( const vector<int> &vec1, const vector<int> &vec2, map<int,int> &mapVec1IndexToVec2);
void FindEvenDistriPoints(double valMin, double valMax, double valResolution, int maxNumPoints, vector<double> &listChosenVals);
double CalcProductBetween(int lb, int ub);
void CreateClustersFromMultisets( const multiset< multiset<int> > &setMultisets, map<multiset<int>, vector<  multiset<int> > > &mapMultisetClusters );
void CountMultiset( const multiset<int> &s1, map<int, int> &msMap );
bool IsMultisetContainedIn( const multiset<int> &s1, const multiset<int> &s2 );
void DumpIntMultiset(const multiset<int> & ms);
void OutputStringsToFile(const char *filename, const vector<string> &listStrsOut);
//void ConvIntToVecGen( unsigned int val, vector<int> &vec, int numBits, int base);
unsigned int ConvVecToIntGen( const vector<int> &vec, int base);
//void ConvIntToVecMSBGen( unsigned int val, vector<int> &vec, int numBits, int base);
unsigned int ConvVecToIntGenMSB( const vector<int> &vec, int base);
int ConvVecToIntGenBounds( const vector<int> &vec, const vector<int> &bounds);
void ConvIntToVecGen( int val, const vector<int> &bounds, vector<int> &vec );
int ConvRowMajorPosVecToIntGenBounds( const vector<int> &vec, const vector<int> &bounds);
void ConvRowMajorIntPosToVecGen( int val, const vector<int> &bounds, vector<int> &vec );
void ClusterLinearPoints( const vector<double> &listPoints, double ratioMaxInOutCmp, vector<int> &listBkpts );
void FindConsecutiveIntervals(const set<int> &setItems, vector< pair<int,int> > &listIVs );
void ComplementIntSet(int numTot, set<int> &setToComp);
void ComplementIntVec(int numTot, vector<int> &vecToComp);
void ComplementIntSetIfMiss(int numTot, int nid, set<int> &setToComp);
void GetCountsItems(int range, const set<int> &listNumbers, vector<int> &listCnts);
void FindGapBlocksWithinPosVec(const vector<int> &posvec, int numItemsEnum, int numItemsGap, vector< pair<int,int> > &listSegs);
void ReadFromFileListOfIntsSingleLine( ifstream &fileIn, vector<int> &listInts );
void ConsDupPosForVec(const vector<int> &vecDup, vector<set<int> > &vecDupPos);
void GetAllSubsets(const set<int> &setCurr, set<set<int> > &setAllSS);
void ConvConsecutiveVecTo(const vector<int> &vecMap, const vector<int> &vecToMap, vector<int> &vecMapped);
double CalcSumDoublesPrecise(const vector<double> &listNums);


// bits operation
bool IsBitSetInt(int val, int posBit);
int ToggleBitInt(int val, int posBit);

// Prob related utilties
double CalcPoisonProb(double rate, int numEvts);

// region search utility
class LeftRightSearch
{
public:
    LeftRightSearch(int boundLeftIn, int boundRightIn, int posStartIn);
    void Init();
    void Next();
    bool IsDone() const;
    int GetCurPos() const;
    bool IsRight() const { return dirSearch == 0; }
    void ChangeDirection() { dirSearch = 1; }
    
private:
    int boundLeft;
    int boundRight;
    int posStart;
    int posCur;
    int dirSearch;
};

// automatically search around the center
class LeftRightSearchV2
{
public:
    LeftRightSearchV2(int boundLeftIn, int boundRightIn, int posStartIn);
    void Init();
    void Next();
    bool IsDone() const;
    int GetCurPos() const;
    
private:
    int boundLeft;
    int boundRight;
    int posCur;
    int posCurLeft;
    int posCurRight;
    int dirSearch;
};


#endif /* defined(____Utils4__) */
