#include <ctype.h>
#include <assert.h>
#include <sys/stat.h>
#include "khash.h"
#include "ksort.h"
#include "kstring.h"
#include "bam_endian.h"
#ifdef _USE_KNETFILE
#include "knetfile.h"
#endif
#include "pairix.h"
#include <time.h>

#define TAD_MIN_CHUNK_GAP 32768
// 1<<14 is the size of minimum bin.
#define TAD_LIDX_SHIFT    14
#define DEFAULT_DELIMITER '\t'
#define MAX_REGION_STR_LEN 10000

typedef struct {
	uint64_t u, v;
} pair64_t;

#define pair64_lt(a,b) ((a).u < (b).u)
KSORT_INIT(offt, pair64_t, pair64_lt)

typedef struct {
	uint32_t m, n;
	pair64_t *list;
} ti_binlist_t;

typedef struct {
	int32_t n, m;
	uint64_t *offset;
} ti_lidx_t;

KHASH_MAP_INIT_INT(i, ti_binlist_t)
KHASH_MAP_INIT_STR(s, int)

struct __ti_index_t {
        ti_conf_t conf;
        int32_t n, max;
        khash_t(s) *tname;
        khash_t(i) **index;
        ti_lidx_t *index2;
        int linecount;
};

struct __ti_iter_t {
        int from_first; // read from the first record; no random access
        int tid, beg, end, beg2, end2, n_off, i, finished;
        uint64_t curr_off;
        kstring_t str;
        const ti_index_t *idx;
        pair64_t *off;
        ti_intv_t intv;
};

ti_conf_t ti_conf_null = { 0, 0, 0, 0, 0, 0, 0, '\t', DEFAULT_REGION_SPLIT_CHARACTER, '#', 0 };
ti_conf_t ti_conf_gff = { 0, 1, 4, 5, 0, 0, 0, '\t', DEFAULT_REGION_SPLIT_CHARACTER, '#', 0 };
ti_conf_t ti_conf_bed = { TI_FLAG_UCSC, 1, 2,  3, 0, 0, 0, '\t', DEFAULT_REGION_SPLIT_CHARACTER, '#', 0 };
ti_conf_t ti_conf_psltbl = { TI_FLAG_UCSC, 15, 17, 18, 0, 0, 0, '\t', DEFAULT_REGION_SPLIT_CHARACTER, '#', 0 };
ti_conf_t ti_conf_sam = { TI_PRESET_SAM, 3, 4, 0, 0, 0, 0, '\t', DEFAULT_REGION_SPLIT_CHARACTER, '@', 0 };
ti_conf_t ti_conf_vcf = { TI_PRESET_VCF, 1, 2, 0, 0, 0, 0, '\t', DEFAULT_REGION_SPLIT_CHARACTER, '#', 0 };
ti_conf_t ti_conf_pairs = { TI_PRESET_PAIRS, 2, 3, 3, 4, 5, 5, '\t', DEFAULT_REGION_SPLIT_CHARACTER, '#', 0 };
ti_conf_t ti_conf_merged_nodups = { TI_PRESET_MERGED_NODUPS, 2, 3, 3, 6, 7, 7, ' ', DEFAULT_REGION_SPLIT_CHARACTER, '#', 0 };
ti_conf_t ti_conf_old_merged_nodups = { TI_PRESET_OLD_MERGED_NODUPS, 3, 4, 4, 7, 8, 8, ' ', DEFAULT_REGION_SPLIT_CHARACTER, '#', 0 };

char global_region_split_character = DEFAULT_REGION_SPLIT_CHARACTER;


/***************
 * read a line *
 ***************/

/*
int ti_readline(BGZF *fp, kstring_t *str)
{
	int c, l = 0;
	str->l = 0;
	while ((c = bgzf_getc(fp)) >= 0 && c != '\n') {
		++l;
		if (c != '\r') kputc(c, str);
	}
	if (c < 0 && l == 0) return -1; // end of file
	return str->l;
}
*/

/* Below is a faster implementation largely equivalent to the one
 * commented out above. */
int ti_readline(BGZF *fp, kstring_t *str)
{
	return bgzf_getline(fp, '\n', str);
}

/*************************************
 * get the interval from a data line *
 *************************************/

static inline int ti_reg2bin(uint32_t beg, uint32_t end)
{
	--end;
	if (beg>>14 == end>>14) return 4681 + (beg>>14);
	if (beg>>17 == end>>17) return  585 + (beg>>17);
	if (beg>>20 == end>>20) return   73 + (beg>>20);
	if (beg>>23 == end>>23) return    9 + (beg>>23);
	if (beg>>26 == end>>26) return    1 + (beg>>26);
	return 0;
}

static int get_tid(ti_index_t *idx, const char *ss)
{
	khint_t k;
	int tid;
	k = kh_get(s, idx->tname, ss);
	if (k == kh_end(idx->tname)) { // a new target sequence
		int ret, size;
		// update idx->n, ->max, ->index and ->index2
		if (idx->n == idx->max) {
			idx->max = idx->max? idx->max<<1 : 8;
			idx->index = realloc(idx->index, idx->max * sizeof(void*));
			idx->index2 = realloc(idx->index2, idx->max * sizeof(ti_lidx_t));
		}
		memset(&idx->index2[idx->n], 0, sizeof(ti_lidx_t));
		idx->index[idx->n++] = kh_init(i);
		// update ->tname
		tid = size = kh_size(idx->tname);
		k = kh_put(s, idx->tname, strdup(ss), &ret);
		kh_value(idx->tname, k) = size;
		assert(idx->n == kh_size(idx->tname));
	} else tid = kh_value(idx->tname, k);
	return tid;
}

int ti_get_intv(const ti_conf_t *conf, int len, char *line, ti_interval_t *intv)
{
	int i, b = 0, id = 1, ncols = 0;
	char *s;
	intv->ss = intv->se = 0; intv->ss2 = intv->se2 = 0; intv->beg = intv->end = -1; intv->beg2 = intv->end2 = -1;
	for (i = 0; i <= len; ++i) {
		if (line[i] == conf->delimiter || line[i] == 0) {
                        ++ncols;
			if (id == conf->sc) {
				intv->ss = line + b; intv->se = line + i;
                        } else if (conf->sc2 && id == conf->sc2) {
				intv->ss2 = line + b; intv->se2 = line + i;
			} else if (id == conf->bc) {
				// here ->beg is 0-based.
				intv->beg = intv->end = strtol(line + b, &s, 0);
				if (!(conf->preset&TI_FLAG_UCSC)) --intv->beg;
				else ++intv->end;
				if (intv->beg < 0) intv->beg = 0;
				if (intv->end < 1) intv->end = 1;
			} else if (conf->bc2 && id == conf->bc2) {
				// here ->beg is 0-based.
				intv->beg2 = intv->end2 = strtol(line + b, &s, 0);
				if (!(conf->preset&TI_FLAG_UCSC)) --intv->beg2;
				else ++intv->end2;
				if (intv->beg2 < 0) intv->beg2 = 0;
				if (intv->end2 < 1) intv->end2 = 1;
			} else if(id == conf->ec) {
				if ((conf->preset&0xffff) != TI_PRESET_VCF && (conf->preset&0xffff) != TI_PRESET_SAM) {
					intv->end = strtol(line + b, &s, 0);
				} else if ((conf->preset&0xffff) == TI_PRESET_SAM) {
					if (id == 6) { // CIGAR
						int l = 0, op;
						char *t;
						for (s = line + b; s < line + i;) {
							long x = strtol(s, &t, 10);
							op = toupper(*t);
							if (op == 'M' || op == 'D' || op == 'N') l += x;
							s = t + 1;
						}
						if (l == 0) l = 1;
						intv->end = intv->beg + l;
					}
				} else if ((conf->preset&0xffff) == TI_PRESET_VCF) {
					// FIXME: the following is NOT tested and is likely to be buggy
					if (id == 4) {
						if (b < i) intv->end = intv->beg + (i - b);
					} else if (id == 8) { // look for "END="
						int c = line[i];
						line[i] = 0;
						s = strstr(line + b, "END=");
						if (s == line + b) s += 4;
						else if (s) {
							s = strstr(line + b, ";END=");
							if (s) s += 5;
						}
						if (s) intv->end = strtol(s, &s, 0);
						line[i] = c;
					}
				}
			} else if(conf->ec2 && id == conf->ec2) {
				if ((conf->preset&0xffff) != TI_PRESET_VCF && (conf->preset&0xffff) != TI_PRESET_SAM) {
					intv->end2 = strtol(line + b, &s, 0);
                                }
	      	        } else {
				if ((conf->preset&0xffff) == TI_PRESET_SAM) {
					if (id == 6) { // CIGAR
						int l = 0, op;
						char *t;
						for (s = line + b; s < line + i;) {
							long x = strtol(s, &t, 10);
							op = toupper(*t);
							if (op == 'M' || op == 'D' || op == 'N') l += x;
							s = t + 1;
						}
						if (l == 0) l = 1;
						intv->end = intv->beg + l;
					}
				} else if ((conf->preset&0xffff) == TI_PRESET_VCF) {
					// FIXME: the following is NOT tested and is likely to be buggy
					if (id == 4) {
						if (b < i) intv->end = intv->beg + (i - b);
					} else if (id == 8) { // look for "END="
						int c = line[i];
						line[i] = 0;
						s = strstr(line + b, "END=");
						if (s == line + b) s += 4;
						else if (s) {
							s = strstr(line + b, ";END=");
							if (s) s += 5;
						}
						if (s) intv->end = strtol(s, &s, 0);
						line[i] = c;
					}
				}

                        }
			b = i + 1;
			++id;
		}
	}
	if (intv->ss == 0 || intv->se == 0 || intv->beg < 0 || intv->end < 0 ) return -1;
        if(conf->sc2 && (intv->ss2 == 0 || intv->se2 == 0)) return -1;
        if(conf->bc2 && (intv->beg2 < 0 || intv->end2 < 0)) return -1;
        if(conf->ec2 && (intv->beg2 < 0 || intv->end2 < 0)) return -1;
	return 0;
}


static int get_intv(ti_index_t *idx, kstring_t *str, ti_intv_t *intv)
{
	ti_interval_t x;
        char *str_ptr;
        char sname_double[strlen(str->s)+1];
	intv->tid = intv->beg = intv->end = intv->beg2 = intv->end2 = intv->bin = intv->bin2 =  -1;
        char region_split_character = idx->conf.region_split_character;
	if (ti_get_intv(&idx->conf, str->l, str->s, &x) == 0) {

		char c = *x.se;
                *x.se = '\0';

                if(!x.se2){ //single-chromosome
                  intv->tid = get_tid(idx, x.ss);
                } else { //double-chromosome
		  char c2 = *x.se2;
                  *x.se2 = '\0';
                  strcpy(sname_double,x.ss);
                  str_ptr = sname_double+strlen(sname_double);
                  *str_ptr=region_split_character;
                  str_ptr++;
                  strcpy(str_ptr,x.ss2);
                  intv->tid = get_tid(idx, sname_double);
                  *x.se2=c2;
                }

                *x.se = c;
		intv->beg = x.beg; intv->end = x.end;
		intv->beg2 = x.beg2; intv->end2 = x.end2;
		intv->bin = ti_reg2bin(intv->beg, intv->end);
		intv->bin2 = ti_reg2bin(intv->beg2, intv->end2);

		return (intv->tid >= 0 && intv->beg >= 0 && intv->end >= 0 && ((!idx->conf.bc2 && !idx->conf.ec2) || (intv->beg2 >=0 && intv->end2 >=0))  )? 0 : -1;
	} else {
		fprintf(stderr, "[%s] the following line cannot be parsed and skipped: %s\n", __func__, str->s);
		return -1;
	}
}

/************
 * indexing *
 ************/

// requirement: len <= LEN_MASK
static inline void insert_offset(khash_t(i) *h, int bin, uint64_t beg, uint64_t end)
{
	khint_t k;
	ti_binlist_t *l;
	int ret;
	k = kh_put(i, h, bin, &ret);
	l = &kh_value(h, k);
	if (ret) { // not present
		l->m = 1; l->n = 0;
		l->list = (pair64_t*)calloc(l->m, 16);
	}
	if (l->n == l->m) {
		l->m <<= 1;
		l->list = (pair64_t*)realloc(l->list, l->m * 16);
	}
	l->list[l->n].u = beg; l->list[l->n++].v = end;
}

static inline uint64_t insert_offset2(ti_lidx_t *index2, int _beg, int _end, uint64_t offset)
{
	int i, beg, end;
	beg = _beg >> TAD_LIDX_SHIFT;
	end = (_end - 1) >> TAD_LIDX_SHIFT;
	if (index2->m < end + 1) {
		int old_m = index2->m;
		index2->m = end + 1;
		kroundup32(index2->m);
		index2->offset = (uint64_t*)realloc(index2->offset, index2->m * 8);
		memset(index2->offset + old_m, 0, 8 * (index2->m - old_m));
	}
	if (beg == end) {
		if (index2->offset[beg] == 0) index2->offset[beg] = offset;
	} else {
		for (i = beg; i <= end; ++i)
			if (index2->offset[i] == 0) index2->offset[i] = offset;
	}
	if (index2->n < end + 1) index2->n = end + 1;
	return (uint64_t)beg<<32 | end;
}

static void merge_chunks(ti_index_t *idx)
{
	khash_t(i) *index;
	int i, l, m;
	khint_t k;
	for (i = 0; i < idx->n; ++i) {
		index = idx->index[i];
		for (k = kh_begin(index); k != kh_end(index); ++k) {
			ti_binlist_t *p;
			if (!kh_exist(index, k)) continue;
			p = &kh_value(index, k);
			m = 0;
			for (l = 1; l < p->n; ++l) {
				if (p->list[m].v>>16 == p->list[l].u>>16) p->list[m].v = p->list[l].v;
				else p->list[++m] = p->list[l];
			} // ~for(l)
			p->n = m + 1;
		} // ~for(k)
	} // ~for(i)
}

static void fill_missing(ti_index_t *idx)
{
	int i, j;
	for (i = 0; i < idx->n; ++i) {
		ti_lidx_t *idx2 = &idx->index2[i];
		for (j = 1; j < idx2->n; ++j)
			if (idx2->offset[j] == 0)
				idx2->offset[j] = idx2->offset[j-1];
	}
}

ti_index_t *ti_index_core(BGZF *fp, const ti_conf_t *conf)
{
	int ret;
	ti_index_t *idx;
	uint32_t last_bin, save_bin;
	int32_t last_coor, last_tid, save_tid;
	uint64_t save_off, last_off, lineno = 0, offset0 = (uint64_t)-1, tmp;
	kstring_t *str;

	str = calloc(1, sizeof(kstring_t));

	idx = (ti_index_t*)calloc(1, sizeof(ti_index_t));
	idx->conf = *conf;
	idx->n = idx->max = 0;
	idx->tname = kh_init(s);
	idx->index = 0;
	idx->index2 = 0;
        idx->linecount=0; 

	save_bin = save_tid = last_tid = last_bin = 0xffffffffu;
	save_off = last_off = bgzf_tell(fp); last_coor = 0xffffffffu;
	while ((ret = ti_readline(fp, str)) >= 0) {
                idx->linecount++; 
		ti_intv_t intv;
		++lineno;
		if (lineno <= idx->conf.line_skip || str->s[0] == idx->conf.meta_char) {
			last_off = bgzf_tell(fp);
			continue;
		}
		get_intv(idx, str, &intv);
                if ( intv.beg<0 || intv.end<0 )
                {
                    fprintf(stderr,"[ti_index_core] the indexes overlap or are out of bounds\n");
                    return(NULL);
                }
		if (last_tid != intv.tid) { // change of chromosomes
                if (last_tid>intv.tid )
                {
                    fprintf(stderr,"[ti_index_core] the chromosome blocks not continuous at line %llu, is the file sorted? [pos %d]\n",(unsigned long long)lineno,intv.beg+1);
                    return(NULL);
                }
		    last_tid = intv.tid;
		    last_bin = 0xffffffffu;
		} else if (last_coor > intv.beg) {
		    fprintf(stderr, "[ti_index_core] the file out of order at line %llu\n", (unsigned long long)lineno);
		    return(NULL);
		}
		tmp = insert_offset2(&idx->index2[intv.tid], intv.beg, intv.end, last_off);
		if (last_off == 0) offset0 = tmp;
		if (intv.bin != last_bin) { // then possibly write the binning index
			if (save_bin != 0xffffffffu) // save_bin==0xffffffffu only happens to the first record
				insert_offset(idx->index[save_tid], save_bin, save_off, last_off);
			save_off = last_off;
			save_bin = last_bin = intv.bin;
			save_tid = intv.tid;
			if (save_tid < 0) break;
		}
		if (bgzf_tell(fp) <= last_off) {
			fprintf(stderr, "[ti_index_core] bug in BGZF: %llx < %llx\n",
					(unsigned long long)bgzf_tell(fp), (unsigned long long)last_off);
			return(NULL);
		}
		last_off = bgzf_tell(fp);
		last_coor = intv.beg;
	}
	if (save_tid >= 0) insert_offset(idx->index[save_tid], save_bin, save_off, bgzf_tell(fp));
	merge_chunks(idx);
	fill_missing(idx);
	if (offset0 != (uint64_t)-1 && idx->n && idx->index2[0].offset) {
		int i, beg = offset0>>32, end = offset0&0xffffffffu;
		for (i = beg; i <= end; ++i) idx->index2[0].offset[i] = 0;
	}

	free(str->s); free(str);
	return idx;
}

void ti_index_destroy(ti_index_t *idx)
{
	khint_t k;
	int i;
	if (idx == 0) return;
	// destroy the name hash table
	for (k = kh_begin(idx->tname); k != kh_end(idx->tname); ++k) {
		if (kh_exist(idx->tname, k))
			free((char*)kh_key(idx->tname, k));
	}
	kh_destroy(s, idx->tname);
	// destroy the binning index
	for (i = 0; i < idx->n; ++i) {
		khash_t(i) *index = idx->index[i];
		ti_lidx_t *index2 = idx->index2 + i;
		for (k = kh_begin(index); k != kh_end(index); ++k) {
			if (kh_exist(index, k))
				free(kh_value(index, k).list);
		}
		kh_destroy(i, index);
		free(index2->offset);
	}
	free(idx->index);
	// destroy the linear index
	free(idx->index2);
	free(idx);
}

/******************
 * index file I/O *
 ******************/

void ti_index_save(const ti_index_t *idx, BGZF *fp)
{
	int32_t i, size, ti_is_be;
	khint_t k;
	ti_is_be = bam_is_big_endian();
	bgzf_write(fp, "PX2.002\1", 8);
	if (ti_is_be) {
		uint32_t x = idx->n;
		bgzf_write(fp, bam_swap_endian_4p(&x), 4);
	} else bgzf_write(fp, &idx->n, 4);
	if (ti_is_be) {
		uint32_t x = idx->linecount;
		bgzf_write(fp, bam_swap_endian_4p(&x), 4);
	} else bgzf_write(fp, &idx->linecount, 4);
	assert(sizeof(ti_conf_t) == 40);
	if (ti_is_be) { // write ti_conf_t;
		uint32_t x[6];
		memcpy(x, &idx->conf, 40);
		for (i = 0; i < 6; ++i) bgzf_write(fp, bam_swap_endian_4p(&x[i]), 4);
	} else bgzf_write(fp, &idx->conf, sizeof(ti_conf_t));
	{ // write target names
		char **name;
		int32_t l = 0;
		name = calloc(kh_size(idx->tname), sizeof(void*));
		for (k = kh_begin(idx->tname); k != kh_end(idx->tname); ++k)
			if (kh_exist(idx->tname, k))
				name[kh_value(idx->tname, k)] = (char*)kh_key(idx->tname, k);
		for (i = 0; i < kh_size(idx->tname); ++i)
			l += strlen(name[i]) + 1;
		if (ti_is_be) bgzf_write(fp, bam_swap_endian_4p(&l), 4);
		else bgzf_write(fp, &l, 4);
		for (i = 0; i < kh_size(idx->tname); ++i)
			bgzf_write(fp, name[i], strlen(name[i]) + 1);
		free(name);
	}
	for (i = 0; i < idx->n; ++i) {
		khash_t(i) *index = idx->index[i];
		ti_lidx_t *index2 = idx->index2 + i;
		// write binning index
		size = kh_size(index);
		if (ti_is_be) { // big endian
			uint32_t x = size;
			bgzf_write(fp, bam_swap_endian_4p(&x), 4);
		} else bgzf_write(fp, &size, 4);
		for (k = kh_begin(index); k != kh_end(index); ++k) {
			if (kh_exist(index, k)) {
				ti_binlist_t *p = &kh_value(index, k);
				if (ti_is_be) { // big endian
					uint32_t x;
					x = kh_key(index, k); bgzf_write(fp, bam_swap_endian_4p(&x), 4);
					x = p->n; bgzf_write(fp, bam_swap_endian_4p(&x), 4);
					for (x = 0; (int)x < p->n; ++x) {
						bam_swap_endian_8p(&p->list[x].u);
						bam_swap_endian_8p(&p->list[x].v);
					}
					bgzf_write(fp, p->list, 16 * p->n);
					for (x = 0; (int)x < p->n; ++x) {
						bam_swap_endian_8p(&p->list[x].u);
						bam_swap_endian_8p(&p->list[x].v);
					}
				} else {
					bgzf_write(fp, &kh_key(index, k), 4);
					bgzf_write(fp, &p->n, 4);
					bgzf_write(fp, p->list, 16 * p->n);
				}
			}
		}
		// write linear index (index2)
		if (ti_is_be) {
			int x = index2->n;
			bgzf_write(fp, bam_swap_endian_4p(&x), 4);
		} else bgzf_write(fp, &index2->n, 4);
		if (ti_is_be) { // big endian
			int x;
			for (x = 0; (int)x < index2->n; ++x)
				bam_swap_endian_8p(&index2->offset[x]);
			bgzf_write(fp, index2->offset, 8 * index2->n);
			for (x = 0; (int)x < index2->n; ++x)
				bam_swap_endian_8p(&index2->offset[x]);
		} else bgzf_write(fp, index2->offset, 8 * index2->n);
	}
}

static ti_index_t *ti_index_load_core(BGZF *fp)
{
	int i, ti_is_be;
	char magic[8];
	ti_index_t *idx;
	ti_is_be = bam_is_big_endian();
	if (fp == 0) {
		fprintf(stderr, "[ti_index_load_core] fail to load index.\n");
		return 0;
	}
	bgzf_read(fp, magic, 8);
	if (strncmp(magic, "PX2.002\1", 8)) {
		fprintf(stderr, "[ti_index_load] wrong magic number. Re-index if your index file was created by an earlier version of pairix.\n");
		return 0;
	}
	idx = (ti_index_t*)calloc(1, sizeof(ti_index_t));
	bgzf_read(fp, &idx->n, 4);
	if (ti_is_be) bam_swap_endian_4p(&idx->n);
	bgzf_read(fp, &idx->linecount, 4);
	if (ti_is_be) bam_swap_endian_4p(&idx->linecount);
	idx->tname = kh_init(s);
	idx->index = (khash_t(i)**)calloc(idx->n, sizeof(void*));
	idx->index2 = (ti_lidx_t*)calloc(idx->n, sizeof(ti_lidx_t));
	// read idx->conf
	bgzf_read(fp, &idx->conf, sizeof(ti_conf_t));
	if (ti_is_be) {
		bam_swap_endian_4p(&idx->conf.preset);
		bam_swap_endian_4p(&idx->conf.sc);
		bam_swap_endian_4p(&idx->conf.bc);
		bam_swap_endian_4p(&idx->conf.ec);
		bam_swap_endian_4p(&idx->conf.sc2);
		bam_swap_endian_4p(&idx->conf.bc2);
		bam_swap_endian_4p(&idx->conf.ec2);
		bam_swap_endian_4p(&idx->conf.delimiter);
		bam_swap_endian_4p(&idx->conf.meta_char);
		bam_swap_endian_4p(&idx->conf.line_skip);
	}
	{ // read target names
		int j, ret;
		kstring_t *str;
		int32_t l;
		uint8_t *buf;
		bgzf_read(fp, &l, 4);
		if (ti_is_be) bam_swap_endian_4p(&l);
		buf = calloc(l, 1);
		bgzf_read(fp, buf, l);
		str = calloc(1, sizeof(kstring_t));
		for (i = j = 0; i < l; ++i) {
			if (buf[i] == 0) {
				khint_t k = kh_put(s, idx->tname, strdup(str->s), &ret);
				kh_value(idx->tname, k) = j++;
				str->l = 0;
			} else kputc(buf[i], str);
		}
		free(str->s); free(str); free(buf);
	}
	for (i = 0; i < idx->n; ++i) {
		khash_t(i) *index;
		ti_lidx_t *index2 = idx->index2 + i;
		uint32_t key, size;
		khint_t k;
		int j, ret;
		ti_binlist_t *p;
		index = idx->index[i] = kh_init(i);
		// load binning index
		bgzf_read(fp, &size, 4);
		if (ti_is_be) bam_swap_endian_4p(&size);
		for (j = 0; j < (int)size; ++j) {
			bgzf_read(fp, &key, 4);
			if (ti_is_be) bam_swap_endian_4p(&key);
			k = kh_put(i, index, key, &ret);
			p = &kh_value(index, k);
			bgzf_read(fp, &p->n, 4);
			if (ti_is_be) bam_swap_endian_4p(&p->n);
			p->m = p->n;
			p->list = (pair64_t*)malloc(p->m * 16);
			bgzf_read(fp, p->list, 16 * p->n);
			if (ti_is_be) {
				int x;
				for (x = 0; x < p->n; ++x) {
					bam_swap_endian_8p(&p->list[x].u);
					bam_swap_endian_8p(&p->list[x].v);
				}
			}
		}
		// load linear index
		bgzf_read(fp, &index2->n, 4);
		if (ti_is_be) bam_swap_endian_4p(&index2->n);
		index2->m = index2->n;
		index2->offset = (uint64_t*)calloc(index2->m, 8);
		bgzf_read(fp, index2->offset, index2->n * 8);
		if (ti_is_be)
			for (j = 0; j < index2->n; ++j) bam_swap_endian_8p(&index2->offset[j]);
	}
	return idx;
}

ti_index_t *ti_index_load_local(const char *fnidx)
{
	BGZF *fp;
	fp = bgzf_open(fnidx, "r");
	if (fp) {
		ti_index_t *idx = ti_index_load_core(fp);
		bgzf_close(fp);
		return idx;
	} else return 0;
}

#ifdef _USE_KNETFILE
static void download_from_remote(const char *url)
{
	const int buf_size = 1 * 1024 * 1024;
	char *fn;
	FILE *fp;
	uint8_t *buf;
	knetFile *fp_remote;
	int l;
	if (strstr(url, "ftp://") != url && strstr(url, "http://") != url) return;
	l = strlen(url);
	for (fn = (char*)url + l - 1; fn >= url; --fn)
		if (*fn == '/') break;
	++fn; // fn now points to the file name
	fp_remote = knet_open(url, "r");
	if (fp_remote == 0) {
		fprintf(stderr, "[download_from_remote] fail to open remote file.\n");
		return;
	}
	if ((fp = fopen(fn, "w")) == 0) {
		fprintf(stderr, "[download_from_remote] fail to create file in the working directory.\n");
		knet_close(fp_remote);
		return;
	}
	buf = (uint8_t*)calloc(buf_size, 1);
	while ((l = knet_read(fp_remote, buf, buf_size)) != 0)
		fwrite(buf, 1, l, fp);
	free(buf);
	fclose(fp);
	knet_close(fp_remote);
}
#else
static void download_from_remote(const char *url)
{
	return;
}
#endif

static char *get_local_version(const char *fn)
{
        struct stat sbuf;
	char *fnidx = (char*)calloc(strlen(fn) + 5, 1);
	strcat(strcpy(fnidx, fn), ".px2");
	if ((strstr(fnidx, "ftp://") == fnidx || strstr(fnidx, "http://") == fnidx)) {
		char *p, *url;
		int l = strlen(fnidx);
		for (p = fnidx + l - 1; p >= fnidx; --p)
			if (*p == '/') break;
		url = fnidx; fnidx = strdup(p + 1);
		if (stat(fnidx, &sbuf) == 0) {
			free(url);
			return fnidx;
		}
		fprintf(stderr, "[%s] downloading the index file...\n", __func__);
		download_from_remote(url);
		free(url);
	}
        if (stat(fnidx, &sbuf) == 0) return fnidx;
	free(fnidx); 
        return 0;
}

const char **ti_seqname(const ti_index_t *idx, int *n)
{
	const char **names;
	khint_t k;
	*n = idx->n;
	names = calloc(idx->n, sizeof(void*));
	for (k = kh_begin(idx->tname); k < kh_end(idx->tname); ++k)
		if (kh_exist(idx->tname, k))
			names[kh_val(idx->tname, k)] = kh_key(idx->tname, k);
	return names;
}

ti_index_t *ti_index_load(const char *fn)
{
	ti_index_t *idx;
        char *fname = get_local_version(fn);
	if (fname == 0) return 0;
	idx = ti_index_load_local(fname);
	if (idx == 0) fprintf(stderr, "[ti_index_load] fail to load the index: %s\n", fname);
        if(fname) free(fname);
	return idx;
}

int ti_index_build2(const char *fn, const ti_conf_t *conf, const char *_fnidx)
{
	char *fnidx;
	BGZF *fp, *fpidx;
	ti_index_t *idx;
	if ((fp = bgzf_open(fn, "r")) == 0) {
		fprintf(stderr, "[ti_index_build2] fail to open the file: %s\n", fn);
		return -1;
	}
	idx = ti_index_core(fp, conf);
        if(!idx) return -1;
	bgzf_close(fp);
	if (_fnidx == 0) {
		fnidx = (char*)calloc(strlen(fn) + 5, 1);
		strcpy(fnidx, fn); strcat(fnidx, ".px2");
	} else fnidx = strdup(_fnidx);
	fpidx = bgzf_open(fnidx, "w");
	if (fpidx == 0) {
		fprintf(stderr, "[ti_index_build2] fail to create the index file.\n");
		free(fnidx);
		return -1;
	}
	ti_index_save(idx, fpidx);
	ti_index_destroy(idx);
	bgzf_close(fpidx);
	free(fnidx);
	return 0;
}

int ti_index_build(const char *fn, const ti_conf_t *conf)
{
	return ti_index_build2(fn, conf, 0);
}

/********************************************
 * parse a region in the format chr:beg-end *
 ********************************************/

int ti_get_tid(const ti_index_t *idx, const char *name)
{
	khiter_t iter;
	const khash_t(s) *h = idx->tname;
	iter = kh_get(s, h, name); /* get the tid */
	if (iter == kh_end(h)) return -1;
	return kh_value(h, iter);
}

int ti_parse_region(const ti_index_t *idx, const char *str, int *tid, int *begin, int *end)
{
	char *s, *p;
	int i, l, k;
	l = strlen(str);
	p = s = (char*)malloc(l+1);
	/* squeeze out "," */
	for (i = k = 0; i != l; ++i)
		if (str[i] != ',' && !isspace(str[i])) s[k++] = str[i];
	s[k] = 0;
	for (i = 0; i != k; ++i) if (s[i] == ':') break;
	s[i] = 0;
	if ((*tid = ti_get_tid(idx, s)) < 0) {
		free(s);
		return -1;
	}
	if (i == k) { /* dump the whole sequence */
		*begin = 0; *end = 1<<29; free(s);
		return 0;
	}
	for (p = s + i + 1; i != k; ++i) if (s[i] == '-') break;
	*begin = atoi(p);
	if (i < k) {
		p = s + i + 1;
		*end = atoi(p);
	} else *end = 1<<29;
	if (*begin > 0) --*begin;
	free(s);
	if (*begin > *end) return -2;

	return 0;
}


// thie function can handle both 1d and 2d query automatically
// if 1d, begin2 and end2 will have value -1.
// query string error: -1
// region_split_character not matching error: -2
int ti_parse_region2d(const ti_index_t *idx, const char *str, int *tid, int *begin, int *end, int *begin2, int *end2)
{
	char *s, *p, *sname;
	int i, l, k, h;
        int coord1s, coord1e, coord2s, coord2e, pos1s, pos2s;
        char region_split_character = idx->conf.region_split_character;

	l = strlen(str);
	p = s = (char*)malloc(l+1);
	/* squeeze out "," */
	for (i = k = 0; i != l; ++i)
	  if (str[i] != ',' && !isspace(str[i])) s[k++] = str[i];
	s[k] = 0;

        /* split by dimension */
        for(i = 0; i != k; i++) if( s[i] == region_split_character) break;
        s[i]=0;

        if(i == k) { //1d query
          *begin2=-1; *end2=-1;
          int res = ti_parse_region(idx,str,tid,begin,end);
          free(s); return (res);

        }else{ //2d query
          coord1s=0; coord1e=i; coord2s=i+1; coord2e=k;

          /* split into chr and pos */
          for (i = coord1s; i != coord1e; ++i) if (s[i] == ':') break;
          s[i]=0; pos1s=i+1;
          for (i = coord2s; i != coord2e; ++i) if (s[i] == ':') break;
          s[i]=0; pos2s=i+1;

          /* concatenate chromosomes */
          sname = (char*)malloc(l+1);
          strcpy(sname, s + coord1s);
          h=strlen(sname);
          sname[h]= region_split_character;
          strcpy(sname+h+1, s+coord2s);


  	  if ((*tid = ti_get_tid(idx, sname)) < 0) {
  		free(s); free(sname);
  		return -1;
  	  }

          /* parsing pos1 */
  	  if (pos1s-1 == coord1e) { /* dump the whole sequence */
  		*begin = 0; *end = 1<<29;
    	  } else {
            p = s + pos1s;
  	    for (i = pos1s ; i != coord1e; ++i) if (s[i] == '-') break;
  	    *begin = atoi(p);
  	    if (i < coord1e) {
  		p = s + i + 1;
  		*end = atoi(p);
    	    } else *end = 1<<29;
    	    if (*begin > 0) --*begin;
          }

          /* parsing pos2 */
  	  if (pos2s-1 == coord2e) { /* dump the whole sequence */
  		*begin2 = 0; *end2 = 1<<29;
  	  } else{
            p = s + pos2s;
  	    for (i = pos2s ; i != coord2e; ++i) if (s[i] == '-') break;
  	    *begin2 = atoi(p);
  	    if (i < coord2e) {
  		p = s + i + 1;
  		*end2 = atoi(p);
  	    } else *end2 = 1<<29;
  	    if (*begin2 > 0) --*begin2;
          }

  	  free(s); free(sname);
  	  if (*begin > *end) return -1;
     	  if (*begin2!=-1 && *begin2 > *end2) return -1;
   	  return 0;
        }
}




/*******************************
 * retrieve a specified region *
 *******************************/

#define MAX_BIN 37450 // =(8^6-1)/7+1

static inline int reg2bins(uint32_t beg, uint32_t end, uint16_t list[MAX_BIN])
{
	int i = 0, k;
	if (beg >= end) return 0;
	if (end >= 1u<<29) end = 1u<<29;
	--end;
	list[i++] = 0;
	for (k =    1 + (beg>>26); k <=    1 + (end>>26); ++k) list[i++] = k;
	for (k =    9 + (beg>>23); k <=    9 + (end>>23); ++k) list[i++] = k;
	for (k =   73 + (beg>>20); k <=   73 + (end>>20); ++k) list[i++] = k;
	for (k =  585 + (beg>>17); k <=  585 + (end>>17); ++k) list[i++] = k;
	for (k = 4681 + (beg>>14); k <= 4681 + (end>>14); ++k) list[i++] = k;
	return i;
}

ti_iter_t ti_iter_first()
{
	ti_iter_t iter;
	iter = calloc(1, sizeof(struct __ti_iter_t));
	iter->from_first = 1;
	return iter;
}


int get_linecount(const ti_index_t *idx)
{
        return(idx->linecount);
}


ti_iter_t ti_iter_query(const ti_index_t *idx, int tid, int beg, int end, int beg2, int end2 ){ //beg2, end2 should be -1 for 1d query.
	uint16_t *bins;
	int i, n_bins, n_off;
	pair64_t *off;
	khint_t k;
	khash_t(i) *index;
	uint64_t min_off;
	ti_iter_t iter = 0;

	if (beg < 0) beg = 0;
	if (end < beg) return 0;
	// initialize the iterator
	iter = calloc(1, sizeof(struct __ti_iter_t));
	iter->idx = idx; iter->tid = tid; iter->beg = beg; iter->end = end; iter->beg2 = beg2; iter->end2 = end2;  iter->i = -1;
	// random access
	bins = (uint16_t*)calloc(MAX_BIN, 2);
	n_bins = reg2bins(beg, end, bins);
	index = idx->index[tid];
	if (idx->index2[tid].n > 0) {
		min_off = (beg>>TAD_LIDX_SHIFT >= idx->index2[tid].n)? idx->index2[tid].offset[idx->index2[tid].n-1]
			: idx->index2[tid].offset[beg>>TAD_LIDX_SHIFT];
		if (min_off == 0) { // improvement for index files built by tabix prior to 0.1.4
			int n = beg>>TAD_LIDX_SHIFT;
			if (n > idx->index2[tid].n) n = idx->index2[tid].n;
			for (i = n - 1; i >= 0; --i)
				if (idx->index2[tid].offset[i] != 0) break;
			if (i >= 0) min_off = idx->index2[tid].offset[i];
		}
	} else min_off = 0; // tabix 0.1.2 may produce such index files
	for (i = n_off = 0; i < n_bins; ++i) {
		if ((k = kh_get(i, index, bins[i])) != kh_end(index))
			n_off += kh_value(index, k).n;
	}
	if (n_off == 0) {
		free(bins); return iter;
	}
	off = (pair64_t*)calloc(n_off, 16);
	for (i = n_off = 0; i < n_bins; ++i) {
		if ((k = kh_get(i, index, bins[i])) != kh_end(index)) {
			int j;
			ti_binlist_t *p = &kh_value(index, k);
			for (j = 0; j < p->n; ++j)
				if (p->list[j].v > min_off) off[n_off++] = p->list[j];
		}
	}
	if (n_off == 0) {
		free(bins); free(off); return iter;
	}
	free(bins);
	{
		int l;
		ks_introsort(offt, n_off, off);
		// resolve completely contained adjacent blocks
		for (i = 1, l = 0; i < n_off; ++i)
			if (off[l].v < off[i].v)
				off[++l] = off[i];
		n_off = l + 1;
		// resolve overlaps between adjacent blocks; this may happen due to the merge in indexing
		for (i = 1; i < n_off; ++i)
			if (off[i-1].v >= off[i].u) off[i-1].v = off[i].u;
		{ // merge adjacent blocks
			for (i = 1, l = 0; i < n_off; ++i) {
				if (off[l].v>>16 == off[i].u>>16) off[l].v = off[i].v;
				else off[++l] = off[i];
			}
			n_off = l + 1;
		}
	}
	iter->n_off = n_off; iter->off = off;
	return iter;
}


const char *ti_iter_read(BGZF *fp, ti_iter_t iter, int *len, char seqonly)
{
        if (!iter) return 0;
	if (iter->finished) return 0;
	if (iter->from_first) {
		int ret;
		if ((ret = ti_readline(fp, &iter->str)) < 0) {
			iter->finished = 1;
			return 0;
		} else {
			if (len) *len = iter->str.l;
			return iter->str.s;
		}
	}
	if (iter->n_off == 0) return 0;
	while (1) {
		int ret;
		if (iter->curr_off == 0 || iter->curr_off >= iter->off[iter->i].v) { // then jump to the next chunk
			if (iter->i == iter->n_off - 1) break; // no more chunks
			if (iter->i >= 0) assert(iter->curr_off == iter->off[iter->i].v); // otherwise bug
			if (iter->i < 0 || iter->off[iter->i].v != iter->off[iter->i+1].u) { // not adjacent chunks; then seek
				bgzf_seek(fp, iter->off[iter->i+1].u, SEEK_SET);
				iter->curr_off = bgzf_tell(fp);
			}
			++iter->i;
		}
		if ((ret = ti_readline(fp, &iter->str)) >= 0) {
			iter->curr_off = bgzf_tell(fp);
			if (iter->str.s[0] == iter->idx->conf.meta_char) continue;
			get_intv((ti_index_t*)iter->idx, &iter->str, &iter->intv);
                        if(seqonly) 
                                if(iter->intv.tid == iter->tid) { 
                                      if (len) *len = iter->str.l;
                                      return iter->str.s;  // compare only chromosome (chromosome pair) not position.
                                } else break;
			else if (iter->intv.tid != iter->tid || iter->intv.beg >= iter->end ) break; // no need to proceed
			else if (iter->intv.end > iter->beg && iter->end > iter->intv.beg && ( iter->beg2==-1 || iter->end2==-1 || (iter->intv.end2 > iter->beg2 && iter->end2 > iter->intv.beg2) )) {
				if (len) *len = iter->str.l;
				return iter->str.s;
			} else continue; 
		} else break; // end of file
	}
	iter->finished = 1;
	return 0;
}

void ti_iter_destroy(ti_iter_t iter)
{
	if (iter) {
		free(iter->str.s); free(iter->off);
		free(iter);
	}
}

int ti_fetch(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end, void *data, ti_fetch_f func)
{
	ti_iter_t iter;
	const char *s;
	int len;
	iter = ti_iter_query(idx, tid, beg, end, -1, -1);
	while ((s = ti_iter_read(fp, iter, &len, 0)) != 0)
		func(len, s, data);
	ti_iter_destroy(iter);
	return 0;
}

int ti_fetch_2d(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end, int beg2, int end2, void *data, ti_fetch_f func)
{
	ti_iter_t iter;
	const char *s;
	int len;
	iter = ti_iter_query(idx, tid, beg, end, beg2, end2);
	while ((s = ti_iter_read(fp, iter, &len, 0)) != 0)
		func(len, s, data);
	ti_iter_destroy(iter);
	return 0;
}

const ti_conf_t *ti_get_conf(ti_index_t *idx) { return idx? &idx->conf : 0; }

/* Get 0-based column index and delimiter */
int ti_get_sc(ti_index_t *idx) { return idx? idx->conf.sc-1 : -1; }
int ti_get_sc2(ti_index_t *idx) { return idx? idx->conf.sc2-1 : -1; }
int ti_get_bc(ti_index_t *idx) { return idx? idx->conf.bc-1 : -1; }
int ti_get_bc2(ti_index_t *idx) { return idx? idx->conf.bc2-1 : -1; }
int ti_get_ec(ti_index_t *idx) { return idx? idx->conf.ec-1 : -1; }
int ti_get_ec2(ti_index_t *idx) { return idx? idx->conf.ec2-1 : -1; }
char ti_get_delimiter(ti_index_t *idx) { return idx? idx->conf.delimiter : 0; }


/*******************
 * High-level APIs *
 *******************/

pairix_t *ti_open(const char *fn, const char *fnidx)
{
	pairix_t *t;
	BGZF *fp;
	if ((fp = bgzf_open(fn, "r")) == 0) return 0;
	t = calloc(1, sizeof(pairix_t));
	t->fn = strdup(fn);
	if (fnidx) t->fnidx = strdup(fnidx);
	t->fp = fp;
	return t;
}

void ti_close(pairix_t *t)
{
	if (t) {
		bgzf_close(t->fp);
		if (t->idx) ti_index_destroy(t->idx);
		free(t->fn); free(t->fnidx);
		free(t);
	}
}

int ti_lazy_index_load(pairix_t *t)
{
	if (t->idx == 0) { // load index
		if (t->fnidx) t->idx = ti_index_load_local(t->fnidx);
		else t->idx = ti_index_load(t->fn);
		if (t->idx == 0) return -1; // fail to load index
	}
	return 0;
}


// reg can contain wildcard for one of the two mates ('*')
sequential_iter_t *ti_querys_2d_general(pairix_t *t, const char *reg)
{
   int n_seqpair_list;
   int n_sub_list;
   char *sp, **chr1list, **chr2list;
   char *chrend;
   char chronly=1;
   int i;

   char region_split_character = t->idx->conf.region_split_character;

   if((sp = strchr(reg, region_split_character)) != NULL){
      if(sp == reg + 1 && reg[0]=='*') {    // '*|c:s-e'
         char *chr2 = sp + 1;

         // get the second chromosome and the list of the first chromosomes pairing with the second chromosome.
         // extract only chr part temporarily (split the beg and end part for now)
         if( (chrend = strchr(chr2, ':')) != NULL) { 
            *chrend=0; chronly=0;  
         }
         char **chrpairlist = ti_seqname(t->idx, &n_seqpair_list);
         chr1list = get_seq1_list_for_given_seq2(chr2, chrpairlist, n_seqpair_list, &n_sub_list);
         if(chronly==0) *chrend=':';  // revert to original region string including beg and end
         // create an array of regions in string.
         char **regions = malloc(n_sub_list * sizeof(char*));
         for(i=0;i<n_sub_list;i++){
            regions[i] = malloc((strlen(chr1list[i]) + strlen(chr2) + 2) * sizeof(char));
            strcpy(regions[i], chr1list[i]); 
            *(regions[i] + strlen(regions[i]) + 1) = 0;
            *(regions[i] + strlen(regions[i])) = region_split_character; 
            strcat(regions[i], chr2);
         }
         free(chrpairlist);
         for(i=0;i<n_sub_list;i++) free(chr1list[i]);
         free(chr1list);

         // multi-region query
         sequential_iter_t *siter = ti_querys_2d_multi(t, regions, n_sub_list);
         for(i=0;i<n_sub_list;i++) free(regions[i]);
         free(regions);
         return(siter);

      } else if(strlen(sp) == 2 && sp[1]=='*'){  // 'c:s-e|*'
         *sp=0; char *chr1 = reg;
         if( (chrend = strchr(chr1, ':')) != NULL) { 
            *chrend=0; chronly=0;
         }
         char **chrpairlist = ti_seqname(t->idx, &n_seqpair_list);
         chr2list = get_seq2_list_for_given_seq1(chr1, chrpairlist, n_seqpair_list, &n_sub_list);
         if(chronly==0) *chrend=':';  // revert to original region string including beg and end

         // create an array of regions in string.
         char **regions = malloc(n_sub_list * sizeof(char*));
         for(i=0;i<n_sub_list;i++){
            regions[i] = malloc((strlen(chr2list[i]) + strlen(chr1) + 2) * sizeof(char));
            strcpy(regions[i], chr1);
            *(regions[i] + strlen(regions[i]) + 1) = 0;
            *(regions[i] + strlen(regions[i])) = region_split_character;
            strcat(regions[i], chr2list[i]);
         }
         free(chrpairlist);
         for(i=0;i<n_sub_list;i++) free(chr2list[i]);
         free(chr2list);

         sequential_iter_t *siter = ti_querys_2d_multi(t, regions, n_sub_list);
         for(i=0;i<n_sub_list;i++) free(regions[i]);
         free(regions);
         return(siter);

      } else {  // no wildcard
         sequential_iter_t *siter = create_sequential_iter(t);
         add_to_sequential_iter ( siter, ti_querys_2d(t,reg) );
         return(siter);
      }
   }else {  // 1d query (let's regard it as 1d query on 1d index for now)
         sequential_iter_t *siter = create_sequential_iter(t);
         add_to_sequential_iter ( siter, ti_querys_2d(t,reg) );
         return(siter);
   }
}



ti_iter_t ti_queryi(pairix_t *t, int tid, int beg, int end)
{
        return ti_queryi_2d(t,tid,beg,end,-1,-1);
}

sequential_iter_t *ti_queryi_general(pairix_t *t, int tid, int beg, int end)
{
        return ti_queryi_2d_general(t,tid,beg,end,-1,-1);
}

ti_iter_t ti_queryi_2d(pairix_t *t, int tid, int beg, int end, int beg2, int end2)
{
	if (tid < 0) return ti_iter_first();
	if (ti_lazy_index_load(t) != 0) return 0;
	return ti_iter_query(t->idx, tid, beg, end, beg2, end2);
}

sequential_iter_t *ti_queryi_2d_general(pairix_t *t, int tid, int beg, int end, int beg2, int end2)
{
    sequential_iter_t *siter = create_sequential_iter(t);
    add_to_sequential_iter ( siter, ti_queryi_2d(t,tid,beg,end,beg2,end2) );
    return(siter);
}

ti_iter_t ti_querys(pairix_t *t, const char *reg)
{
        return ti_querys_2d(t,reg);
}

sequential_iter_t *ti_querys_general(pairix_t *t, const char *reg)
{   
    sequential_iter_t *siter = create_sequential_iter(t);
    add_to_sequential_iter ( siter, ti_querys(t, reg) );
    return(siter);
}


ti_iter_t ti_querys_2d(pairix_t *t, const char *reg)
{
	int tid, beg, end, beg2, end2;
	if (reg == 0) return ti_iter_first();
	if (ti_lazy_index_load(t) != 0) return 0;
	if (ti_parse_region2d(t->idx, reg, &tid, &beg, &end, &beg2, &end2) < 0) return 0;
	return ti_iter_query(t->idx, tid, beg, end, beg2, end2);
}

sequential_iter_t *ti_querys_2d_multi(pairix_t *t, const char **regs, int nRegs)
{
    sequential_iter_t *siter = create_sequential_iter(t);
    int i;
    for(i=0;i<nRegs;i++){
      add_to_sequential_iter ( siter, ti_querys_2d(t, regs[i]) );
    }
    return(siter);
}

ti_iter_t ti_query(pairix_t *t, const char *name, int beg, int end)
{
	int tid;
	if (name == 0) return ti_iter_first();
	// then need to load the index
	if (ti_lazy_index_load(t) != 0) return 0;
	if ((tid = ti_get_tid(t->idx, name)) < 0) return 0;
	return ti_iter_query(t->idx, tid, beg, end, -1, -1);
}

sequential_iter_t *ti_query_general(pairix_t *t, const char *name, int beg, int end)
{
    sequential_iter_t *siter = create_sequential_iter(t);
    add_to_sequential_iter (siter, ti_query(t, name, beg, end));
    return(siter);
}

ti_iter_t ti_query_2d(pairix_t *t, const char *name, int beg, int end, const char *name2, int beg2, int end2)
{
	int tid;
        char namepair[1000], *str_ptr;
        char region_split_character = t->idx->conf.region_split_character;

        strcpy(namepair,name);
        str_ptr = namepair + strlen(namepair);
        *str_ptr = region_split_character;
        str_ptr++;
        strcpy(str_ptr,name2);

	if (name == 0) return ti_iter_first();
	// then need to load the index
	if (ti_lazy_index_load(t) != 0) return 0;
	if ((tid = ti_get_tid(t->idx, namepair)) < 0) return 0;
	return ti_iter_query(t->idx, tid, beg, end, beg2, end2);
}

sequential_iter_t *ti_query_2d_general(pairix_t *t, const char *name, int beg, int end, const char *name2, int beg2, int end2)
{
    sequential_iter_t *siter = create_sequential_iter(t);
    add_to_sequential_iter (siter, ti_query_2d(t, name, beg, end, name2, beg2, end2));
    return(siter);
}

int ti_querys_tid(pairix_t *t, const char *reg)
{
        return ti_querys_2d_tid(t,reg);
}

int ti_querys_2d_tid(pairix_t *t, const char *reg)
{
	int tid, beg, end, beg2, end2, parse_err;
	if (reg == 0) return -3;  // null region
	if (ti_lazy_index_load(t) != 0) return -3; // index not loaded
	parse_err = ti_parse_region2d(t->idx, reg, &tid, &beg, &end, &beg2, &end2);
        if(tid != -1 && tid != -3 && parse_err == -1) tid = -2;  // -2 is parsing error. 
        return(tid);  // -1 means chromosome (pair) doesn't exist.
}

int ti_query_tid(pairix_t *t, const char *name, int beg, int end)
{
	if (name == 0) return -3 ;
	// then need to load the index
	if (ti_lazy_index_load(t) != 0) return -3;
	if (ti_get_tid(t->idx, name) <0) return -1;
        if (beg > end) return -2;
        else return 0;
}

int ti_query_2d_tid(pairix_t *t, const char *name, int beg, int end, const char *name2, int beg2, int end2)
{
        char namepair[1000], *str_ptr;
        char region_split_character = t->idx->conf.region_split_character;

        strcpy(namepair,name);
        str_ptr = namepair + strlen(namepair);
        *str_ptr = region_split_character;
        str_ptr++;
        strcpy(str_ptr,name2);

	if (name == 0) return ti_iter_first();
	// then need to load the index
	if (ti_lazy_index_load(t) != 0) return 0;
        if(ti_get_tid(t->idx, namepair) <0) return -1;
        if (beg > end) return -2;
        if (beg2 > end2) return -2;
        else return 0;
}


const char *ti_read(pairix_t *t, ti_iter_t iter, int *len)
{
	return ti_iter_read(t->fp, iter, len, 0);
}


// create an empty sequential iterator
sequential_iter_t *create_sequential_iter(pairix_t *t)
{
  sequential_iter_t *siter = malloc(sizeof(sequential_iter_t));
  siter->t = t;
  siter->n = 0;
  siter->curr = 0;
  siter->iter = NULL; 
  return(siter);
}

// destroy a sequential iterator
void destroy_sequential_iter(sequential_iter_t *siter)
{
  int i;
  if(siter){
    for(i=0;i<siter->n;i++) ti_iter_destroy(siter->iter[i]);
    free(siter->iter);
    free(siter);
  }
}

// add an iter element to a sequential iterator, the size is dynamically incremented.
void add_to_sequential_iter(sequential_iter_t *siter, ti_iter_t iter)
{
  siter->n++;
  siter->iter=realloc(siter->iter, siter->n * sizeof(ti_iter_t));
  siter->iter[siter->n-1] = iter;
}

// create an empty merged_iter_t struct with predefined length
merged_iter_t *create_merged_iter(int n)
{
   int i;
   merged_iter_t *miter = malloc(sizeof(merged_iter_t));
   if(miter){
     if( miter->iu = calloc(n,sizeof(iter_unit*))) {
       miter->n = n;
       miter->first=1;
       for(i=0;i<n;i++) miter->iu[i] = calloc(1,sizeof(iter_unit));
     } else { fprintf(stderr,"Cannot allocate memory for iter_unit array in merged_iter_t\n"); }
     return(miter);
   } else { 
      fprintf(stderr,"Cannot allocate memory for merged_iter_t\n"); 
      return(NULL); 
   }
}

void destroy_merged_iter(merged_iter_t *miter)
{
  int i;
  if(miter && miter->n>0 && miter->iu){
    for(i=0;i<miter->n;i++){
      ti_iter_destroy(miter->iu[i]->iter);
      if(miter->iu[i]->len) free(miter->iu[i]->len); 
      if(miter->iu[i]) free(miter->iu[i]); 
    }
    free(miter->iu);
    free(miter);
  }
}

// fill in an iter_unit struct given a pointer
void create_iter_unit(pairix_t *t, ti_iter_t iter, iter_unit *iu)
{
   if(iu){
     iu->t=t; iu->iter=iter; iu->len=malloc(sizeof(int)); iu->s=NULL;
   }
}


int compare_iter_unit (const void *a, const void *b)
{
  iter_unit *aa=*(iter_unit**)a;
  iter_unit *bb=*(iter_unit**)b;
  if (aa == NULL || aa->s == NULL) {
    if (bb == NULL) return(0);
    else if (bb->s == NULL) return(0);
    else return(1); // non-NULL iter should go before a NULL iter.
  } else if(bb == NULL || bb->s == NULL) {
    return(-1);
  } else {
    int res = aa->iter->intv.beg - bb->iter->intv.beg;  // sort first by beg
    if (res == 0 && aa->iter->intv.beg2 && bb->iter->intv.beg2) return ( aa->iter->intv.beg2 - bb->iter->intv.beg2 );  // sort second by beg2 (skip if beg2 doesn't exist - 1D case)
    else return (res); 
  }
}


// read method for merged_iter
const char *merged_ti_read(merged_iter_t *miter, int *len)
{
    iter_unit *tmp_iu;
    int i,k;
    char *s;
    char seqonly=1; // seqonly=1 forces ti_iter_read to only check tid not positions. (faster, given the position comparison is done within this function)

    if(!miter) { fprintf(stderr,"Null merged_iter_t\n"); return(NULL); }
    if(miter->n<=0) { fprintf(stderr,"No iter_unit lement in merged_iter_t\n"); return(NULL); }
    iter_unit **miu = miter->iu;

    // initi al sorting of the iterators based on their first entry
    if (miter->first){ 
      for(i=0;i<miter->n;i++) { 
        miu[i]->s = ti_iter_read(miu[i]->t->fp, miu[i]->iter, miu[i]->len, seqonly); 
      } // get first entry for each iter
      qsort((void*)(miu), miter->n, sizeof(iter_unit*), compare_iter_unit);  // sort by the first entry. finished iters go to the end.
      miter->first=0;
    }
    else if(miu[0]->s==NULL) {
      miu[0]->s = ti_iter_read(miu[0]->t->fp, miu[0]->iter, miu[0]->len, seqonly); // get next entry for the flushed iter 
   
      //qsort((void*)(miu), miter->n, sizeof(iter_unit*), compare_iter_unit); // sort again
 
      // put it at the right place in the sorted iu array
      k=0;
      while( k < miter->n-1 && compare_iter_unit((void*)(miu), (void*)(miu + k + 1 )) >0 ) k++;
      if (k>=1) {
        tmp_iu = miu[0];
        for(i=1;i<=k;i++) miu[i-1] = miu[i];
        miu[k]= tmp_iu; 
      }
    }
    if(miu[0]->iter==NULL) return(NULL);

    // flush the lowest
    s=miu[0]->s;
    miu[0]->s=NULL;

    *len = *(miu[0]->len);

    return ( s ); 
}


const char *sequential_ti_read(sequential_iter_t *siter, int *len)
{
    if(!siter) { fprintf(stderr,"Null sequential_iter_t\n"); return(NULL); }
    if(siter->n<=0) { fprintf(stderr,"No iter_unit lement in merged_iter_t\n"); return(NULL); }

    char *s = ti_iter_read(siter->t->fp,siter->iter[siter->curr], len, 0); 
    while(s==NULL && siter->curr < siter->n - 1) { 
      siter->curr++; 
      s = ti_iter_read(siter->t->fp,siter->iter[siter->curr], len, 0); 
    }
    return s;
}




//compare two strings, but different from strcmp.
//for a pair of strings 'chr1|chr2' vs 'chr10|chr13', it compares chr1 vs chr10 first and then do chr2 vs chr13. This results in an ordering different from strcmp-based sort, because 'chr10' comes before 'chr1|' whereas 'chr1' comes before 'chr10'.
int strcmp2d(const void* a, const void* b)
{
    //char aa[strlen(*(char**)a)], bb[strlen(*(char**)b)];
    //strcpy(aa, *(char**)a);
    //strcpy(bb, *(char**)b);
    int res;
    char c,d;
    char *aa=*(char**)a;
    char *bb=*(char**)b;
    char *a2,*b2;
    char *a_split = strchr(aa, global_region_split_character);
    char *b_split = strchr(bb, global_region_split_character);
    if(a_split && b_split) {  // 2D name
      c = a_split[0]; d=b_split[0];
      a2 = a_split+1; b2 = b_split+1;
      a_split[0]=0;
      b_split[0]=0;
      if ((res = strcmp(aa, bb))==0) res =strcmp(a2, b2) ;
      a_split[0]=c; b_split[0]=d;
      return (res) ;
    } else if(!a_split && !b_split) {   // 1D name
       return strcmp(aa, bb);
    } else {  // weird mix?
       fprintf(stderr, "Warning: Mix of 1D and 2D indexed files? (%s vs %s)\n",aa,bb);
       return strcmp(aa, bb);
    }
}


// same as strcmp, argument types modified to be compatible with qsort
int strcmp1d(const void* a, const void* b)
{
   return( strcmp(*(char**)a, *(char**)b) );
}

pairix_t *load_from_file(char *fn)
{
      char *fnidx = calloc(strlen(fn) + 5, 1);
      strcat(strcpy(fnidx, fn), ".px2");
      pairix_t *tb = ti_open(fn, fnidx);
      free(fnidx);
      if(tb){
         tb->idx = ti_index_load(fn);
      } else return 0;
      return(tb);
}


// The returned array must be freed later.
char** get_unique_merged_seqname(pairix_t **tbs, int n, int *pn_uniq_seq)
{
     int i,j, k, len, n_seq_list=0;
     char **conc_seq_list=NULL;
     char **seqnames=NULL;
     if(n<=0) { fprintf(stderr, "Null pairix_t array\n"); return(0); }
     for(i=0;i<n;i++){
      if(tbs[i] && tbs[i]->idx){
            seqnames = ti_seqname(tbs[i]->idx,&len);
            if(seqnames){
              conc_seq_list = realloc(conc_seq_list, sizeof(char*)*(n_seq_list+len));
              for(k=0,j=n_seq_list;j<n_seq_list+len;j++,k++)
                conc_seq_list[j] = seqnames[k];
              n_seq_list += len;
              free(seqnames);
            }else { fprintf(stderr, "Cannot retrieve seqnames.\n"); return(0); }
      } else {
         for(j=0;j<i;j++) ti_close(tbs[j]);
         if(conc_seq_list) free(conc_seq_list);
         fprintf(stderr,"Not all files can be open.\n");
         return (0);
      }
    }

    if(conc_seq_list){
      // given an array, do sort|uniq, but doing it as if sorting by two chromosome columns (e.g. by chr1 first then chr2) rather than by a single merged chromosome pair string (e.g. 'chr1|chr2')
      qsort(conc_seq_list, n_seq_list, sizeof(char*), strcmp2d);  // This part does the sorting. see strcmp2d for more details.
      char **uniq_seq_list = uniq(conc_seq_list, n_seq_list, pn_uniq_seq);
      free(conc_seq_list);
      return ( uniq_seq_list ); 
    } else { fprintf(stderr,"Null concatenated seq list\n"); return(0); }
}


// given a chromosome for mate1 (seq1='chr1') return the array containing all seqpairs matching seq1 ('chr1', 'chr2', ... for seqpairs 'chr1|chr1', 'chr1|chr2', ... )
// the returned subarray contains copies of seq2 sequences (need to be freed later) 
char **get_seq2_list_for_given_seq1(char *seq1, char **seqpair_list, int n_seqpair_list, int *pn_sub_list)
{
    int i,k;
    char *b_split, b;
    char **sublist;

    // first round, count the number 
    k=0;
    for(i=0;i<n_seqpair_list;i++){
      b_split = strchr(seqpair_list[i], global_region_split_character);
      b = b_split[0];
      b_split[0] = 0;
      if ( strcmp(seqpair_list[i], seq1)==0 ) k++;
      b_split[0] = b;
    }
    *pn_sub_list = k;

    // second round, get the list of pointers
    sublist = malloc((*pn_sub_list)*sizeof(char*));
    k=0;
    for(i=0;i<n_seqpair_list;i++){
      b_split = strchr(seqpair_list[i], global_region_split_character);
      b = b_split[0];
      b_split[0] = 0;
      if ( strcmp(seqpair_list[i], seq1)==0 ) { 
         sublist[k] = malloc((strlen(b_split+1)+1)*sizeof(char));
         strcpy(sublist[k], b_split+1); 
         k++; 
      }
      b_split[0] = b;
    }
    assert (k = *pn_sub_list);

    return(sublist);
}


// given a chromosome for mate2 (seq2='chr1') return the array containing all seqpairs matching seq2 ('chr1','chr2', ... for seqpairs 'chr1|chr1', 'chr2|chr1', ... )
// the returned subarray contains copies of seq1 sequences (need to be freed later) 
char **get_seq1_list_for_given_seq2(char *seq2, char **seqpair_list, int n_seqpair_list, int *pn_sub_list)
{
    int i,k;
    char *b_split;
    char **sublist;

    // first round, count the number 
    k=0;
    for(i=0;i<n_seqpair_list;i++){
      b_split = strchr(seqpair_list[i], global_region_split_character);
      if ( strcmp(b_split+1, seq2)==0 ) k++;
    }
    *pn_sub_list = k;

    // second round, get the list of pointers
    sublist = malloc((*pn_sub_list)*sizeof(char*));
    k=0;
    for(i=0;i<n_seqpair_list;i++){
      b_split = strchr(seqpair_list[i], global_region_split_character);
      if ( strcmp(b_split+1, seq2)==0 ) { 
         *b_split=0;
         sublist[k] = malloc((strlen(seqpair_list[i])+1)*sizeof(char));
         strcpy(sublist[k], seqpair_list[i]); 
         *b_split =  global_region_split_character;
         k++;
      }
    }
    assert (k = *pn_sub_list);

    return(sublist);
}

/* convert string 'region1|region2' to 'region2|region1' */
char *flip_region ( char* s, char region_split_character) {
    char s_flp[MAX_REGION_STR_LEN];
    int l, i, l2, split_pos;  
    l = strlen(s);
    for(i = 0; i != l; i++) if( s[i] == region_split_character) break;
    s[i]=0;
    split_pos = i;
    l2 = l-1-i;
    strcpy(s_flp, s + i + 1);
    s_flp[l2] = region_split_character;
    strcpy(s_flp + l2 + 1, s);
    s[i]= region_split_character;
    return(s_flp);
}

// given a chromosome for mate1 (seq1='chr1') return the array containing all seqpairs matching seq1 ('chr1|chr1', 'chr1|chr2', ... )
// the returned subarray contains pointers to the original seqpair_list elements.
char **get_sub_seq_list_for_given_seq1(char *seq1, char **seqpair_list, int n_seqpair_list, int *pn_sub_list)
{
    int i,k;
    char *b_split, b;
    char **sublist;

    // first round, count the number 
    k=0;
    for(i=0;i<n_seqpair_list;i++){
      b_split = strchr(seqpair_list[i], global_region_split_character);
      b = b_split[0];
      b_split[0] = 0;
      if ( strcmp(seqpair_list[i], seq1)==0 ) k++;
      b_split[0] = b;
    }
    *pn_sub_list = k;

    // second round, get the list of pointers
    sublist = malloc((*pn_sub_list)*sizeof(char*));
    k=0;
    for(i=0;i<n_seqpair_list;i++){
      b_split = strchr(seqpair_list[i], global_region_split_character);
      b = b_split[0];
      b_split[0] = 0;
      if ( strcmp(seqpair_list[i], seq1)==0 ) { sublist[k] = seqpair_list[i]; k++; }
      b_split[0] = b;
    }
    assert (k = *pn_sub_list);

    return(sublist);
}


// given a chromosome for mate2 (seq2='chr1') return the array containing all seqpairs matching seq2 ('chr1|chr1', 'chr2|chr1', ... )
// the returned subarray contains pointers to the original seqpair_list elements.
char **get_sub_seq_list_for_given_seq2(char *seq2, char **seqpair_list, int n_seqpair_list, int *pn_sub_list)
{
    int i,k;
    char *b_split;
    char **sublist;

    // first round, count the number 
    k=0;
    for(i=0;i<n_seqpair_list;i++){
      b_split = strchr(seqpair_list[i], global_region_split_character);
      if ( strcmp(b_split+1, seq2)==0 ) k++;
    }
    *pn_sub_list = k;

    // second round, get the list of pointers
    sublist = malloc((*pn_sub_list)*sizeof(char*));
    k=0;
    for(i=0;i<n_seqpair_list;i++){
      b_split = strchr(seqpair_list[i], global_region_split_character);
      if ( strcmp(b_split+1, seq2)==0 ) { sublist[k] = seqpair_list[i]; k++; }
    }
    assert (k = *pn_sub_list);

    return(sublist);
}


// given a list of seqpairs (either unique or non-unique), return an sorted array of unique seq1 list
// the returned array must be freed later. (first for the string elements and then itself)
char **get_seq1_list_from_seqpair_list(char** seqpair_list, int n_seqpair_list, int *pn_seq1)
{
    if(seqpair_list){
        char *b_split,b;
        char *seq1_list[n_seqpair_list];
        char **uniq_seq1_list=NULL;
        char *seqpair;
        int i;

        // extract seq1 from all seqpairs in the seqpair_list 
        for(i=0;i<n_seqpair_list;i++){
          seqpair = seqpair_list[i];
          b_split = strchr(seqpair, global_region_split_character);
          b = b_split[0];
          b_split[0] = 0;
          seq1_list[i] = malloc((strlen(seqpair)+1)*sizeof(char));
          strcpy(seq1_list[i], seqpair);
          b_split[0] = b;
        }

        // uniquefy
        qsort(seq1_list, n_seqpair_list, sizeof(char*), strcmp1d); // sort first
        uniq_seq1_list = uniq(seq1_list, n_seqpair_list, pn_seq1); // uniq

        // free the non-unique seq1 list
        for(i=0;i<n_seqpair_list;i++) free(seq1_list[i]);

        // return the uniq list
        return(uniq_seq1_list);

    } else {
      fprintf(stderr, "Null seqpair list\n"); 
      return(0); 
    } 
}

char **uniq(char** seq_list, int n_seq_list, int *pn_uniq_seq)
{
    int k,i,prev_i;
    char **uniq_seq_list=NULL;

    // first round, just simulate uniquifying to get the length of the output array.
    k=0; prev_i=0;
    for(i=1;i<n_seq_list;i++){
      if ( strcmp(seq_list[i], seq_list[prev_i])==0) continue;   // This part does the uniquifying
      else {
        k++;
        prev_i=i;
      }
    }
    *pn_uniq_seq=k+1;
    fprintf(stderr,"(total %d unique seq names)\n",*pn_uniq_seq);

    // second round, allocate memory and actually create an array containing uniquified array
    if( (uniq_seq_list = malloc((*pn_uniq_seq)*sizeof(char*))) != NULL ) {
      k=0; prev_i=0; 
      uniq_seq_list[0] = malloc((strlen(seq_list[0])+1)*sizeof(char));
      strcpy(uniq_seq_list[0],seq_list[0]);
      for(i=1;i<n_seq_list;i++){
        if ( strcmp(seq_list[i], seq_list[prev_i])==0) continue;  // uniquifying
        else {
          ++k;
          uniq_seq_list[k] = malloc((strlen(seq_list[i])+1)*sizeof(char));
          strcpy(uniq_seq_list[k],seq_list[i]);
          prev_i=i;
        }
      }
      assert(k+1==*pn_uniq_seq);
    } else { fprintf(stderr, "Cannot allocate memory for unique_seq_list\n"); return(0); }

    return(uniq_seq_list);
}


char get_region_split_character(pairix_t *t)
{
    return(t->idx->conf.region_split_character);
}
