'''
Copyright (C)
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with this program. If not, see https://tldrlegal.com/license/gnu-general-public-license-v3-(gpl-3)#fulltext. 

For license issues, please contact:

Dr. Bing Ye
Life Sciences Institute
University of Michigan
210 Washtenaw Avenue, Room 5403
Ann Arbor, MI 48109-2216
USA

Email: bingye@umich.edu
'''




from .tools import *
import os
import gc
import cv2
import datetime
import numpy as np
import math
from scipy.spatial import distance
from collections import deque
import tensorflow as tf
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.image import img_to_array
import pandas as pd
import seaborn as sb
import functools
import operator



class AnalyzeAnimal():

	def __init__(self):

		self.path_to_video=None
		self.basename=None
		self.fps=None
		self.framewidth=None
		self.frameheight=None
		self.kernel=3
		self.results_path=None
		self.dim_tconv=8
		self.dim_conv=8
		self.channel=1
		self.include_bodyparts=False
		self.std=0
		self.categorize_behavior=False
		self.animation_analyzer=True
		self.delta=None
		self.animal_number=None
		self.autofind_t=False
		self.t=0
		self.duration=5
		self.length=None
		self.animal_area=None
		self.animal_vs_bg=None
		self.background=None
		self.background_low=None
		self.background_high=None
		self.skipped_frames=[]
		self.all_time=[]
		self.total_analysis_framecount=None
		self.to_deregister={}
		self.count_to_deregister=None
		self.register_counts={}
		self.animal_contours={}
		self.animal_centers={}
		self.animal_existingcenters={}
		self.animal_heights={}
		self.animal_inners={}
		self.animal_blobs={}
		self.animations={}
		self.pattern_images={}
		self.event_probability={}
		self.all_behavior_parameters={}
		self.log=[]
		

	def prepare_analysis(self,
		path_to_video, # path to the video for generating behavior examples or behavior analysis
		results_path, # the folder that stores the generated behavior examples or analysis results
		animal_number, # the number of animals / objects in a video
		delta=1.2, # an estimated fold change (1.2) of the light intensity for optogenetics only
		names_and_colors=None, # the behavior names and their representing colors
		framewidth=None, # the width of the frame to resize
		stable_illumination=True, # whether the illumination in videos is stable
		dim_tconv=8, # input dim of Animation Analyzer
		dim_conv=8, # input dim of Pattern Recognizer
		channel=1, # color of Animation Analyzer
		include_bodyparts=False, # whether to include body parts in pattern images
		std=0, # for excluding static pixels in pattern images
		categorize_behavior=True, # whether to categorize behaviors or just track animals without behavioral categorization
		animation_analyzer=True, # whether to include Animation Analyzer
		path_background=None, # if not None, load backgrounds (need to put the background images under the name 'background.jpg', 'background_low.jpg' and 'background_high.jpg' in the 'path_background' path)
		autofind_t=False, # whether to use the time point of the first time of illumination change as start_t
		t=0, # if autofind_t is False, t is the start_t
		duration=5, # the duration for example generation / analysis
		ex_start=0, # the start time point for background extraction
		ex_end=None, # the end time point for background extraction, if None, use the entire video
		length=15, # the duration (number of frames) of a behavior example (a behavior episode)
		animal_vs_bg=0 # 0: animals brighter than the background; 1: animals darker than the background; 2: hard to tell
		):
		
		print('Preparation started...')
		self.log.append('Preparation started...')
		print(datetime.datetime.now())
		self.log.append(str(datetime.datetime.now()))

		self.path_to_video=path_to_video
		self.basename=os.path.basename(self.path_to_video)
		self.framewidth=framewidth
		self.results_path=os.path.join(results_path,os.path.splitext(self.basename)[0])
		self.animal_number=animal_number
		self.dim_tconv=dim_tconv
		self.dim_conv=dim_conv
		self.channel=channel
		self.include_bodyparts=include_bodyparts
		self.std=std
		self.categorize_behavior=categorize_behavior
		self.animation_analyzer=animation_analyzer
		self.autofind_t=autofind_t
		self.t=t
		self.duration=duration
		self.length=length
		os.makedirs(self.results_path,exist_ok=True)
		capture=cv2.VideoCapture(self.path_to_video)
		self.fps=round(capture.get(cv2.CAP_PROP_FPS))
		self.count_to_deregister=self.fps*2

		if self.duration<=0:
			self.total_analysis_framecount=int(capture.get(cv2.CAP_PROP_FRAME_COUNT))+1
		else:
			self.total_analysis_framecount=int(self.fps*self.duration)+1

		while True:
			retval,frame=capture.read()
			break
		capture.release()

		print('Video fps: '+str(self.fps))
		self.log.append('Video fps: '+str(self.fps))
		print('The original video framesize: '+str(int(frame.shape[0]))+' X '+str(int(frame.shape[1])))
		self.log.append('The original video framesize: '+str(int(frame.shape[0]))+' X '+str(int(frame.shape[1])))

		if self.framewidth is not None:
			self.frameheight=int(frame.shape[0]*self.framewidth/frame.shape[1])
			self.background=cv2.resize(frame,(self.framewidth,self.frameheight),interpolation=cv2.INTER_AREA)
			print('The resized video framesize: '+str(self.frameheight)+' X '+str(self.framewidth))
			self.log.append('The resized video framesize: '+str(self.frameheight)+' X '+str(self.framewidth))
		else:
			self.background=frame
		framesize=min(self.background.shape[0],self.background.shape[1])

		if framesize/self.animal_number<250:
			self.kernel=3
		elif framesize/self.animal_number<500:
			self.kernel=5
		elif framesize/self.animal_number<1000:
			self.kernel=7
		elif framesize/self.animal_number<1500:
			self.kernel=9
		else:
			self.kernel=11

		self.delta=delta
		self.autofind_t=autofind_t
		self.animal_vs_bg=animal_vs_bg
		if self.autofind_t:
			es_start=None
		else:
			es_start=self.t
		constants=estimate_constants(self.path_to_video,self.delta,self.animal_number,framewidth=self.framewidth,frameheight=self.frameheight,stable_illumination=stable_illumination,ex_start=ex_start,ex_end=ex_end,t=es_start,duration=self.duration,animal_vs_bg=self.animal_vs_bg,path_background=path_background,kernel=self.kernel)
		self.animal_area=constants[4]
		self.log.append('The area of single animal is: '+str(self.animal_area)+'.')
		self.background=constants[0]
		self.background_low=constants[1]
		self.background_high=constants[2]
		cv2.imwrite(os.path.join(self.results_path,'background.jpg'),constants[0])
		cv2.imwrite(os.path.join(self.results_path,'background_low.jpg'),constants[1])
		cv2.imwrite(os.path.join(self.results_path,'background_high.jpg'),constants[2])
		if self.autofind_t:
			self.t=constants[3]

		if self.categorize_behavior:
			for behavior_name in names_and_colors:
				self.all_behavior_parameters[behavior_name]={}
				self.all_behavior_parameters[behavior_name]['color']=names_and_colors[behavior_name]
				for parameter_name in ['acceleration','count','distance','duration','intensity_area','intensity_length','latency','magnitude_area','magnitude_length','probability','speed','velocity','vigor_area','vigor_length']:
					self.all_behavior_parameters[behavior_name][parameter_name]={}
		else:
			self.dim_conv=8
			self.animation_analyzer=False
			for parameter_name in ['acceleration','distance','intensity_area','intensity_length','magnitude_area','magnitude_length','speed','velocity','vigor_area','vigor_length']:
				self.all_behavior_parameters[parameter_name]={}

		for i in range(self.animal_number):
			self.to_deregister[i]=0
			self.register_counts[i]=None
			self.animal_contours[i]=[None]*self.total_analysis_framecount
			self.animal_centers[i]=[None]*self.total_analysis_framecount
			self.animal_existingcenters[i]=(-10000,-10000)
			self.animal_heights[i]=[None]*self.total_analysis_framecount
			if self.include_bodyparts:
				self.animal_inners[i]=deque(maxlen=self.length)
			if self.animation_analyzer:
				self.animal_blobs[i]=deque([np.zeros((self.dim_tconv,self.dim_tconv,self.channel),dtype='uint8')],maxlen=self.length)*self.length
				self.animations[i]=[np.zeros((self.length,self.dim_tconv,self.dim_tconv,self.channel),dtype='uint8')]*self.total_analysis_framecount
			self.pattern_images[i]=[np.zeros((self.dim_conv,self.dim_conv,3),dtype='uint8')]*self.total_analysis_framecount

		print('Preparation completed!')
		self.log.append('Preparation completed!')


	def track_animal(self,frame_count_analyze,contours,centers,heights,inners=None):

		# frame_count_analyze: the analyzed frame count
		# contours: the contours of detected animals
		# centers: the centers of detected animals
		# heights: the heights of detected animals
		# inners: the inner contours of detected animals when body parts are included in pattern images

		unused_existing_indices=list(self.animal_existingcenters)
		existing_centers=list(self.animal_existingcenters.values())
		unused_new_indices=list(range(len(centers)))
		dt_flattened=distance.cdist(existing_centers,centers).flatten()
		dt_sort_index=dt_flattened.argsort()
		length=len(centers)

		for idx in dt_sort_index:
			index_in_existing=int(idx/length)
			index_in_new=int(idx%length)
			if index_in_existing in unused_existing_indices:
				if index_in_new in unused_new_indices:
					unused_existing_indices.remove(index_in_existing)
					unused_new_indices.remove(index_in_new)
					if self.register_counts[index_in_existing] is None:
						self.register_counts[index_in_existing]=frame_count_analyze
					self.to_deregister[index_in_existing]=0
					self.animal_contours[index_in_existing][frame_count_analyze]=contours[index_in_new]
					center=centers[index_in_new]
					self.animal_centers[index_in_existing][frame_count_analyze]=center
					self.animal_existingcenters[index_in_existing]=center
					self.animal_heights[index_in_existing][frame_count_analyze]=heights[index_in_new]
					if self.include_bodyparts:
						self.animal_inners[index_in_existing].append(inners[index_in_new])
						pattern_image=generate_patternimage(self.background,self.animal_contours[index_in_existing][max(0,(frame_count_analyze-self.length+1)):frame_count_analyze+1],inners=self.animal_inners[index_in_existing],std=self.std)
					else:
						pattern_image=generate_patternimage(self.background,self.animal_contours[index_in_existing][max(0,(frame_count_analyze-self.length+1)):frame_count_analyze+1],inners=None,std=0)
					pattern_image=cv2.resize(pattern_image,(self.dim_conv,self.dim_conv),interpolation=cv2.INTER_AREA)
					self.pattern_images[index_in_existing][frame_count_analyze]=np.array(pattern_image)

		if len(unused_existing_indices)>0:
			for i in unused_existing_indices:
				if self.to_deregister[i]<=self.count_to_deregister:
					self.to_deregister[i]+=1		
				else:
					self.animal_existingcenters[i]=(-10000,-10000)
				if self.include_bodyparts:
					self.animal_inners[i].append(None)


	def acquire_information(self,background_free=True,black_background=True):

		# background_free: whether to include background in animations
		# black_background: whether to set background black

		print('Acquiring information in each frame...')
		self.log.append('Acquiring information in each frame...')
		print(datetime.datetime.now())
		self.log.append(str(datetime.datetime.now()))

		capture=cv2.VideoCapture(self.path_to_video)

		background=self.background
		background_low=self.background_low
		background_high=self.background_high
		if self.animal_vs_bg==1:
			background=np.uint8(255-background)
			background_low=np.uint8(255-background_low)
			background_high=np.uint8(255-background_high)

		frame_count=frame_count_analyze=0
		temp_frames=deque(maxlen=self.length)
		animation=deque([np.zeros((self.dim_tconv,self.dim_tconv,self.channel),dtype='uint8')],maxlen=self.length)*self.length

		start_t=round((self.t-self.length/self.fps),2)
		if start_t<0:
			start_t=0.00
		if self.duration==0:
			end_t=float('inf')
		else:
			end_t=start_t+self.duration

		while True:

			retval,frame=capture.read()
			time=round((frame_count+1)/self.fps,2)

			if time>=end_t or frame is None:
				break

			if time>=start_t:

				self.all_time.append(round((time-start_t),2))
				
				if (frame_count_analyze+1)%1000==0:
					print(str(frame_count_analyze+1)+' frames processed...')
					self.log.append(str(frame_count_analyze+1)+' frames processed...')
					print(datetime.datetime.now())
					self.log.append(str(datetime.datetime.now()))

				if self.framewidth is not None:
					frame=cv2.resize(frame,(self.framewidth,self.frameheight),interpolation=cv2.INTER_AREA)

				temp_frames.append(frame)

				(contours,centers,heights,inners)=contour_frame(frame,self.animal_number,background,background_low,background_high,self.delta,self.animal_area,animal_vs_bg=self.animal_vs_bg,include_bodyparts=self.include_bodyparts,animation_analyzer=self.animation_analyzer,channel=self.channel,kernel=self.kernel,black_background=black_background)

				if len(contours)==0:

					self.skipped_frames.append(frame_count_analyze)

					for i in self.animal_centers:
						if self.include_bodyparts:
							self.animal_inners[i].append(None)

				else:

					self.track_animal(frame_count_analyze,contours,centers,heights,inners=inners)

					if self.animation_analyzer:
						for i in self.animal_centers:
							for n,f in enumerate(temp_frames):
								contour=self.animal_contours[i][max(0,(frame_count_analyze-self.length+1)):frame_count_analyze+1][n]
								if contour is None:
									blob=np.zeros((self.dim_tconv,self.dim_tconv,self.channel),dtype='uint8')
								else:
									blob=extract_blob_background(f,self.animal_contours[i][max(0,(frame_count_analyze-self.length+1)):frame_count_analyze+1],contour=contour,channel=self.channel,background_free=background_free,black_background=black_background)
									blob=cv2.resize(blob,(self.dim_tconv,self.dim_tconv),interpolation=cv2.INTER_AREA)
								animation.append(img_to_array(blob))
							self.animations[i][frame_count_analyze]=np.array(animation)

				frame_count_analyze+=1

			frame_count+=1

		capture.release()

		print('Information acquisition completed!')
		self.log.append('Information acquisition completed!')


	def acquire_information_interact_basic(self,background_free=True,black_background=True):

		# background_free: whether to include background in animations
		# black_background: whether to set background black

		print('Acquiring information in each frame...')
		self.log.append('Acquiring information in each frame...')
		print(datetime.datetime.now())
		self.log.append(str(datetime.datetime.now()))

		self.register_counts={}
		self.register_counts[0]=None
		if self.animation_analyzer:
			self.animations={}
			self.animations[0]=[np.zeros((self.length,self.dim_tconv,self.dim_tconv,self.channel),dtype='uint8')]*self.total_analysis_framecount
		self.pattern_images={}
		self.pattern_images[0]=[np.zeros((self.dim_conv,self.dim_conv,3),dtype='uint8')]*self.total_analysis_framecount
		self.animal_contours={}
		self.animal_contours[0]=[None]*self.total_analysis_framecount
		self.animal_centers={}
		self.animal_centers[0]=[None]*self.total_analysis_framecount

		background=self.background
		background_low=self.background_low
		background_high=self.background_high
		if self.animal_vs_bg==1:
			background=np.uint8(255-background)
			background_low=np.uint8(255-background_low)
			background_high=np.uint8(255-background_high)

		capture=cv2.VideoCapture(self.path_to_video)
		frame_count=frame_count_analyze=0
		temp_frames=deque(maxlen=self.length)
		temp_contours=deque(maxlen=self.length)
		temp_inners=deque(maxlen=self.length)
		animation=deque([np.zeros((self.dim_tconv,self.dim_tconv,self.channel),dtype='uint8')],maxlen=self.length)*self.length

		start_t=round((self.t-self.length/self.fps),2)
		if start_t<0:
			start_t=0.00
		if self.duration==0:
			end_t=float('inf')
		else:
			end_t=start_t+self.duration

		while True:

			retval,frame=capture.read()
			time=round((frame_count+1)/self.fps,2)

			if time>=end_t or frame is None:
				break

			if time>=start_t:

				self.all_time.append(round((time-start_t),2))
				
				if (frame_count_analyze+1)%1000==0:
					print(str(frame_count_analyze+1)+' frames processed...')
					self.log.append(str(frame_count_analyze+1)+' frames processed...')
					print(datetime.datetime.now())
					self.log.append(str(datetime.datetime.now()))

				if self.framewidth is not None:
					frame=cv2.resize(frame,(self.framewidth,self.frameheight),interpolation=cv2.INTER_AREA)

				temp_frames.append(frame)

				(contours,centers,heights,inners)=contour_frame(frame,self.animal_number,background,background_low,background_high,self.delta,self.animal_area,animal_vs_bg=self.animal_vs_bg,include_bodyparts=self.include_bodyparts,animation_analyzer=False,channel=self.channel,kernel=self.kernel,black_background=black_background)

				if len(contours)==0:

					self.skipped_frames.append(frame_count_analyze)

					temp_contours.append(None)
					temp_inners.append(None)
					animation.append(np.zeros((self.dim_tconv,self.dim_tconv,self.channel),dtype='uint8'))

				else:

					temp_contours.append(contours)
					temp_inners.append(inners)

					if self.register_counts[0] is None:
						self.register_counts[0]=frame_count_analyze
					self.animal_contours[0][frame_count_analyze]=contours

					(y_bt,y_tp,x_lf,x_rt)=crop_frame(frame,functools.reduce(operator.iconcat,[ct for ct in temp_contours if ct is not None],[]))
					self.animal_centers[0][frame_count_analyze]=(x_lf+5,y_bt+10)

					if self.include_bodyparts:
						pattern_image=generate_patternimage_all(frame,y_bt,y_tp,x_lf,x_rt,[ct for ct in temp_contours if ct is not None],[inr for inr in temp_inners if inr is not None],std=self.std)
					else:
						pattern_image=generate_patternimage_all(frame,y_bt,y_tp,x_lf,x_rt,[ct for ct in temp_contours if ct is not None],None,std=0)
					self.pattern_images[0][frame_count_analyze]=np.array(cv2.resize(pattern_image,(self.dim_conv,self.dim_conv),interpolation=cv2.INTER_AREA))

					if self.animation_analyzer:
						for i,f in enumerate(temp_frames):
							if self.animal_contours[0][max(0,(frame_count_analyze-self.length+1)):frame_count_analyze+1][i] is None:
								blob=np.zeros((self.dim_tconv,self.dim_tconv,self.channel),dtype='uint8')
							else:
								blob=extract_blob_all(f,y_bt,y_tp,x_lf,x_rt,contours=temp_contours[i],channel=self.channel,background_free=background_free,black_background=black_background)
							blob=cv2.resize(blob,(self.dim_tconv,self.dim_tconv),interpolation=cv2.INTER_AREA)
							animation.append(img_to_array(blob))
							self.animations[0][frame_count_analyze]=np.array(animation)

				frame_count_analyze+=1

			frame_count+=1

		capture.release()

		self.animations[0]=self.animations[0][:len(self.all_time)]
		self.pattern_images[0]=self.pattern_images[0][:len(self.all_time)]
		self.animal_contours[0]=self.animal_contours[0][:len(self.all_time)]
		self.animal_centers[0]=self.animal_centers[0][:len(self.all_time)]

		print('Information acquisition completed!')
		self.log.append('Information acquisition completed!')


	def craft_data(self):

		print('Crafting data...')
		self.log.append('Crafting data...')
		print(datetime.datetime.now())
		self.log.append(str(datetime.datetime.now()))

		lengths=[]
		length=len(self.all_time)
		to_delete=[]
		IDs=list(self.animal_centers.keys())

		for i in IDs:
			check=0
			for x in self.animal_heights[i]:
				if x is not None:
					check+=1
			lengths.append(check)
			if self.register_counts[i] is None:
				to_delete.append(i)

		if len(IDs)==len(to_delete):
			to_delete.remove(np.argsort(lengths)[-1])

		for i in IDs:
			if i in to_delete:
				del self.to_deregister[i]
				del self.register_counts[i]
				del self.animal_centers[i]
				del self.animal_existingcenters[i]
				del self.animal_contours[i]
				del self.animal_heights[i]
				if self.include_bodyparts:
					del self.animal_inners[i]
				if self.animation_analyzer:
					del self.animal_blobs[i]
					del self.animations[i]
				del self.pattern_images[i]

		for i in self.animal_centers:
			self.animal_centers[i]=self.animal_centers[i][:length]
			self.animal_contours[i]=self.animal_contours[i][:length]
			self.animal_heights[i]=self.animal_heights[i][:length]
			if self.animation_analyzer:
				self.animations[i]=self.animations[i][:length]
			self.pattern_images[i]=self.pattern_images[i][:length]

		print('Data crafting completed!')
		self.log.append('Data crafting completed!')


	def categorize_behaviors(self,path_to_categorizer,uncertain=0,min_length=None):

		# path_to_categorizer: path to the Categorizer
		# uncertain: a threshold between the highest the 2nd highest probablity of behaviors to determine if output an 'NA' in behavior classification
		# min_length: the minimum length (in frames) a behavior should last, can be used to filter out the brief false positives

		print('Categorizing behaviors...')
		self.log.append('Categorizing behaviors...')
		print(datetime.datetime.now())
		self.log.append(str(datetime.datetime.now()))

		IDs=list(self.pattern_images.keys())

		if self.animation_analyzer:
			animations=self.animations[IDs[0]]
		pattern_images=self.pattern_images[IDs[0]]

		if len(self.pattern_images)>1:
			for n in IDs[1:]:
				if self.animation_analyzer:
					animations+=self.animations[n]
				pattern_images+=self.pattern_images[n]

		del self.animations
		del self.pattern_images
		gc.collect()

		with tf.device('CPU'):
			if self.animation_analyzer:
				animations=tf.convert_to_tensor(np.array(animations,dtype='float32')/255.0)
			pattern_images=tf.convert_to_tensor(np.array(pattern_images,dtype='float32')/255.0)

		if self.animation_analyzer:
			inputs=[animations,pattern_images]
		else:
			inputs=pattern_images

		categorizer=load_model(path_to_categorizer)
		predictions=categorizer.predict(inputs,batch_size=32)

		for behavior_name in self.all_behavior_parameters:
			for i in IDs:
				self.all_behavior_parameters[behavior_name]['probability'][i]=[np.nan]*len(self.all_time)
				self.event_probability[i]=[['NA',-1]]*len(self.all_time)

		idx=0
		for n in IDs:
			i=self.length+self.register_counts[n]
			idx+=i
			while i<len(self.animal_contours[n]):
				if self.animal_contours[n][i] is not None:
					check=0
					for c in self.animal_contours[n][i-self.length+1:i+1]:
						if c is None:
							check+=1
					if check<=self.length/2:
						prediction=predictions[idx]
						behavior_names=list(self.all_behavior_parameters.keys())
						for behavior_name in behavior_names:
							if len(behavior_names)==2:
								if behavior_names.index(behavior_name)==0:
									probability=1-prediction[0]
								else:
									probability=prediction[0]
							else:
								probability=prediction[behavior_names.index(behavior_name)]
							self.all_behavior_parameters[behavior_name]['probability'][n][i]=probability
						if len(behavior_names)==2:
							if prediction[0]>0.5:
								if prediction[0]-(1-prediction[0])>uncertain:
									self.event_probability[n][i]=[behavior_names[1],prediction[0]]
							if prediction[0]<0.5:
								if (1-prediction[0])-prediction[0]>uncertain:
									self.event_probability[n][i]=[behavior_names[0],1-prediction[0]]		
						else:
							if sorted(prediction)[-1]-sorted(prediction)[-2]>uncertain:
								self.event_probability[n][i]=[behavior_names[np.argmax(prediction)],max(prediction)]
				idx+=1
				i+=1

		del predictions
		gc.collect()

		if min_length is not None:
			for n in IDs:
				i=self.length+self.register_counts[n]
				continued_length=1
				while i<len(self.event_probability[n]):
					if self.event_probability[n][i][0]==self.event_probability[n][i-1][0]:
						continued_length+=1
					else:
						if continued_length<min_length:
							self.event_probability[n][i-continued_length:i]=[['NA',-1]]*continued_length
						continued_length=1
					i+=1

		print('Behavioral categorization completed!')
		self.log.append('Behavioral categorization completed!')


	def annotate_video(self,behavior_to_include,show_legend=True,interact_all=False):

		# behavior_to_include: behaviors that are included in the annotation
		# show_legend: whether to show the legend of behavior names in video frames
		# interact_all: whether is the interactive basic mode

		print('Annotating video...')
		self.log.append('Annotating video...')
		print(datetime.datetime.now())
		self.log.append(str(datetime.datetime.now()))

		text_scl=max(0.5,round((self.background.shape[0]+self.background.shape[1])/1080,1))
		text_tk=max(1,round((self.background.shape[0]+self.background.shape[1])/540))

		if self.categorize_behavior:
			colors={}
			for behavior_name in self.all_behavior_parameters:
				if self.all_behavior_parameters[behavior_name]['color'][1][0]!='#':
					colors[behavior_name]=(255,255,255)
				else:
					hex_color=self.all_behavior_parameters[behavior_name]['color'][1].lstrip('#')
					color=tuple(int(hex_color[i:i+2],16) for i in (0,2,4))
					colors[behavior_name]=color[::-1]
			
			if len(behavior_to_include)!=len(self.all_behavior_parameters):
				for behavior_name in self.all_behavior_parameters:
					if behavior_name not in behavior_to_include:
						del colors[behavior_name]
			
			if show_legend:	
				scl=self.background.shape[0]/1024
				if 25*(len(colors)+1)<self.background.shape[0]:
					intvl=25
				else:
					intvl=int(self.background.shape[0]/(len(colors)+1))

		capture=cv2.VideoCapture(self.path_to_video)
		writer=None
		frame_count=frame_count_analyze=index=0

		total_animal_number=0
		df=pd.DataFrame(self.animal_centers,index=self.all_time)
		df.to_excel(os.path.join(self.results_path,'all_centers.xlsx'),index_label='time/ID')
		for i in self.animal_centers:
			total_animal_number+=1
		if total_animal_number<=0:
			total_animal_number=1
		color_diff=int(510/total_animal_number)

		start_t=round((self.t-self.length/self.fps),2)
		if start_t<0:
			start_t=0.00
		if self.duration==0:
			end_t=float('inf')
		else:
			end_t=start_t+self.duration

		while True:
			retval,frame=capture.read()
			time=round((frame_count+1)/self.fps,2)

			if time>=end_t or frame is None:
				break

			if time>=start_t:

				if self.framewidth is not None:
					frame=cv2.resize(frame,(self.framewidth,self.frameheight),interpolation=cv2.INTER_AREA)

				if self.categorize_behavior:
					if show_legend:
						n=1
						for i in colors:
							cv2.putText(frame,i,(10,intvl*n),cv2.FONT_HERSHEY_SIMPLEX,scl,colors[i],text_tk)
							n+=1

				current_animal_number=0

				if frame_count_analyze not in self.skipped_frames:

					for i in self.animal_contours:

						if frame_count_analyze<len(self.animal_contours[i]):

							if self.animal_contours[i][frame_count_analyze] is not None:

								cx=self.animal_centers[i][frame_count_analyze][0]
								cy=self.animal_centers[i][frame_count_analyze][1]

								if self.animal_centers[i][max(frame_count_analyze-1,0)] is not None:
									cxp=self.animal_centers[i][max(frame_count_analyze-1,0)][0]
									cyp=self.animal_centers[i][max(frame_count_analyze-1,0)][1]
									cv2.line(self.background,(cx,cy),(cxp,cyp),(abs(int(color_diff*(total_animal_number-current_animal_number)-255)),int(color_diff*current_animal_number/2),int(color_diff*(total_animal_number-current_animal_number)/2)),int(text_tk))
								else:
									cv2.circle(self.background,(cx,cy),int(text_tk),(abs(int(color_diff*(total_animal_number-current_animal_number)-255)),int(color_diff*current_animal_number/2),int(color_diff*(total_animal_number-current_animal_number)/2)),-1)

								if interact_all is False:
									cv2.putText(frame,str(i),(cx-10,cy-10),cv2.FONT_HERSHEY_SIMPLEX,text_scl,(255,255,255),text_tk)
									cv2.circle(frame,(cx,cy),int(text_tk*3),(255,0,0),-1)

								if self.categorize_behavior:
									if self.event_probability[i][frame_count_analyze][0]=='NA':
										if interact_all:
											cv2.drawContours(frame,self.animal_contours[i][frame_count_analyze],-1,(255,255,255),1)
										else:
											cv2.drawContours(frame,[self.animal_contours[i][frame_count_analyze]],0,(255,255,255),1)
										cv2.putText(frame,'NA',(cx+10,cy-10),cv2.FONT_HERSHEY_SIMPLEX,text_scl,(255,255,255),text_tk)
									else:
										name=self.event_probability[i][frame_count_analyze][0]
										probability=str(round(self.event_probability[i][frame_count_analyze][1]*100))+'%'
										if name in colors:
											color=colors[self.event_probability[i][frame_count_analyze][0]]
											if interact_all:
												cv2.drawContours(frame,self.animal_contours[i][frame_count_analyze],-1,color,1)
											else:
												cv2.drawContours(frame,[self.animal_contours[i][frame_count_analyze]],0,color,1)
											cv2.putText(frame,name+' '+probability,(cx+10,cy-10),cv2.FONT_HERSHEY_SIMPLEX,text_scl,color,text_tk)
										else:
											if interact_all:
												cv2.drawContours(frame,self.animal_contours[i][frame_count_analyze],-1,(255,255,255),1)
											else:
												cv2.drawContours(frame,[self.animal_contours[i][frame_count_analyze]],0,(255,255,255),1)
											cv2.putText(frame,'NA',(cx+10,cy-10),cv2.FONT_HERSHEY_SIMPLEX,text_scl,(255,255,255),text_tk)
								else:
									cv2.drawContours(frame,[self.animal_contours[i][frame_count_analyze]],0,(255,255,255),1)

						current_animal_number+=1

					index+=1

				if writer is None:
					(h,w)=frame.shape[:2]
					writer=cv2.VideoWriter(os.path.join(self.results_path,'Annotated video.avi'),cv2.VideoWriter_fourcc(*'MJPG'),self.fps,(w,h),True)

				writer.write(frame)

				frame_count_analyze+=1

			frame_count+=1

		capture.release()
		writer.release()

		cv2.imwrite(os.path.join(self.results_path,'Trajectory.jpg'),self.background)

		print('Video annotation completed!')
		self.log.append('Video annotation completed!')


	def analyze_parameters(self,normalize_distance=True,parameter_to_analyze=[]):

		# normalize_distance: whether to normalize the distance (in pixel) to the animal contour area
		# parameter_to_analyze: the behavior parameters that are selected in the analysis

		all_parameters=[]
		if '3 length parameters' in parameter_to_analyze:
			all_parameters+=['intensity_length','magnitude_length','vigor_length']
		if '3 areal parameters' in parameter_to_analyze:
			all_parameters+=['intensity_area','magnitude_area','vigor_area']
		if '4 locomotion parameters' in parameter_to_analyze:
			all_parameters+=['acceleration','speed','velocity']

		if self.categorize_behavior:

			for behavior_name in self.all_behavior_parameters:

				for i in self.event_probability:
					if 'count' in parameter_to_analyze:
						self.all_behavior_parameters[behavior_name]['count'][i]=0
					if 'duration' in parameter_to_analyze:
						self.all_behavior_parameters[behavior_name]['duration'][i]=0.0
					if '4 locomotion parameters' in parameter_to_analyze:
						self.all_behavior_parameters[behavior_name]['distance'][i]=0.0
					if 'latency' in parameter_to_analyze:
						self.all_behavior_parameters[behavior_name]['latency'][i]='NA'
			
				for parameter_name in all_parameters:
					for i in self.event_probability:
						self.all_behavior_parameters[behavior_name][parameter_name][i]=[np.nan]*len(self.all_time)

		else:

			for i in self.animal_contours:
				if '4 locomotion parameters' in parameter_to_analyze:
					self.all_behavior_parameters['distance'][i]=0.0
				for parameter_name in all_parameters:
					self.all_behavior_parameters[parameter_name][i]=[np.nan]*len(self.all_time)
				
		if len(parameter_to_analyze)>0:

			for i in self.animal_contours:

				n=self.length+self.register_counts[i]

				while n<len(self.animal_contours[i]):

					if self.categorize_behavior:

						behavior_name=self.event_probability[i][n][0]

						if behavior_name!='NA':

							if 'count' in parameter_to_analyze:
								if behavior_name!=self.event_probability[i][n-1][0]:
									self.all_behavior_parameters[behavior_name]['count'][i]+=1
									
							if 'duration' in parameter_to_analyze:
								self.all_behavior_parameters[behavior_name]['duration'][i]+=round(1/self.fps,2)

							if 'latency' in parameter_to_analyze:
								if self.all_behavior_parameters[behavior_name]['latency'][i]=='NA':
									self.all_behavior_parameters[behavior_name]['latency'][i]=self.all_time[n]

							if '3 length parameters' in parameter_to_analyze:
								heights_diffs=[]
								for h in self.animal_heights[i][n-self.length+1:n+1]:
									if h is None or self.animal_heights[i][n] is None:
										height_diff=0.0
									else:
										height_diff=abs(h-self.animal_heights[i][n])/h	
									heights_diffs.append(height_diff)
								magnitude_length=max(heights_diffs)
								vigor_length=magnitude_length/((self.length-np.argmax(heights_diffs))/self.fps)
								intensity_length=sum(heights_diffs)/(self.length/self.fps)
								if magnitude_length>0:
									self.all_behavior_parameters[behavior_name]['magnitude_length'][i][n]=magnitude_length
								if vigor_length>0:
									self.all_behavior_parameters[behavior_name]['vigor_length'][i][n]=vigor_length
								if intensity_length>0:
									self.all_behavior_parameters[behavior_name]['intensity_length'][i][n]=intensity_length

							if '4 locomotion parameters' in parameter_to_analyze:
								distance_traveled=0.0
								d=n-self.length
								while d<n-1:
									start_center=self.animal_centers[i][d]
									end_center=self.animal_centers[i][d+1]
									if start_center is None or end_center is None:
										dt=0.0
									else:
										dt=math.dist(end_center,start_center)
									distance_traveled+=dt
									d+=1
								if normalize_distance:
									calibrator=math.sqrt(self.animal_area)
									distance_traveled=distance_traveled/calibrator
								self.all_behavior_parameters[behavior_name]['speed'][i][n]=distance_traveled/(self.length/self.fps)
								end_center=self.animal_centers[i][n]
								if end_center is not None:
									displacements=[]
									for ct in self.animal_centers[i][n-self.length+1:n+1]:
										if ct is None:
											displacements.append(0)
										else:
											displacements.append(math.dist(end_center,ct))		
									displacement=max(displacements)
									if normalize_distance:
										displacement=displacement/calibrator
									velocity=displacement/((self.length-np.argmax(displacements))/self.fps)
									self.all_behavior_parameters[behavior_name]['velocity'][i][n]=velocity
									start_center=self.animal_centers[i][n-1]
									if start_center is not None:
										dt=math.dist(end_center,start_center)
										if normalize_distance:
											dt=dt/calibrator
										self.all_behavior_parameters[behavior_name]['distance'][i]+=dt
								velocities_max=[]
								velocities_min=[]
								for v in self.all_behavior_parameters[behavior_name]['velocity'][i][n-self.length+1:n+1]:
									if v!=np.nan:
										velocities_max.append(v)
										velocities_min.append(v)
									else:
										velocities_max.append(-float('inf'))
										velocities_min.append(float('inf'))
								vmax=max(velocities_max)
								vmin=min(velocities_min)
								if vmax!=-float('inf') and vmin!=float('inf'):
									if np.argmax(velocities_max)!=np.argmin(velocities_min):
										t=abs(np.argmax(velocities_max)-np.argmin(velocities_min))/self.fps
										self.all_behavior_parameters[behavior_name]['acceleration'][i][n]=(vmax-vmin)/t

							if '3 areal parameters' in parameter_to_analyze:
								mask=np.zeros_like(self.background)
								contour=self.animal_contours[i][n]
								if contour is not None:
									cv2.drawContours(mask,[contour],0,(255,255,255),-1)
									mask=cv2.cvtColor(mask,cv2.COLOR_BGR2GRAY)
									area_diffs=[]
									for ct in self.animal_contours[i][n-self.length+1:n+1]:
										if ct is not None:
											prev_mask=np.zeros_like(self.background)
											cv2.drawContours(prev_mask,[ct],0,(255,255,255),-1)
											prev_mask=cv2.cvtColor(prev_mask,cv2.COLOR_BGR2GRAY)
											xlf1,ybt1,w,h=cv2.boundingRect(contour)
											xrt1=xlf1+w
											ytp1=ybt1+h
											xlf2,ybt2,w,h=cv2.boundingRect(ct)
											xrt2=xlf2+w
											ytp2=ybt2+h
											xlf=min(xlf1,xlf2)
											xrt=max(xrt1,xrt2)
											ybt=min(ybt1,ybt2)
											ytp=max(ytp1,ytp2)
											curr_mask=mask[ybt:ytp,xlf:xrt]
											prev_mask=prev_mask[ybt:ytp,xlf:xrt]
											diff=cv2.bitwise_xor(prev_mask,curr_mask)
											#diff[diff!=0]=255
											area_diff=(np.sum(diff)/255)/(np.sum(prev_mask)/255)
											area_diffs.append(area_diff)
									if len(area_diffs)>0:
										magnitude_area=max(area_diffs)
										vigor_area=magnitude_area/((self.length-np.argmax(area_diffs))/self.fps)
										intensity_area=sum(area_diffs)/(self.length/self.fps)
										if magnitude_area>0:
											self.all_behavior_parameters[behavior_name]['magnitude_area'][i][n]=magnitude_area
										if vigor_area>0:
											self.all_behavior_parameters[behavior_name]['vigor_area'][i][n]=vigor_area
										if intensity_area>0:
											self.all_behavior_parameters[behavior_name]['intensity_area'][i][n]=intensity_area

					else:

						if '3 length parameters' in parameter_to_analyze:
							heights_diffs=[]
							for h in self.animal_heights[i][n-self.length+1:n+1]:
								if h is None or self.animal_heights[i][n] is None:
									height_diff=0.0
								else:
									height_diff=abs(h-self.animal_heights[i][n])/h
								heights_diffs.append(height_diff)
							magnitude_length=max(heights_diffs)
							vigor_length=magnitude_length/((self.length-np.argmax(heights_diffs))/self.fps)
							intensity_length=sum(heights_diffs)/(self.length/self.fps)
							if magnitude_length>0:
								self.all_behavior_parameters['magnitude_length'][i][n]=magnitude_length
							if vigor_length>0:
								self.all_behavior_parameters['vigor_length'][i][n]=vigor_length
							if intensity_length>0:
								self.all_behavior_parameters['intensity_length'][i][n]=intensity_length

						if '4 locomotion parameters' in parameter_to_analyze:
							distance_traveled=0.0
							d=n-self.length
							while d<n-1:
								start_center=self.animal_centers[i][d]
								end_center=self.animal_centers[i][d+1]
								if start_center is None or end_center is None:
									dt=0.0
								else:
									dt=math.dist(end_center,start_center)
								distance_traveled+=dt
								d+=1
							if normalize_distance:
								calibrator=math.sqrt(self.animal_area)
								distance_traveled=distance_traveled/calibrator
							self.all_behavior_parameters['speed'][i][n]=distance_traveled/(self.length/self.fps)
							end_center=self.animal_centers[i][n]
							if end_center is not None:
								displacements=[]
								for ct in self.animal_centers[i][n-self.length+1:n+1]:
									if ct is None:
										displacements.append(0)
									else:
										displacements.append(math.dist(end_center,ct))
								displacement=max(displacements)
								if normalize_distance:
									displacement=displacement/calibrator
								velocity=displacement/((self.length-np.argmax(displacements))/self.fps)
								self.all_behavior_parameters['velocity'][i][n]=velocity
								start_center=self.animal_centers[i][n-1]
								if start_center is not None:
									dt=math.dist(end_center,start_center)
									if normalize_distance:
										dt=dt/calibrator
									self.all_behavior_parameters['distance'][i]+=dt

							velocities_max=[]
							velocities_min=[]
							for v in self.all_behavior_parameters['velocity'][i][n-self.length+1:n+1]:
								if v!=np.nan:
									velocities_max.append(v)
									velocities_min.append(v)
								else:
									velocities_max.append(-float('inf'))
									velocities_min.append(float('inf'))
							vmax=max(velocities_max)
							vmin=min(velocities_min)
							if vmax!=-float('inf') and vmin!=float('inf'):
								if np.argmax(velocities_max)!=np.argmin(velocities_min):
									t=abs(np.argmax(velocities_max)-np.argmin(velocities_min))/self.fps
									self.all_behavior_parameters['acceleration'][i][n]=(vmax-vmin)/t

						if '3 areal parameters' in parameter_to_analyze:
							mask=np.zeros_like(self.background)
							contour=self.animal_contours[i][n]
							if contour is not None:
								cv2.drawContours(mask,[contour],0,(255,255,255),-1)
								mask=cv2.cvtColor(mask,cv2.COLOR_BGR2GRAY)
								area_diffs=[]
								for ct in self.animal_contours[i][n-self.length+1:n+1]:
									if ct is not None:
										prev_mask=np.zeros_like(self.background)
										cv2.drawContours(prev_mask,[ct],0,(255,255,255),-1)
										prev_mask=cv2.cvtColor(prev_mask,cv2.COLOR_BGR2GRAY)
										xlf1,ybt1,w,h=cv2.boundingRect(contour)
										xrt1=xlf1+w
										ytp1=ybt1+h
										xlf2,ybt2,w,h=cv2.boundingRect(ct)
										xrt2=xlf2+w
										ytp2=ybt2+h
										xlf=min(xlf1,xlf2)
										xrt=max(xrt1,xrt2)
										ybt=min(ybt1,ybt2)
										ytp=max(ytp1,ytp2)
										curr_mask=mask[ybt:ytp,xlf:xrt]
										prev_mask=prev_mask[ybt:ytp,xlf:xrt]
										diff=cv2.bitwise_xor(prev_mask,curr_mask)
										#diff[diff!=0]=255
										area_diff=(np.sum(diff)/255)/(np.sum(prev_mask)/255)
										area_diffs.append(area_diff)
								if len(area_diffs)>0:
									magnitude_area=max(area_diffs)
									vigor_area=magnitude_area/((self.length-np.argmax(area_diffs))/self.fps)
									intensity_area=sum(area_diffs)/(self.length/self.fps)
									if magnitude_area>0:
										self.all_behavior_parameters['magnitude_area'][i][n]=magnitude_area
									if vigor_area>0:
										self.all_behavior_parameters['vigor_area'][i][n]=vigor_area
									if intensity_area>0:
										self.all_behavior_parameters['intensity_area'][i][n]=intensity_area

					n+=1

				if self.categorize_behavior:
				
					if 'duration' in parameter_to_analyze:
						for behavior_name in self.all_behavior_parameters:
							if self.all_behavior_parameters[behavior_name]['duration'][i]!=0.0:
								self.all_behavior_parameters[behavior_name]['duration'][i]+=round(self.length/self.fps,2)
							else:
								self.all_behavior_parameters[behavior_name]['duration'][i]='NA'

					if '4 locomotion parameters' in parameter_to_analyze:
						for behavior_name in self.all_behavior_parameters:
							if self.all_behavior_parameters[behavior_name]['distance'][i]==0.0:
								self.all_behavior_parameters[behavior_name]['distance'][i]='NA'


	def export_results(self,normalize_distance=True,parameter_to_analyze=[]):

		# normalize_distance: whether to normalize the distance (in pixel) to the animal contour area
		# parameter_to_analyze: the behavior parameters that are selected in the analysis

		print('Quantifying behaviors...')
		self.log.append('Quantifying behaviors...')
		print(datetime.datetime.now())
		self.log.append(str(datetime.datetime.now()))

		self.analyze_parameters(normalize_distance=normalize_distance,parameter_to_analyze=parameter_to_analyze)

		print('Behavioral quantification completed!')
		self.log.append('Behavioral quantification Completed!')

		print('Exporting results...')
		self.log.append('Exporting results...')
		print(datetime.datetime.now())
		self.log.append(str(datetime.datetime.now()))

		if self.categorize_behavior:
			events_df=pd.DataFrame(self.event_probability,index=self.all_time)
			events_df.to_excel(os.path.join(self.results_path,'all_event_probability.xlsx'),float_format='%.2f',index_label='time/ID')

		all_parameters=[]

		if self.categorize_behavior:
			all_parameters.append('probability')
			
		if 'count' in parameter_to_analyze:
			all_parameters.append('count')
		if 'duration' in parameter_to_analyze:
			all_parameters.append('duration')
		if 'latency' in parameter_to_analyze:
			all_parameters.append('latency')
		if '3 length parameters' in parameter_to_analyze:
			all_parameters+=['intensity_length','magnitude_length','vigor_length']
		if '3 areal parameters' in parameter_to_analyze:
			all_parameters+=['intensity_area','magnitude_area','vigor_area']
		if '4 locomotion parameters' in parameter_to_analyze:
			all_parameters+=['acceleration','distance','speed','velocity']

		if self.categorize_behavior:

			for behavior_name in self.all_behavior_parameters:
				os.makedirs(os.path.join(self.results_path,behavior_name),exist_ok=True)

				summary=[]

				for parameter_name in all_parameters:
					if parameter_name in ['count','duration','distance','latency']:
						summary.append(pd.DataFrame.from_dict(self.all_behavior_parameters[behavior_name][parameter_name],orient='index',columns=[parameter_name]).reset_index(drop=True))
					else:
						individual_df=pd.DataFrame.from_dict(self.all_behavior_parameters[behavior_name][parameter_name],orient='index',columns=self.all_time)
						if parameter_name!='probability':
							summary.append(individual_df.mean(axis=1,skipna=True).to_frame().reset_index(drop=True).rename(columns={0:parameter_name+'_mean'}))
							summary.append(individual_df.max(axis=1,skipna=True).to_frame().reset_index(drop=True).rename(columns={0:parameter_name+'_max'}))
							summary.append(individual_df.min(axis=1,skipna=True).to_frame().reset_index(drop=True).rename(columns={0:parameter_name+'_min'}))
						individual_df=pd.DataFrame(self.all_behavior_parameters[behavior_name][parameter_name],index=self.all_time)
						individual_df.to_excel(os.path.join(self.results_path,behavior_name,parameter_name+'.xlsx'),float_format='%.2f',index_label='time/ID')

				if len(summary)>=1:
					pd.concat(summary,axis=1).to_excel(os.path.join(self.results_path,behavior_name,'all_summary.xlsx'),float_format='%.2f',index_label='ID/parameter')

		else:

			summary=[]

			for parameter_name in all_parameters:
				if parameter_name=='distance':
					summary.append(pd.DataFrame.from_dict(self.all_behavior_parameters[parameter_name],orient='index',columns=['distance']).reset_index(drop=True))
				else:
					individual_df=pd.DataFrame.from_dict(self.all_behavior_parameters[parameter_name],orient='index',columns=self.all_time)
					summary.append(individual_df.mean(axis=1,skipna=True).to_frame().reset_index(drop=True).rename(columns={0:parameter_name+'_mean'}))
					summary.append(individual_df.max(axis=1,skipna=True).to_frame().reset_index(drop=True).rename(columns={0:parameter_name+'_max'}))
					summary.append(individual_df.min(axis=1,skipna=True).to_frame().reset_index(drop=True).rename(columns={0:parameter_name+'_min'}))
					individual_df=pd.DataFrame(self.all_behavior_parameters[parameter_name],index=self.all_time)
					individual_df.to_excel(os.path.join(self.results_path,parameter_name+'.xlsx'),float_format='%.2f',index_label='time/ID')

			if len(summary)>=1:
				pd.concat(summary,axis=1).to_excel(os.path.join(self.results_path,'all_summary.xlsx'),float_format='%.2f',index_label='ID/parameter')			

		print('All results exported in: '+str(self.results_path))
		self.log.append('All results exported in: '+str(self.results_path))
		self.log.append('Analysis completed!')
		if len(self.log)>0:
			with open(os.path.join(self.results_path,'Analysis log.txt'),'w') as analysis_log:
				analysis_log.write('\n'.join(str(i) for i in self.log))


	def generate_data(self,background_free=True,black_background=True,skip_redundant=1):

		# background_free: whether to include background in animations
		# black_background: whether to set background black
		# skip_redundant: the interval (in frames) of two consecutively generated behavior example pairs
		
		print('Generating behavior examples...')
		print(datetime.datetime.now())

		background=self.background
		background_low=self.background_low
		background_high=self.background_high
		if self.animal_vs_bg==1:
			background=np.uint8(255-background)
			background_low=np.uint8(255-background_low)
			background_high=np.uint8(255-background_high)

		capture=cv2.VideoCapture(self.path_to_video)
		frame_count=frame_count_analyze=0
		temp_frames=deque(maxlen=self.length)

		for i in range(self.animal_number):
			os.makedirs(os.path.join(self.results_path,'examples',str(i)),exist_ok=True)

		start_t=round((self.t-self.length/self.fps),2)
		if start_t<0:
			start_t=0.00
		if self.duration==0:
			end_t=float('inf')
		else:
			end_t=start_t+self.duration

		while True:

			retval,frame=capture.read()
			time=round((frame_count+1)/self.fps,2)

			if time>=end_t or frame is None:
				break

			if time>=start_t:

				if self.framewidth is not None:
					frame=cv2.resize(frame,(self.framewidth,self.frameheight),interpolation=cv2.INTER_AREA)

				temp_frames.append(frame)

				(contours,centers,heights,inners)=contour_frame(frame,self.animal_number,background,background_low,background_high,self.delta,self.animal_area,animal_vs_bg=self.animal_vs_bg,include_bodyparts=self.include_bodyparts,animation_analyzer=False,kernel=self.kernel,black_background=black_background)

				if len(contours)>0:

					self.track_animal(frame_count_analyze,contours,centers,heights,inners=inners)

					if frame_count_analyze>=self.length and frame_count_analyze%skip_redundant==0:

						for n in self.animal_centers:

							h=w=0
							animation=[]
							for i,f in enumerate(temp_frames):
								contour=self.animal_contours[n][frame_count_analyze-self.length+1:frame_count_analyze+1][i]
								if contour is None:
									blob=np.zeros_like(f)
								else:
									blob=extract_blob_background(f,self.animal_contours[n][frame_count_analyze-self.length+1:frame_count_analyze+1],contour=contour,channel=3,background_free=background_free,black_background=black_background)
									h,w=blob.shape[:2]
								animation.append(blob)

							if h>0:

								if self.include_bodyparts:
									animation_name=os.path.splitext(self.basename)[0]+'_'+str(n)+'_'+str(frame_count_analyze)+'_len'+str(self.length)+'_std'+str(self.std)+'.avi'
									pattern_image_name=os.path.splitext(self.basename)[0]+'_'+str(n)+'_'+str(frame_count_analyze)+'_len'+str(self.length)+'_std'+str(self.std)+'.jpg'
									pattern_image=generate_patternimage(self.background,self.animal_contours[n][frame_count_analyze-self.length+1:frame_count_analyze+1],inners=self.animal_inners[n],std=self.std)
								else:
									animation_name=os.path.splitext(self.basename)[0]+'_'+str(n)+'_'+str(frame_count_analyze)+'_len'+str(self.length)+'.avi'
									pattern_image_name=os.path.splitext(self.basename)[0]+'_'+str(n)+'_'+str(frame_count_analyze)+'_len'+str(self.length)+'.jpg'
									pattern_image=generate_patternimage(self.background,self.animal_contours[n][frame_count_analyze-self.length+1:frame_count_analyze+1],inners=None,std=0)

								path_animation=os.path.join(self.results_path,'examples',str(n),animation_name)
								path_pattern_image=os.path.join(self.results_path,'examples',str(n),pattern_image_name)

								writer=cv2.VideoWriter(path_animation,cv2.VideoWriter_fourcc(*'MJPG'),self.fps/5,(w,h),True)
								for blob in animation:
									writer.write(cv2.resize(blob,(w,h),interpolation=cv2.INTER_AREA))
								writer.release()

								cv2.imwrite(path_pattern_image,pattern_image)

				frame_count_analyze+=1

			frame_count+=1

		capture.release()

		print('Behavior example generation completed!')


	def generate_data_interact_basic(self,background_free=True,black_background=True,skip_redundant=1):

		# background_free: whether to include background in animations
		# black_background: whether to set background black
		# skip_redundant: the interval (in frames) of two consecutively generated behavior example pairs

		print('Generating behavior examples...')
		print(datetime.datetime.now())

		background=self.background
		background_low=self.background_low
		background_high=self.background_high
		if self.animal_vs_bg==1:
			background=np.uint8(255-background)
			background_low=np.uint8(255-background_low)
			background_high=np.uint8(255-background_high)

		capture=cv2.VideoCapture(self.path_to_video)
		frame_count=frame_count_analyze=0
		temp_frames=deque(maxlen=self.length)
		temp_contours=deque(maxlen=self.length)
		temp_inners=deque(maxlen=self.length)
		animation=deque(maxlen=self.length)
		os.makedirs(os.path.join(self.results_path,'examples','0'),exist_ok=True)
		
		start_t=round((self.t-self.length/self.fps),2)
		if start_t<0:
			start_t=0.00
		if self.duration==0:
			end_t=float('inf')
		else:
			end_t=start_t+self.duration

		while True:

			retval,frame=capture.read()
			time=round((frame_count+1)/self.fps,2)

			if time>=end_t or frame is None:
				break

			if time>=start_t:

				if self.framewidth is not None:
					frame=cv2.resize(frame,(self.framewidth,self.frameheight),interpolation=cv2.INTER_AREA)

				temp_frames.append(frame)

				(contours,centers,heights,inners)=contour_frame(frame,self.animal_number,background,background_low,background_high,self.delta,self.animal_area,animal_vs_bg=self.animal_vs_bg,include_bodyparts=self.include_bodyparts,animation_analyzer=False,kernel=self.kernel,black_background=black_background)

				if len(contours)==0:

					temp_contours.append(None)
					temp_inners.append(None)
					animation.append(np.zeros_like(self.background))

				else:

					temp_contours.append(contours)
					temp_inners.append(inners)

					if frame_count_analyze>=self.length and frame_count_analyze%skip_redundant==0:

						(y_bt,y_tp,x_lf,x_rt)=crop_frame(frame,functools.reduce(operator.iconcat,[ct for ct in temp_contours if ct is not None],[]))

						h=w=0
						for i,f in enumerate(temp_frames):
							if temp_contours[i] is None:
								blob=np.zeros_like(f)
							else:
								blob=extract_blob_all(f,y_bt,y_tp,x_lf,x_rt,contours=temp_contours[i],channel=3,background_free=background_free,black_background=black_background)
								h,w=blob.shape[:2]
							animation.append(blob)

						if h>0:

							if self.include_bodyparts:
								animation_name=os.path.splitext(self.basename)[0]+'_'+str(frame_count_analyze)+'_len'+str(self.length)+'_std'+str(self.std)+'_itbs.avi'
								pattern_image_name=os.path.splitext(self.basename)[0]+'_'+str(frame_count_analyze)+'_len'+str(self.length)+'_std'+str(self.std)+'_itbs.jpg'
								pattern_image=generate_patternimage_all(frame,y_bt,y_tp,x_lf,x_rt,[ct for ct in temp_contours if ct is not None],[inr for inr in temp_inners if inr is not None],std=self.std)
							else:
								animation_name=os.path.splitext(self.basename)[0]+'_'+str(frame_count_analyze)+'_len'+str(self.length)+'_itbs.avi'
								pattern_image_name=os.path.splitext(self.basename)[0]+'_'+str(frame_count_analyze)+'_len'+str(self.length)+'_itbs.jpg'
								pattern_image=generate_patternimage_all(frame,y_bt,y_tp,x_lf,x_rt,[ct for ct in temp_contours if ct is not None],None,std=0)

							path_animation=os.path.join(self.results_path,'examples','0',animation_name)
							path_pattern_image=os.path.join(self.results_path,'examples','0',pattern_image_name)
						
							writer=cv2.VideoWriter(path_animation,cv2.VideoWriter_fourcc(*'MJPG'),self.fps/5,(w,h),True)
							for blob in animation:
								writer.write(cv2.resize(blob,(w,h),interpolation=cv2.INTER_AREA))
							writer.release()

							cv2.imwrite(path_pattern_image,pattern_image)

				frame_count_analyze+=1

			frame_count+=1

		capture.release()

		print('Behavior example generation completed!')


