#!/usr/bin/env python
# -*- coding: ascii -*-
from __future__ import print_function

# NOTICE... this file is generated by TkGridGUI.
# Any code or comments added by the user must be in designated areas ONLY.
# User additions will be lost if they are placed in code-generated areas.
# (i.e. Saving from TkGridGUI will over-write code-generated areas.)

# TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "imports"


from __future__ import unicode_literals
from future import standard_library
standard_library.install_aliases()
from builtins import str
from builtins import range
from builtins import object

from tkinter.ttk import Combobox, Progressbar, Separator, Treeview, Notebook

from tkinter import *
from tkinter import Button, Canvas, Checkbutton, Entry, Frame, Label, LabelFrame
from tkinter import Listbox, Message, Radiobutton, Spinbox, Text
from tkinter import OptionMenu
import tkinter.filedialog
from tkinter import _setit as set_command


# >>>>>>insert any user code below this comment for section "imports"
# Place any user import statements here
import sys
import os

from introrl.mdp_data.simple_grid_world import get_env as simple_grid_world_get_env

from introrl.black_box_sims.cliff_walking import CliffWalkingSimulation
from introrl.black_box_sims.blocking_maze import BlockingMaze

from introrl.td_funcs.qlearning_epsilon_greedy import qlearning_epsilon_greedy
from introrl.td_funcs.sarsa_epsilon_greedy import sarsa_epsilon_greedy
from introrl.td_funcs.expected_sarsa_eps_greedy import expected_sarsa_eps_greedy
from introrl.agents.dyna_q import DynaQ

from introrl.dp_funcs.dp_value_iter import dp_value_iteration

from introrl.utils.running_ave import RunningAve
from introrl.agent_supt.learning_tracker import LearnTracker
from introrl.policy import Policy


# TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "top_of_init"
class _tk_grid_world:
    def __init__(self, master):
    
        grid_frame = Frame( master )
        self.grid_frame = grid_frame
        grid_frame.pack(expand=1, fill=BOTH)
        self.master = master
        
        self.x, self.y, self.w, self.h = 10, 10, 778, 228

        self.master.title("tk_grid_world")

        self.make_Button_1( self.grid_frame )          #      Button: Run : at Main(1,1)
        self.make_Button_2( self.grid_frame )          #      Button: Step : at Main(1,3)
        self.make_Button_3( self.grid_frame )          #      Button: Build Model : at Main(1,4)
        self.make_Button_4( self.grid_frame )          #      Button: 1 Episode : at Main(1,2)
        self.make_Combobox_1( self.grid_frame )        #    Combobox: Q-Learn SARSA DynaQ DynaQ+ : at Main(1,6)
        self.make_Combobox_2( self.grid_frame )        #    Combobox: Display Size : at Main(1,5)
        self.make_Entry_1( self.grid_frame )           #       Entry: Step Cost : at Main(3,1)
        self.make_Entry_2( self.grid_frame )           #       Entry: Alpha : at Main(3,2)
        self.make_Entry_3( self.grid_frame )           #       Entry: Gamma : at Main(3,4)
        self.make_Entry_4( self.grid_frame )           #       Entry: Epsilon : at Main(3,5)
        self.make_Entry_5( self.grid_frame )           #       Entry: DynaQ+ Factor : at Main(3,6)
        self.make_Entry_6( self.grid_frame )           #       Entry: Initial Q(s,a) Factor : at Main(3,3)
        self.make_Frame_1( self.grid_frame )           #       Frame:  at Main(4,1)
        self.make_Label_1( self.grid_frame )           #       Label: Step Cost : at Main(2,1)
        self.make_Label_10( self.grid_frame )          #       Label:  at Main(5,5)
        self.make_Label_11( self.grid_frame )          #       Label: Q(s,a) Initial : at Main(2,3)
        self.make_Label_12( self.grid_frame )          #       Label:  at Main(5,6)
        self.make_Label_13( self.grid_frame )          #       Label:   : at Main(1,7)
        self.make_Label_14( self.grid_frame )          #       Label:  at Main(5,7)
        self.make_Label_2( self.grid_frame )           #       Label: Alpha : at Main(2,2)
        self.make_Label_3( self.grid_frame )           #       Label: Gamma : at Main(2,4)
        self.make_Label_4( self.grid_frame )           #       Label: Epsilon : at Main(2,5)
        self.make_Label_5( self.grid_frame )           #       Label: DynaQ+ Factor : at Main(2,6)
        self.make_Label_6( self.grid_frame )           #       Label:  at Main(5,1)
        self.make_Label_7( self.grid_frame )           #       Label:  at Main(5,2)
        self.make_Label_8( self.grid_frame )           #       Label:  at Main(5,3)
        self.make_Label_9( self.grid_frame )           #       Label:  at Main(5,4)
        self.make_Canvas_1( self.Frame_1 )             #      Canvas:  at Frame_1(0,0)

        self.grid_frame.rowconfigure(2, weight=1)
        self.grid_frame.columnconfigure(7, weight=1)


        self.menuBar = Menu(master, relief = "raised", bd=2)

        top_Model = Menu(self.menuBar, tearoff=0)

        top_Model.add("command", label = "Simple Grid World", command=self.menu_Model_Simple_Grid_World, underline=0, accelerator="Ctrl+S")
        top_Model.add("command", label = "Cliff Walking", command=self.menu_Model_Cliff_Walking, underline=0, accelerator="Ctrl+C")
        top_Model.add("command", label = "Blocking Maze", command=self.menu_Model_Blocking_Maze, underline=0, accelerator="Ctrl+B")
        top_Model.add_separator()
        top_Model.add("command", label = "Exit", command=self.menu_Model_Exit, underline=0, accelerator="Ctrl+E")
        self.menuBar.add("cascade", label="Model", menu=top_Model)

        master.config(menu=self.menuBar)




        # use both upper and lower characters for keyboard accelerator options.
        self.master.bind("<Control-S>", lambda event: self.menu_Model_Simple_Grid_World())
        self.master.bind("<Control-s>", lambda event: self.menu_Model_Simple_Grid_World())
        self.master.bind("<Control-C>", lambda event: self.menu_Model_Cliff_Walking())
        self.master.bind("<Control-c>", lambda event: self.menu_Model_Cliff_Walking())
        self.master.bind("<Control-B>", lambda event: self.menu_Model_Blocking_Maze())
        self.master.bind("<Control-b>", lambda event: self.menu_Model_Blocking_Maze())
        self.master.bind("<Control-E>", lambda event: self.menu_Model_Exit())
        self.master.bind("<Control-e>", lambda event: self.menu_Model_Exit())
        # >>>>>>insert any user code below this comment for section "top_of_init"
        
        self.Canvas_1.destroy()
        
        self.Frame_1.config(bg='#666666')
        
        self.env_sim = simple_grid_world_get_env()
        
        self.get_model_settings() # get the initial value for model parameters.
        
        self.grid_canvasD = {} # index=(row,col), value=Canvas object.
        self.s_hash_canvasD = {} # index=s_hash, value=Canvas object.
        
        self.w_total = 1000
        self.h_total = 700
        
        self.build_canvas_objects()
        
    def build_canvas_objects(self):
        """For the env_sim model, build a canvas object for each layout position."""
        
        # destroy any existing Canvas objects
        for _,c in self.grid_canvasD.items():
            c.destroy()
        
        # make canvas objects
        self.grid_canvasD = {}
        self.s_hash_canvasD = {}
        
        self.W = int( self.w_total / self.env_sim.layout.width )
        self.H = int( self.h_total / self.env_sim.layout.height )
        
        # build canvas objects
        for row, rowL in enumerate( self.env_sim.layout.s_hash_rowL ):
            for col, s_hash in enumerate( rowL ):
                self.grid_canvasD[ (row,col) ] = self.make_a_canvas( row, col )
                
                self.s_hash_canvasD[s_hash] = self.grid_canvasD[ (row,col) ]
                
                # put legal action direction arrows
                if self.env_sim.is_legal_state( s_hash ):
                    if not self.env_sim.is_terminal_state( s_hash ):
                        for a_desc in self.env_sim.get_state_legal_action_list( s_hash ):
                            self.show_direction( a_desc, self.grid_canvasD[ (row,col) ] )

        # set background and reward labels
        for row, rowL in enumerate( self.env_sim.layout.s_hash_rowL ):
            for col, s_hash in enumerate( rowL ):
                if self.env_sim.is_legal_state( s_hash ):
                    for a_desc in self.env_sim.get_state_legal_action_list( s_hash ):
                        sn_hash, reward = self.env_sim.get_action_snext_reward( s_hash, a_desc )
                        
                        c = self.s_hash_canvasD[sn_hash]
                        if reward > 0.0:
                            c.config(bg='#aaffaa')
                            self.set_center_number( c, reward, font='Helvetica 16 bold roman')
                        elif reward < 0.0:
                            c.config(bg='#ffaaaa')
                            self.set_center_number( c, reward, font='Helvetica 16 bold roman')
                    
                    if s_hash is None:
                        c = self.s_hash_canvasD[s_hash]
                        c.config(bg='#aaaaaa')
                        self.set_center_text(c, 'Wall' )
                            
                else: # not a legal s_hash
                    c = self.grid_canvasD[ (row,col) ]
                    c.config(bg='#aaaaaa')
                    
                    self.set_center_text(c, s_hash)



        

    def set_center_text(self, c, text_str, color='black', font='Helvetica 16 normal roman'):
        s = str(text_str)
        if s.startswith('"') and s.endswith('"'):
            s = s[1:-1]
        
        c.create_text(self.W/2, self.H/2, text=s, fill=color, font=font)

    def set_center_number(self, c, val, color='black', font='Helvetica 16 normal roman'):        
        c.create_text(self.W/2, self.H/2, text="%g"%val, fill=color, font=font)

    def make_a_canvas(self, row, col ):
        c = Canvas( self.Frame_1 , height="%i"%self.H, width="%i"%self.W)
        c.grid(row=row, column=col)

        c.config(bg='#ffffcc')
        c.bind("<ButtonRelease-1>", self.Canvas_1_Click)
        return c


    def show_direction(self, d, c, amax=None ):
        w = self.W
        h = self.H
        
        if d == amax:
            w6 = int(w / 4)
            h6 = int(h / 4)
        else:
            w6 = int(w / 6)
            h6 = int(h / 6)
        
        if d == 'L':
            pointL = [1,h/2, w6+1,h/2-h6, w6+1,h/2+h6]
        elif d == 'R':
            pointL = [w,h/2, w-w6,h/2-h6, w-w6,h/2+h6]
        elif d == 'U':
            pointL = [w/2,1, w/2-w6,h6+1, w/2+w6,h6+1]
        elif d == 'D':
            pointL = [w/2,h, w/2-w6,h-h6, w/2+w6,h-h6]

        if d == amax:
            color = '#ffaaaa'
        else:
            color = '#d5f6e3'

        c.create_polygon(pointL, outline='#aaaaaa', fill=color, width=1)
        


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Button_1"
    def make_Button_1(self, frame):
        """      Button: Run : at Main(1,1)"""
        self.Button_1 = Button( frame , text="Run", width="15")
        self.Button_1.grid(row=1, column=1)

        # >>>>>>insert any user code below this comment for section "make_Button_1"

        self.Button_1.bind("<ButtonRelease-1>", self.Button_1_Click)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Button_2"
    def make_Button_2(self, frame):
        """      Button: Step : at Main(1,3)"""
        self.Button_2 = Button( frame , text="Step", width="15")
        self.Button_2.grid(row=1, column=3)

        # >>>>>>insert any user code below this comment for section "make_Button_2"

        self.Button_2.bind("<ButtonRelease-1>", self.Button_2_Click)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Button_3"
    def make_Button_3(self, frame):
        """      Button: Build Model : at Main(1,4)"""
        self.Button_3 = Button( frame , text="Build Model", width="15")
        self.Button_3.grid(row=1, column=4)

        # >>>>>>insert any user code below this comment for section "make_Button_3"

        self.Button_3.bind("<ButtonRelease-1>", self.Button_3_Click)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Button_4"
    def make_Button_4(self, frame):
        """      Button: 1 Episode : at Main(1,2)"""
        self.Button_4 = Button( frame , text="1 Episode", width="15")
        self.Button_4.grid(row=1, column=2)

        # >>>>>>insert any user code below this comment for section "make_Button_4"

        self.Button_4.bind("<ButtonRelease-1>", self.Button_4_Click)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Combobox_1"
    def make_Combobox_1(self, frame):
        """    Combobox: Q-Learn SARSA DynaQ DynaQ+ : at Main(1,6)"""
        self.Combobox_1 = Combobox( frame , text="Combobox_1", values="Q-Learn SARSA DynaQ DynaQ+")
        self.Combobox_1.grid(row=1, column=6)
        self.Combobox_1_StringVar = StringVar()

        # >>>>>>insert any user code below this comment for section "make_Combobox_1"

        self.Combobox_1.configure(textvariable=self.Combobox_1_StringVar)
        self.Combobox_1_StringVar.set( "Q-Learn" )
        self.Combobox_1_StringVar_traceName = self.Combobox_1_StringVar.trace_variable("w", self.Combobox_1_StringVar_Callback)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Combobox_2"
    def make_Combobox_2(self, frame):
        """    Combobox: Display Size : at Main(1,5)"""
        self.Combobox_2 = Combobox( frame , text="Display Size", values="1400x800 1200x700 1000x700 800x600")
        self.Combobox_2.grid(row=1, column=5)
        self.Combobox_2_StringVar = StringVar()

        # >>>>>>insert any user code below this comment for section "make_Combobox_2"

        self.Combobox_2.configure(textvariable=self.Combobox_2_StringVar)
        self.Combobox_2_StringVar.set( "1000x700" )
        self.Combobox_2_StringVar_traceName = self.Combobox_2_StringVar.trace_variable("w", self.Combobox_2_StringVar_Callback)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Entry_1"
    def make_Entry_1(self, frame):
        """       Entry: Step Cost : at Main(3,1)"""
        self.Entry_1 = Entry( frame , width="15")
        self.Entry_1.grid(row=3, column=1)
        self.Entry_1_StringVar = StringVar()

        # >>>>>>insert any user code below this comment for section "make_Entry_1"
        self.Entry_1_StringVar.set('0.0')

        self.Entry_1.configure(textvariable=self.Entry_1_StringVar)
        self.Entry_1_StringVar_traceName = self.Entry_1_StringVar.trace_variable("w", self.Entry_1_StringVar_Callback)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Entry_2"
    def make_Entry_2(self, frame):
        """       Entry: Alpha : at Main(3,2)"""
        self.Entry_2 = Entry( frame , width="15")
        self.Entry_2.grid(row=3, column=2)
        self.Entry_2_StringVar = StringVar()

        # >>>>>>insert any user code below this comment for section "make_Entry_2"
        self.Entry_2_StringVar.set('0.1')

        self.Entry_2.configure(textvariable=self.Entry_2_StringVar)
        self.Entry_2_StringVar_traceName = self.Entry_2_StringVar.trace_variable("w", self.Entry_2_StringVar_Callback)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Entry_3"
    def make_Entry_3(self, frame):
        """       Entry: Gamma : at Main(3,4)"""
        self.Entry_3 = Entry( frame , width="15")
        self.Entry_3.grid(row=3, column=4)
        self.Entry_3_StringVar = StringVar()

        # >>>>>>insert any user code below this comment for section "make_Entry_3"
        self.Entry_3_StringVar.set('0.9')

        self.Entry_3.configure(textvariable=self.Entry_3_StringVar)
        self.Entry_3_StringVar_traceName = self.Entry_3_StringVar.trace_variable("w", self.Entry_3_StringVar_Callback)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Entry_4"
    def make_Entry_4(self, frame):
        """       Entry: Epsilon : at Main(3,5)"""
        self.Entry_4 = Entry( frame , width="15")
        self.Entry_4.grid(row=3, column=5)
        self.Entry_4_StringVar = StringVar()

        # >>>>>>insert any user code below this comment for section "make_Entry_4"
        self.Entry_4_StringVar.set('0.1')

        self.Entry_4.configure(textvariable=self.Entry_4_StringVar)
        self.Entry_4_StringVar_traceName = self.Entry_4_StringVar.trace_variable("w", self.Entry_4_StringVar_Callback)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Entry_5"
    def make_Entry_5(self, frame):
        """       Entry: DynaQ+ Factor : at Main(3,6)"""
        self.Entry_5 = Entry( frame , width="15")
        self.Entry_5.grid(row=3, column=6)
        self.Entry_5_StringVar = StringVar()

        # >>>>>>insert any user code below this comment for section "make_Entry_5"
        self.Entry_5_StringVar.set('0.0001')

        self.Entry_5.configure(textvariable=self.Entry_5_StringVar)
        self.Entry_5_StringVar_traceName = self.Entry_5_StringVar.trace_variable("w", self.Entry_5_StringVar_Callback)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Entry_6"
    def make_Entry_6(self, frame):
        """       Entry: Initial Q(s,a) Factor : at Main(3,3)"""
        self.Entry_6 = Entry( frame , width="15")
        self.Entry_6.grid(row=3, column=3)
        self.Entry_6_StringVar = StringVar()

        # >>>>>>insert any user code below this comment for section "make_Entry_6"
        self.Entry_6_StringVar.set('0.0')

        self.Entry_6.configure(textvariable=self.Entry_6_StringVar)
        self.Entry_6_StringVar_traceName = self.Entry_6_StringVar.trace_variable("w", self.Entry_6_StringVar_Callback)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Frame_1"
    def make_Frame_1(self, frame):
        """       Frame:  at Main(4,1)"""
        self.Frame_1 = Frame( frame , width="400", height="300")
        self.Frame_1.grid(row=4, column=1, sticky="nsew", columnspan="7")

        # >>>>>>insert any user code below this comment for section "make_Frame_1"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_1"
    def make_Label_1(self, frame):
        """       Label: Step Cost : at Main(2,1)"""
        self.Label_1 = Label( frame , text="Step Cost", width="15")
        self.Label_1.grid(row=2, column=1)

        # >>>>>>insert any user code below this comment for section "make_Label_1"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_10"
    def make_Label_10(self, frame):
        """       Label:  at Main(5,5)"""
        self.Label_10 = Label( frame , text="", height="3", width="15", background="#c4c4ff", font="Helvetica 10 bold roman")
        self.Label_10.grid(row=5, column=5)

        # >>>>>>insert any user code below this comment for section "make_Label_10"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_11"
    def make_Label_11(self, frame):
        """       Label: Q(s,a) Initial : at Main(2,3)"""
        self.Label_11 = Label( frame , text="Q(s,a) Initial", width="15")
        self.Label_11.grid(row=2, column=3)

        # >>>>>>insert any user code below this comment for section "make_Label_11"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_12"
    def make_Label_12(self, frame):
        """       Label:  at Main(5,6)"""
        self.Label_12 = Label( frame , text="", height="3", font="Helvetica 10 bold roman", width="15", background="#c4c4ff")
        self.Label_12.grid(row=5, column=6, sticky="ew")

        # >>>>>>insert any user code below this comment for section "make_Label_12"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_13"
    def make_Label_13(self, frame):
        """       Label:   : at Main(1,7)"""
        self.Label_13 = Label( frame , text=" ", width="2")
        self.Label_13.grid(row=1, column=7, sticky="ew")

        # >>>>>>insert any user code below this comment for section "make_Label_13"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_14"
    def make_Label_14(self, frame):
        """       Label:  at Main(5,7)"""
        self.Label_14 = Label( frame , text="", height="3", width="2", background="#c4c4ff", font="Helvetica 10 bold roman")
        self.Label_14.grid(row=5, column=7, sticky="ew")

        # >>>>>>insert any user code below this comment for section "make_Label_14"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_2"
    def make_Label_2(self, frame):
        """       Label: Alpha : at Main(2,2)"""
        self.Label_2 = Label( frame , text="Alpha", width="15")
        self.Label_2.grid(row=2, column=2)

        # >>>>>>insert any user code below this comment for section "make_Label_2"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_3"
    def make_Label_3(self, frame):
        """       Label: Gamma : at Main(2,4)"""
        self.Label_3 = Label( frame , text="Gamma", width="15")
        self.Label_3.grid(row=2, column=4)

        # >>>>>>insert any user code below this comment for section "make_Label_3"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_4"
    def make_Label_4(self, frame):
        """       Label: Epsilon : at Main(2,5)"""
        self.Label_4 = Label( frame , text="Epsilon", width="15")
        self.Label_4.grid(row=2, column=5)

        # >>>>>>insert any user code below this comment for section "make_Label_4"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_5"
    def make_Label_5(self, frame):
        """       Label: DynaQ+ Factor : at Main(2,6)"""
        self.Label_5 = Label( frame , text="DynaQ+ Factor", width="15")
        self.Label_5.grid(row=2, column=6)

        # >>>>>>insert any user code below this comment for section "make_Label_5"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_6"
    def make_Label_6(self, frame):
        """       Label:  at Main(5,1)"""
        self.Label_6 = Label( frame , text="", height="3", width="15", background="#c4c4ff", font="Helvetica 10 bold roman")
        self.Label_6.grid(row=5, column=1)

        # >>>>>>insert any user code below this comment for section "make_Label_6"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_7"
    def make_Label_7(self, frame):
        """       Label:  at Main(5,2)"""
        self.Label_7 = Label( frame , text="", height="3", width="15", background="#c4c4ff", font="Helvetica 10 bold roman")
        self.Label_7.grid(row=5, column=2)

        # >>>>>>insert any user code below this comment for section "make_Label_7"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_8"
    def make_Label_8(self, frame):
        """       Label:  at Main(5,3)"""
        self.Label_8 = Label( frame , text="", height="3", width="15", background="#c4c4ff", font="Helvetica 10 bold roman")
        self.Label_8.grid(row=5, column=3)

        # >>>>>>insert any user code below this comment for section "make_Label_8"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Label_9"
    def make_Label_9(self, frame):
        """       Label:  at Main(5,4)"""
        self.Label_9 = Label( frame , text="", height="3", width="15", background="#c4c4ff", font="Helvetica 10 bold roman")
        self.Label_9.grid(row=5, column=4)

        # >>>>>>insert any user code below this comment for section "make_Label_9"


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "make_Canvas_1"
    def make_Canvas_1(self, frame):
        """      Canvas:  at Frame_1(0,0)"""
        self.Canvas_1 = Canvas( frame , height="100", width="100")
        self.Canvas_1.grid(row=0, column=0)

        # >>>>>>insert any user code below this comment for section "make_Canvas_1"

        self.Canvas_1.config(bg='#ffffcc')
        self.Canvas_1.bind("<ButtonRelease-1>", self.Canvas_1_Click)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Button_1_Click"
    def Button_1_Click(self, event): #bind method for component ID=Button_1
        """      Button: Run : at Main(1,1)"""
        pass
        # >>>>>>insert any user code below this comment for section "Button_1_Click"
        # replace, delete, or comment-out the following
        print( "executed method Button_1_Click" )

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Button_2_Click"
    def Button_2_Click(self, event): #bind method for component ID=Button_2
        """      Button: Step : at Main(1,3)"""
        pass
        # >>>>>>insert any user code below this comment for section "Button_2_Click"
        # replace, delete, or comment-out the following
        print( "executed method Button_2_Click" )

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Button_3_Click"
    def Button_3_Click(self, event): #bind method for component ID=Button_3
        """      Button: Build Model : at Main(1,4)"""
        pass
        # >>>>>>insert any user code below this comment for section "Button_3_Click"
        # replace, delete, or comment-out the following
        print( "Building Model" )

        #from introrl.td_funcs.qlearning_epsilon_greedy import qlearning_epsilon_greedy
        #from introrl.td_funcs.sarsa_epsilon_greedy import sarsa_epsilon_greedy
        #from introrl.td_funcs.expected_sarsa_eps_greedy import expected_sarsa_eps_greedy
        #from introrl.agents.dyna_q import DynaQ

        if self.learner_type == 'Q-Learn': # Q-Learn SARSA DynaQ DynaQ+
            pass
        elif  self.learner_type == 'SARSA':
            pass
        elif  self.learner_type == 'DynaQ':
            pass
        elif  self.learner_type == 'DynaQ+':
            pass
        else:
            print('ERROR... Learner Type "%s" NOT RECOGNIZED, using Q-Learn'%self.learner_type)

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Button_4_Click"
    def Button_4_Click(self, event): #bind method for component ID=Button_4
        """      Button: 1 Episode : at Main(1,2)"""
        pass
        # >>>>>>insert any user code below this comment for section "Button_4_Click"
        # replace, delete, or comment-out the following
        print( "executed method Button_4_Click" )

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Canvas_1_Click"
    def Canvas_1_Click(self, event): #bind method for component ID=Canvas_1
        """      Canvas:  at Frame_1(0,0)"""
        pass
        # >>>>>>insert any user code below this comment for section "Canvas_1_Click"
        # replace, delete, or comment-out the following
        print( "executed method Canvas_1_Click" )

        print( "clicked in canvas at x,y =",event.x,event.y )
        w = int(event.widget.cget("width"))
        h = int(event.widget.cget("height"))
        event.widget.create_rectangle((2, 2, w+1, h+1), outline="blue")
    
    def get_real_entry_val(self, svar, def_val=0.0, msg_str=''):
        
        s = svar.get()
        try:
            val = float( s )
            print(msg_str, '"%s"'%s )
        except:
            val = def_val
            svar.set( str(def_val) )
            print(msg_str, '"%s"'%svar.get(), ' Corrected From: "%s"'%s )
        return val
            
    
    def get_model_settings(self):

        print('Model Parameters:')
        # The StringVar callbacks set values individually
        self.step_reward = self.get_real_entry_val( self.Entry_1_StringVar, def_val=0.0,      msg_str='   step_reward_str:' )
        self.alpha = self.get_real_entry_val( self.Entry_2_StringVar, def_val=0.1,            msg_str='         alpha_str:' )
        self.gamma = self.get_real_entry_val( self.Entry_3_StringVar, def_val=0.9,            msg_str='         gamma_str:' )
        self.epsilon = self.get_real_entry_val( self.Entry_4_StringVar, def_val=0.1,          msg_str='       epsilon_str:' )
        self.dyna_q_factor = self.get_real_entry_val( self.Entry_5_StringVar, def_val=0.0001, msg_str='     dyna_q_factor:' )
        self.qsa_initial = self.get_real_entry_val( self.Entry_6_StringVar, def_val=0.0,      msg_str='       qsa_initial:' )
        
        self.learner_type = str( self.Combobox_1_StringVar.get() )
        print('      learner_type:', '"%s"'%self.learner_type)

        
    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Combobox_1_StringVar_traceName"
    def Combobox_1_StringVar_Callback(self, varName, index, mode):
        """    Combobox: Q-Learn SARSA DynaQ DynaQ+ : at Main(1,6)"""
        pass

        # >>>>>>insert any user code below this comment for section "Combobox_1_StringVar_traceName"
        # replace, delete, or comment-out the following
        #print( "Combobox_1_StringVar_Callback varName, index, mode",varName, index, mode )
        #print( "    new StringVar value =",self.Combobox_1_StringVar.get() )

        self.learner_type = str( self.Combobox_1_StringVar.get() )
        print('      learner_type:', '"%s"'%self.learner_type)


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Combobox_2_StringVar_traceName"
    def Combobox_2_StringVar_Callback(self, varName, index, mode):
        """    Combobox: Display Size : at Main(1,5)"""
        pass

        # >>>>>>insert any user code below this comment for section "Combobox_2_StringVar_traceName"
        # replace, delete, or comment-out the following
        #print( "Combobox_2_StringVar_Callback varName, index, mode",varName, index, mode )
        #print( "    new StringVar value =",self.Combobox_2_StringVar.get() )

        s = self.Combobox_2_StringVar.get()
        sL = s.split('x')

        self.w_total = int(sL[0])
        self.h_total = int(sL[1])
        self.build_canvas_objects()



    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Entry_1_StringVar_traceName"
    def Entry_1_StringVar_Callback(self, varName, index, mode):
        """       Entry: Step Cost : at Main(3,1)"""
        pass

        # >>>>>>insert any user code below this comment for section "Entry_1_StringVar_traceName"
        # replace, delete, or comment-out the following
        #print( "Entry_1_StringVar_Callback varName, index, mode",varName, index, mode )
        #print( "    new StringVar value =",self.Entry_1_StringVar.get() )

        self.step_reward = self.get_real_entry_val( self.Entry_1_StringVar, def_val=0.0,      msg_str='   step_reward_str:' )


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Entry_2_StringVar_traceName"
    def Entry_2_StringVar_Callback(self, varName, index, mode):
        """       Entry: Alpha : at Main(3,2)"""
        pass

        # >>>>>>insert any user code below this comment for section "Entry_2_StringVar_traceName"
        # replace, delete, or comment-out the following
        #print( "Entry_2_StringVar_Callback varName, index, mode",varName, index, mode )
        #print( "    new StringVar value =",self.Entry_2_StringVar.get() )
        
        self.alpha = self.get_real_entry_val( self.Entry_2_StringVar, def_val=0.1,            msg_str='         alpha_str:' )



    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Entry_3_StringVar_traceName"
    def Entry_3_StringVar_Callback(self, varName, index, mode):
        """       Entry: Gamma : at Main(3,4)"""
        pass

        # >>>>>>insert any user code below this comment for section "Entry_3_StringVar_traceName"
        # replace, delete, or comment-out the following
        #print( "Entry_3_StringVar_Callback varName, index, mode",varName, index, mode )
        #print( "    new StringVar value =",self.Entry_3_StringVar.get() )

        self.gamma = self.get_real_entry_val( self.Entry_3_StringVar, def_val=0.9,            msg_str='         gamma_str:' )


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Entry_4_StringVar_traceName"
    def Entry_4_StringVar_Callback(self, varName, index, mode):
        """       Entry: Epsilon : at Main(3,5)"""
        pass

        # >>>>>>insert any user code below this comment for section "Entry_4_StringVar_traceName"
        # replace, delete, or comment-out the following
        #print( "Entry_4_StringVar_Callback varName, index, mode",varName, index, mode )
        #print( "    new StringVar value =",self.Entry_4_StringVar.get() )

        self.epsilon = self.get_real_entry_val( self.Entry_4_StringVar, def_val=0.1,          msg_str='       epsilon_str:' )

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Entry_5_StringVar_traceName"
    def Entry_5_StringVar_Callback(self, varName, index, mode):
        """       Entry: DynaQ+ Factor : at Main(3,6)"""
        pass

        # >>>>>>insert any user code below this comment for section "Entry_5_StringVar_traceName"
        # replace, delete, or comment-out the following
        #print( "Entry_5_StringVar_Callback varName, index, mode",varName, index, mode )
        #print( "    new StringVar value =",self.Entry_5_StringVar.get() )
        
        self.dyna_q_factor = self.get_real_entry_val( self.Entry_5_StringVar, def_val=0.0001, msg_str='     dyna_q_factor:' )



    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "Entry_6_StringVar_traceName"
    def Entry_6_StringVar_Callback(self, varName, index, mode):
        """       Entry: Initial Q(s,a) Factor : at Main(3,3)"""
        pass

        # >>>>>>insert any user code below this comment for section "Entry_6_StringVar_traceName"
        # replace, delete, or comment-out the following
        #print( "Entry_6_StringVar_Callback varName, index, mode",varName, index, mode )
        #print( "    new StringVar value =",self.Entry_6_StringVar.get() )

        self.qsa_initial = self.get_real_entry_val( self.Entry_6_StringVar, def_val=0.0,      msg_str='       qsa_initial:' )
        


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "menu_Model_Simple_Grid_World"
    def menu_Model_Simple_Grid_World(self):
        pass
        # >>>>>>insert any user code below this comment for section "menu_Model_Simple_Grid_World"
        # replace, delete, or comment-out the following
        print( "called menu_Model_Simple_Grid_World" )
        
        self.env_sim = simple_grid_world_get_env()
        
        self.build_canvas_objects()


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "menu_Model_Cliff_Walking"
    def menu_Model_Cliff_Walking(self):
        pass
        # >>>>>>insert any user code below this comment for section "menu_Model_Cliff_Walking"
        # replace, delete, or comment-out the following
        print( "called menu_Model_Cliff_Walking" )

        self.env_sim = CliffWalkingSimulation()
        
        self.build_canvas_objects()

    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "menu_Model_Blocking_Maze"
    def menu_Model_Blocking_Maze(self):
        pass
        # >>>>>>insert any user code below this comment for section "menu_Model_Blocking_Maze"
        # replace, delete, or comment-out the following
        print( "called menu_Model_Blocking_Maze" )
        
        self.env_sim = BlockingMaze()
        
        self.build_canvas_objects()


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "menu_Model_"
    def menu_Model_(self):
        pass
        # >>>>>>insert any user code below this comment for section "menu_Model_"
        # replace, delete, or comment-out the following
        print( "called menu_Model_" )


    # TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "menu_Model_Exit"
    def menu_Model_Exit(self):
        pass
        # >>>>>>insert any user code below this comment for section "menu_Model_Exit"
        # replace, delete, or comment-out the following
        print( "called menu_Model_Exit" )
        self.master.destroy()


# TkGridGUI generated code. DO NOT EDIT THE FOLLOWING. section "end"

def main():
    root = Tk()
    app = _tk_grid_world(root)
    root.mainloop()

if __name__ == '__main__':
    main()
