class RL:
    codes = [
        '''
import numpy as np

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15]
        self.actions = ['UP', 'DOWN', 'LEFT', 'RIGHT']

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)
        if action == 'UP':    row = max(row - 1, 0)
        elif action == 'DOWN':  row = min(row + 1, self.grid_size - 1)
        elif action == 'LEFT':  col = max(col - 1, 0)
        elif action == 'RIGHT': col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1  
        done = next_state in self.terminal_states
        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

def generate_episode(env):
    episode = []
    state = env.reset()
    done = False
    while not done:
        action = np.random.choice(env.actions)
        next_state, reward, done = env.step(state, action)
        episode.append((state, action, reward))
        state = next_state
    return episode

def mc_policy_evaluation(env, num_episodes=5000):
    V = np.zeros(env.grid_size * env.grid_size)
    returns = {s: [] for s in range(env.grid_size * env.grid_size)}

    for _ in range(num_episodes):
        episode = generate_episode(env)
        G = 0

        for idx in range(len(episode) - 1, -1, -1):
            state, action, reward = episode[idx]
            G = G + reward

            previous_states = [x[0] for x in episode[:idx]]
            if state not in previous_states:
                returns[state].append(G)
                V[state] = np.mean(returns[state])
    return V

if __name__ == "__main__":
    env = GridWorld()
    print("Running Monte Carlo Policy Evaluation (5000 Episodes)...\n")
    values = mc_policy_evaluation(env)

    print("\nResulting Value Function (V):")
    print(np.round(values.reshape(4, 4), 1))
        ''',
        '''
import numpy as np

def value_iteration_grid_world():
    grid_size = 4
    gamma = 1.0 
    theta = 1e-4

    terminal_states = [0, 15]
    actions = ['UP', 'DOWN', 'LEFT', 'RIGHT']

    V = np.zeros(grid_size * grid_size)

    print("Starting Value Iteration...")
    iteration = 0

    while True:
        delta = 0
        V_new = np.copy(V)

        for s in range(grid_size * grid_size):
            if s in terminal_states:
                continue 

            row, col = divmod(s, grid_size)
            action_values = []

            for action in actions:
        
                next_r, next_c = row, col

                if action == 'UP':    next_r = max(row - 1, 0)
                elif action == 'DOWN':  next_r = min(row + 1, grid_size - 1)
                elif action == 'LEFT':  next_c = max(col - 1, 0)
                elif action == 'RIGHT': next_c = min(col + 1, grid_size - 1)

                next_state = next_r * grid_size + next_c

                reward = -1

                val = reward + gamma * V[next_state]
                action_values.append(val)

            best_value = max(action_values)
            V_new[s] = best_value

            delta = max(delta, abs(best_value - V[s]))

        V = V_new
        iteration += 1

        if delta < theta:
            print(f"Converged after {iteration} iterations.")
            break

    print("\nOptimal Value Function (V*):")
    print(np.round(V.reshape(4, 4), 1))

    print("\nOptimal Policy (Planning Result):")
    arrows = {0: '↑', 1: '↓', 2: '←', 3: '→'}
    policy_grid = []

    for s in range(grid_size * grid_size):
        if s in terminal_states:
            policy_grid.append(" T ")
            continue

        row, col = divmod(s, grid_size)
        best_action_idx = -1
        best_val = -float('inf')

        for i, action in enumerate(actions):
            next_r, next_c = row, col
            if action == 'UP':    next_r = max(row - 1, 0)
            elif action == 'DOWN':  next_r = min(row + 1, grid_size - 1)
            elif action == 'LEFT':  next_c = max(col - 1, 0)
            elif action == 'RIGHT': next_c = min(col + 1, grid_size - 1)

            val = -1 + gamma * V[next_r * grid_size + next_c]
            if val > best_val:
                best_val = val
                best_action_idx = i

        policy_grid.append(f" {arrows[best_action_idx]} ")

    print("-" * 17)
    for i in range(0, 16, 4):
        print("|".join(policy_grid[i:i+4]))
        print("-" * 17)

if __name__ == "__main__":
    value_iteration_grid_world()
        ''',
        '''
import numpy as np

class MDPGridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [15]
        self.actions = ['UP', 'DOWN', 'LEFT', 'RIGHT']
        self.gamma = 1.0

    def get_next_state_reward(self, state, action):
        if state in self.terminal_states:
            return state, 0

        row, col = divmod(state, self.grid_size)

        if action == 'UP':    row = max(row - 1, 0)
        elif action == 'DOWN':  row = min(row + 1, self.grid_size - 1)
        elif action == 'LEFT':  col = max(col - 1, 0)
        elif action == 'RIGHT': col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1 
        return next_state, reward

    def value_iteration(self):
        
        V = np.zeros(self.grid_size * self.grid_size)
        theta = 1e-4

        while True:
            delta = 0
            V_new = np.copy(V)
            for s in range(self.grid_size * self.grid_size):
                if s in self.terminal_states: continue

                action_values = []
                for a in self.actions:
                    ns, r = self.get_next_state_reward(s, a)
                    action_values.append(r + self.gamma * V[ns])

                new_val = max(action_values)
                V_new[s] = new_val
                delta = max(delta, abs(new_val - V[s]))
            V = V_new
            if delta < theta: break
        return V

    def rollout(self, start_state, V):
        
        print(f"\n--- Rolling out Optimal Policy from State {start_state} (0,0) ---")

        curr_state = start_state
        steps = 0
        path = [curr_state]

        while curr_state not in self.terminal_states:
            row, col = divmod(curr_state, self.grid_size)

            best_action = None
            best_val = -float('inf')

            for action in self.actions:
                ns, r = self.get_next_state_reward(curr_state, action)
                val = r + self.gamma * V[ns]

                if val > best_val:
                    best_val = val
                    best_action = action

            next_state, _ = self.get_next_state_reward(curr_state, best_action)

            print(f"Step {steps+1}: At {curr_state} ({row},{col}) -> Action: {best_action} -> New State: {next_state}")

            curr_state = next_state
            path.append(curr_state)
            steps += 1

            if steps > 20: 
                print("Stuck in loop!")
                break

        print(f"\nGoal Reached at State {curr_state}!")
        print(f"Total Path: {path}")

if __name__ == "__main__":
    world = MDPGridWorld()

    print("Computing Optimal Policy...")
    optimal_values = world.value_iteration()

    world.rollout(start_state=0, V=optimal_values)
        ''',
        '''
import numpy as np

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15] 
        self.actions = [0, 1, 2, 3]

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)

        if action == 0:   row = max(row - 1, 0) 
        elif action == 1: row = min(row + 1, self.grid_size - 1)
        elif action == 2: col = max(col - 1, 0)
        elif action == 3: col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1
        done = next_state in self.terminal_states

        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

def q_learning():
    env = GridWorld()

    num_episodes = 5000
    alpha = 0.1   
    gamma = 0.99
    epsilon = 0.1 

    Q = np.zeros((16, 4))

    print("Training with Q-Learning (5000 Episodes)...")

    for episode in range(num_episodes):
        state = env.reset()
        done = False

        while not done:
            
            if np.random.rand() < epsilon:
                action = np.random.choice(env.actions) 
            else:
                action = np.argmax(Q[state]) 

            next_state, reward, done = env.step(state, action)

            old_value = Q[state, action]

            next_max = np.max(Q[next_state])

            new_value = old_value + alpha * (reward + gamma * next_max - old_value)

            Q[state, action] = new_value

            state = next_state

    return Q

def print_results(Q):
    actions_map = {0: '↑', 1: '↓', 2: '←', 3: '→'}
    print("\nLearned Policy (from Q-Table):")
    print("-" * 17)

    grid_output = []
    for s in range(16):
        if s in [0, 15]: 
            grid_output.append(" T ")
            continue

        best_action_idx = np.argmax(Q[s])
        grid_output.append(f" {actions_map[best_action_idx]} ")

    for i in range(0, 16, 4):
        print("|".join(grid_output[i:i+4]))
        print("-" * 17)

    print("\nExample Q-Values for State 1 (Next to Top-Left Goal):")
    print(f"UP: {Q[1,0]:.2f}, DOWN: {Q[1,1]:.2f}, LEFT: {Q[1,2]:.2f}, RIGHT: {Q[1,3]:.2f}")
    print("(Notice 'LEFT' should have the highest value because it leads to the goal)")

if __name__ == "__main__":
    final_Q = q_learning()
    print_results(final_Q)
        ''',
        '''
import numpy as np

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15] 
        self.actions = ['UP', 'DOWN', 'LEFT', 'RIGHT']

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)

        if action == 'UP':    row = max(row - 1, 0)
        elif action == 'DOWN':  row = min(row + 1, self.grid_size - 1)
        elif action == 'LEFT':  col = max(col - 1, 0)
        elif action == 'RIGHT': col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1
        done = next_state in self.terminal_states
        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

def td_zero_learning(env, num_episodes=5000, alpha=0.1, gamma=1.0):

    V = np.zeros(env.grid_size * env.grid_size)

    print(f"Running TD(0) Learning for {num_episodes} episodes...")

    for _ in range(num_episodes):
        state = env.reset()
        done = False

        while not done:
        
            action = np.random.choice(env.actions)
            next_state, reward, done = env.step(state, action)

            current_value = V[state]

            next_state_value = 0 if done else V[next_state]
            td_target = reward + gamma * next_state_value

            V[state] = current_value + alpha * (td_target - current_value)

            state = next_state

    return V

if __name__ == "__main__":
    env = GridWorld()

    v_values = td_zero_learning(env)

    print("\nState-Value Function V(s) learned via TD(0):")
    print("-" * 30)

    print(np.round(v_values.reshape(4, 4), 2))
        ''',
        '''
import numpy as np

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15]
        self.actions = ['UP', 'DOWN', 'LEFT', 'RIGHT']

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)

        if action == 'UP':    row = max(row - 1, 0)
        elif action == 'DOWN':  row = min(row + 1, self.grid_size - 1)
        elif action == 'LEFT':  col = max(col - 1, 0)
        elif action == 'RIGHT': col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1
        done = next_state in self.terminal_states
        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

def td_lambda_learning(env, lam=0.5, num_episodes=5000, alpha=0.1, gamma=1.0):

    V = np.zeros(env.grid_size * env.grid_size)

    print(f"Running TD({lam}) Learning...")

    for episode in range(num_episodes):
        
        E = np.zeros(env.grid_size * env.grid_size)

        state = env.reset()
        done = False

        while not done:
            action = np.random.choice(env.actions)
            next_state, reward, done = env.step(state, action)

            target = reward + (0 if done else gamma * V[next_state])
            delta = target - V[state]

            E[state] += 1
            V += alpha * delta * E
            E *= gamma * lam

            state = next_state

    return V

if __name__ == "__main__":
    env = GridWorld()

    v_values = td_lambda_learning(env, lam=0.5)

    print(f"\nState-Value Function V(s) via TD(0.5):")
    print("-" * 30)
    print(np.round(v_values.reshape(4, 4), 2))
        ''',
        '''
import numpy as np

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15]
        self.actions = [0, 1, 2, 3]

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)

        if action == 0:   row = max(row - 1, 0) 
        elif action == 1: row = min(row + 1, self.grid_size - 1) 
        elif action == 2: col = max(col - 1, 0) 
        elif action == 3: col = min(col + 1, self.grid_size - 1) 

        next_state = row * self.grid_size + col
        reward = -1
        done = next_state in self.terminal_states
        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

def sarsa_learning():
    env = GridWorld()

    num_episodes = 5000
    alpha = 0.1
    gamma = 1.0
    epsilon = 0.1

    Q = np.zeros((16, 4))

    print("Training with SARSA (5000 Episodes)...")

    for _ in range(num_episodes):
        state = env.reset()

        if np.random.rand() < epsilon:
            action = np.random.choice(env.actions)
        else:
            action = np.argmax(Q[state])

        done = False

        while not done:
            next_state, reward, done = env.step(state, action)

            if np.random.rand() < epsilon:
                next_action = np.random.choice(env.actions)
            else:
                next_action = np.argmax(Q[next_state])

            q_next = 0 if done else Q[next_state, next_action]

            target = reward + gamma * q_next
            Q[state, action] += alpha * (target - Q[state, action])

            state = next_state
            action = next_action

    return Q

def print_policy(Q):
    actions_map = {0: '↑', 1: '↓', 2: '←', 3: '→'}
    print("\nFinal Policy (SARSA):")
    print("-" * 17)

    grid_output = []
    for s in range(16):
        if s in [0, 15]:
            grid_output.append(" T ")
            continue
        best_action = np.argmax(Q[s])
        grid_output.append(f" {actions_map[best_action]} ")

    for i in range(0, 16, 4):
        print("|".join(grid_output[i:i+4]))
        print("-" * 17)

if __name__ == "__main__":
    q_table = sarsa_learning()
    print_policy(q_table)
        ''',
        '''
import numpy as np

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15] 
        self.actions = [0, 1, 2, 3]

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)

        # Move Logic
        if action == 0:   row = max(row - 1, 0)
        elif action == 1: row = min(row + 1, self.grid_size - 1)
        elif action == 2: col = max(col - 1, 0) 
        elif action == 3: col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1
        done = next_state in self.terminal_states
        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

def q_learning():
    env = GridWorld()

    num_episodes = 5000
    alpha = 0.1  
    gamma = 1.0  
    epsilon = 0.1 

    Q = np.zeros((16, 4))

    print("Training with Q-Learning (5000 Episodes)...")

    for _ in range(num_episodes):
        state = env.reset()
        done = False

        while not done:
            
            if np.random.rand() < epsilon:
                action = np.random.choice(env.actions)
            else:
                action = np.argmax(Q[state])

            next_state, reward, done = env.step(state, action)

            best_next_action_val = 0 if done else np.max(Q[next_state])

            td_target = reward + gamma * best_next_action_val
            Q[state, action] += alpha * (td_target - Q[state, action])

            state = next_state

    return Q

def print_policy(Q):
    actions_map = {0: '↑', 1: '↓', 2: '←', 3: '→'}
    print("\nFinal Optimal Policy (Q-Learning):")
    print("-" * 17)

    grid_output = []
    for s in range(16):
        if s in [0, 15]:
            grid_output.append(" T ")
            continue

        best_action = np.argmax(Q[s])
        grid_output.append(f" {actions_map[best_action]} ")

    for i in range(0, 16, 4):
        print("|".join(grid_output[i:i+4]))
        print("-" * 17)

if __name__ == "__main__":
    q_table = q_learning()
    print_policy(q_table)
        ''',
        '''
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import random
from collections import deque

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15]
        self.actions = [0, 1, 2, 3] 

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)

        if action == 0:   row = max(row - 1, 0)
        elif action == 1: row = min(row + 1, self.grid_size - 1)
        elif action == 2: col = max(col - 1, 0)
        elif action == 3: col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1
        done = next_state in self.terminal_states
        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

class QNetwork(nn.Module):
    def __init__(self):
        super(QNetwork, self).__init__()
        self.fc1 = nn.Linear(16, 128)
        self.fc2 = nn.Linear(128, 64)
        self.fc3 = nn.Linear(64, 4)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        return self.fc3(x)

def state_to_tensor(state):
    v = torch.zeros(16)
    v[state] = 1.0
    return v.unsqueeze(0)

def train_dqn():
    env = GridWorld()

    episodes = 1000
    gamma = 0.99
    epsilon = 1.0
    epsilon_decay = 0.995
    epsilon_min = 0.1
    learning_rate = 0.001
    batch_size = 32

    policy_net = QNetwork()
    optimizer = optim.Adam(policy_net.parameters(), lr=learning_rate)
    criterion = nn.MSELoss()

    memory = deque(maxlen=2000)

    print("Training DQN (this may take a moment)...")

    for episode in range(episodes):
        state = env.reset()
        done = False

        while not done:
            state_tensor = state_to_tensor(state)

            if random.random() < epsilon:
                action = random.choice(env.actions)
            else:
                with torch.no_grad():
                    q_values = policy_net(state_tensor)
                    action = torch.argmax(q_values).item()

            next_state, reward, done = env.step(state, action)

            memory.append((state, action, reward, next_state, done))
            state = next_state

            if len(memory) > batch_size:
                minibatch = random.sample(memory, batch_size)

                states_b = torch.cat([state_to_tensor(x[0]) for x in minibatch])
                next_states_b = torch.cat([state_to_tensor(x[3]) for x in minibatch])

                q_preds = policy_net(states_b)

                with torch.no_grad():
                    q_next = policy_net(next_states_b)

                target_q_values = q_preds.clone()

                for i, (s, a, r, ns, d) in enumerate(minibatch):
                    target = r
                    if not d:
                        target += gamma * torch.max(q_next[i]).item()
                    target_q_values[i][a] = target

                loss = criterion(q_preds, target_q_values)
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

        if epsilon > epsilon_min:
            epsilon *= epsilon_decay

        if (episode+1) % 200 == 0:
            print(f"Episode {episode+1}/{episodes} completed.")

    return policy_net

if __name__ == "__main__":
    trained_model = train_dqn()

    print("\nVisualizing DQN Policy:")
    print("-" * 17)
    actions_map = {0: '↑', 1: '↓', 2: '←', 3: '→'}
    env = GridWorld()

    output_grid = []
    for s in range(16):
        if s in [0, 15]:
            output_grid.append(" T ")
            continue

        st = state_to_tensor(s)
        with torch.no_grad():
            q = trained_model(st)
            best_a = torch.argmax(q).item()
        output_grid.append(f" {actions_map[best_a]} ")

    for i in range(0, 16, 4):
        print("|".join(output_grid[i:i+4]))
        print("-" * 17)
        ''',
        '''
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15]
        self.actions = [0, 1, 2, 3] 

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)
        if action == 0:   row = max(row - 1, 0)
        elif action == 1: row = min(row + 1, self.grid_size - 1)
        elif action == 2: col = max(col - 1, 0)
        elif action == 3: col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1
        done = next_state in self.terminal_states
        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

class PolicyNetwork(nn.Module):
    def __init__(self):
        super(PolicyNetwork, self).__init__()
        self.fc1 = nn.Linear(16, 128)
        self.fc2 = nn.Linear(128, 4)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.softmax(x, dim=-1)

class ValueNetwork(nn.Module):
    def __init__(self):
        super(ValueNetwork, self).__init__()
        self.fc1 = nn.Linear(16, 128)
        self.fc2 = nn.Linear(128, 1)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        return self.fc2(x)

def state_to_tensor(state):
    v = torch.zeros(16)
    v[state] = 1.0
    return v.unsqueeze(0)

def train_reinforce_baseline():
    env = GridWorld()

    policy_net = PolicyNetwork()
    value_net = ValueNetwork()

    policy_optimizer = optim.Adam(policy_net.parameters(), lr=0.0005)
    value_optimizer = optim.Adam(value_net.parameters(), lr=0.0005)

    num_episodes = 2000
    gamma = 0.99

    print("Training REINFORCE with Baseline")

    for episode in range(num_episodes):
        state = env.reset()
        done = False

        log_probs = []
        values = []
        rewards = []

        while not done:
            state_t = state_to_tensor(state)

            probs = policy_net(state_t)
            value = value_net(state_t)

            dist = torch.distributions.Categorical(probs)
            action = dist.sample()

            next_state, reward, done = env.step(state, action.item())

            log_probs.append(dist.log_prob(action))
            values.append(value)
            rewards.append(reward)

            state = next_state

            if len(rewards) > 100:
                break

        returns = []
        G = 0
        for r in reversed(rewards):
            G = r + gamma * G
            returns.insert(0, G)
        returns = torch.tensor(returns, dtype=torch.float32)

        if len(returns) > 1:
            returns = (returns - returns.mean()) / (returns.std() + 1e-9)
        else:
            returns = returns - returns.mean()

        policy_loss = []
        value_loss = []

        for log_prob, value, G_t in zip(log_probs, values, returns):
            advantage = G_t - value.item()

            policy_loss.append(-log_prob * advantage)

            target = torch.tensor([G_t], dtype=torch.float32)
            value_loss.append(F.mse_loss(value.view(-1), target))

        policy_optimizer.zero_grad()
        value_optimizer.zero_grad()

        if policy_loss:
            loss_p = torch.stack(policy_loss).sum()
            loss_v = torch.stack(value_loss).sum()

            loss_p.backward()
            loss_v.backward()

            torch.nn.utils.clip_grad_norm_(policy_net.parameters(), 1.0)
            torch.nn.utils.clip_grad_norm_(value_net.parameters(), 1.0)

            policy_optimizer.step()
            value_optimizer.step()

        if (episode + 1) % 500 == 0:
            print(f"Episode {episode + 1}/{num_episodes} completed.")

    return policy_net

if __name__ == "__main__":
    trained_policy = train_reinforce_baseline()

    print("\nFinal Policy (REINFORCE):")
    actions_map = {0: '↑', 1: '↓', 2: '←', 3: '→'}

    output_grid = []
    for s in range(16):
        if s in [0, 15]:
            output_grid.append(" T ")
            continue
        st = state_to_tensor(s)
        with torch.no_grad():
            probs = trained_policy(st)
            best_a = torch.argmax(probs).item()
        output_grid.append(f" {actions_map[best_a]} ")

    print("-" * 17)
    for i in range(0, 16, 4):
        print("|".join(output_grid[i:i+4]))
        print("-" * 17)
        ''',
        '''
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

class GridWorld:
    def __init__(self):
        self.grid_size = 4
        self.terminal_states = [0, 15]
        self.actions = [0, 1, 2, 3] 

    def step(self, state, action):
        if state in self.terminal_states:
            return state, 0, True

        row, col = divmod(state, self.grid_size)
        if action == 0:   row = max(row - 1, 0)
        elif action == 1: row = min(row + 1, self.grid_size - 1)
        elif action == 2: col = max(col - 1, 0)
        elif action == 3: col = min(col + 1, self.grid_size - 1)

        next_state = row * self.grid_size + col
        reward = -1
        done = next_state in self.terminal_states
        return next_state, reward, done

    def reset(self):
        start_state = np.random.randint(0, 16)
        while start_state in self.terminal_states:
            start_state = np.random.randint(0, 16)
        return start_state

class PolicyNetwork(nn.Module):
    def __init__(self):
        super(PolicyNetwork, self).__init__()
        self.fc1 = nn.Linear(16, 128)
        self.fc2 = nn.Linear(128, 4)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.softmax(x, dim=-1)

class ValueNetwork(nn.Module):
    def __init__(self):
        super(ValueNetwork, self).__init__()
        self.fc1 = nn.Linear(16, 128)
        self.fc2 = nn.Linear(128, 1)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        return self.fc2(x)

def state_to_tensor(state):
    v = torch.zeros(16)
    v[state] = 1.0
    return v.unsqueeze(0)

def train_reinforce_advantage():
    env = GridWorld()

    policy_net = PolicyNetwork()
    value_net = ValueNetwork()

    policy_optimizer = optim.Adam(policy_net.parameters(), lr=0.0005)
    value_optimizer = optim.Adam(value_net.parameters(), lr=0.0005)

    num_episodes = 2000
    gamma = 0.99

    print("Training REINFORCE using Advantage Function...")

    for episode in range(num_episodes):
        state = env.reset()
        done = False

        log_probs = []
        values = []
        rewards = []

        while not done:
            state_t = state_to_tensor(state)

            probs = policy_net(state_t)
            value = value_net(state_t)

            dist = torch.distributions.Categorical(probs)
            action = dist.sample()

            next_state, reward, done = env.step(state, action.item())

            log_probs.append(dist.log_prob(action))
            values.append(value)
            rewards.append(reward)

            state = next_state
            if len(rewards) > 100: break 

        returns = []
        G = 0
        for r in reversed(rewards):
            G = r + gamma * G
            returns.insert(0, G)
        returns = torch.tensor(returns, dtype=torch.float32)

        if len(returns) > 1:
            returns = (returns - returns.mean()) / (returns.std() + 1e-9)
        else:
            returns = returns - returns.mean()

        policy_loss = []
        value_loss = []

        for log_prob, value, G_t in zip(log_probs, values, returns):
            advantage = G_t - value.item()

            policy_loss.append(-log_prob * advantage)

            target = torch.tensor([G_t], dtype=torch.float32)
            value_loss.append(F.mse_loss(value.view(-1), target))

        policy_optimizer.zero_grad()
        value_optimizer.zero_grad()

        if policy_loss:
            loss_p = torch.stack(policy_loss).sum()
            loss_v = torch.stack(value_loss).sum()

            loss_p.backward()
            loss_v.backward()

            torch.nn.utils.clip_grad_norm_(policy_net.parameters(), 1.0)
            torch.nn.utils.clip_grad_norm_(value_net.parameters(), 1.0)

            policy_optimizer.step()
            value_optimizer.step()

        if (episode + 1) % 500 == 0:
            print(f"Episode {episode + 1}/{num_episodes} completed.")

    return policy_net

if __name__ == "__main__":
    trained_policy = train_reinforce_advantage()

    print("\nFinal Policy (Advantage Method):")
    actions_map = {0: '↑', 1: '↓', 2: '←', 3: '→'}

    output_grid = []
    for s in range(16):
        if s in [0, 15]:
            output_grid.append(" T ")
            continue
        st = state_to_tensor(s)
        with torch.no_grad():
            probs = trained_policy(st)
            best_a = torch.argmax(probs).item()
        output_grid.append(f" {actions_map[best_a]} ")

    print("-" * 17)
    for i in range(0, 16, 4):
        print("|".join(output_grid[i:i+4]))
        print("-" * 17)
        ''',
        '''
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.distributions import Normal

class ContinuousTargetEnv:
    def __init__(self):
        self.state = np.array([-2.0], dtype=np.float32)
        self.max_steps = 200
        self.current_step = 0

    def reset(self):
        self.state = np.array([np.random.uniform(-2, -1)], dtype=np.float32)
        self.current_step = 0
        return self.state

    def step(self, action):
        force = np.clip(action, -1.0, 1.0)
        self.state[0] += force * 0.1
        dist = abs(self.state[0] - 0.0)
        reward = -dist

        self.current_step += 1
        done = dist < 0.1 or self.current_step >= self.max_steps

        if dist < 0.1:
            reward += 10.0

        return self.state, reward, done

class ActorCritic(nn.Module):
    def __init__(self):
        super(ActorCritic, self).__init__()
        self.fc1 = nn.Linear(1, 128)
        self.mu_head = nn.Linear(128, 1)
        self.sigma_head = nn.Linear(128, 1)

        self.value_head = nn.Linear(128, 1)

    def forward(self, x):
        x = F.relu(self.fc1(x))

        mu = torch.tanh(self.mu_head(x)) # Output between -1 and 1
        sigma = F.softplus(self.sigma_head(x)) + 1e-5 # Always positive

        value = self.value_head(x)

        return mu, sigma, value

def train_a2c_continuous():
    env = ContinuousTargetEnv()
    model = ActorCritic()

    optimizer = optim.Adam(model.parameters(), lr=0.001)

    episodes = 1000
    gamma = 0.99

    print("Training A2C for Continuous Control...")

    for episode in range(episodes):
        state = env.reset()
        done = False
        total_reward = 0

        while not done:
            state_t = torch.FloatTensor(state)

            mu, sigma, value = model(state_t)

            dist = Normal(mu, sigma)
            action = dist.sample()

            action_numpy = action.detach().numpy()[0]

            next_state, reward, done = env.step(action_numpy)
            total_reward += reward

            next_state_t = torch.FloatTensor(next_state)

            with torch.no_grad():
                _, _, next_value = model(next_state_t)
                target_value = reward + (0 if done else gamma * next_value.item())

            advantage = target_value - value
            critic_loss = advantage.pow(2)
            log_prob = dist.log_prob(action)
            actor_loss = -log_prob * advantage.detach()
            loss = actor_loss + critic_loss

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            state = next_state

        if (episode + 1) % 100 == 0:
            print(f"Episode {episode + 1}/episodes: Total Reward = {total_reward:.2f}")

    return model

if __name__ == "__main__":
    trained_model = train_a2c_continuous()

    print("\nTesting Trained Policy (Moving from -2.0 to 0.0):")
    env = ContinuousTargetEnv()
    state = env.reset()

    for i in range(10):
        state_t = torch.FloatTensor(state)
        with torch.no_grad():
            mu, sigma, _ = trained_model(state_t)
            action = mu.item()

        next_state, _, done = env.step(action)
        print(f"Step {i+1}: Pos {state[0]:.2f} -> Action {action:.2f} -> New Pos {next_state[0]:.2f}")
        state = next_state
        if done: break
        ''',
        '''
import numpy as np

class MultiAgentGridWorld:
    def __init__(self):
        self.grid_size = 4
        self.actions = [0, 1, 2, 3] 
        self.goal_A = (3, 3)
        self.goal_B = (0, 3)

    def reset(self):
        self.pos_A = (0, 0)
        self.pos_B = (3, 0)

        return self.pos_A + self.pos_B

    def step(self, action_A, action_B):
        new_pos_A = self._move(self.pos_A, action_A)
        new_pos_B = self._move(self.pos_B, action_B)

        reward_A = -1
        reward_B = -1
        done_A = False
        done_B = False

        if new_pos_A == new_pos_B:
           
            reward_A = -10
            reward_B = -10
            new_pos_A = self.pos_A
            new_pos_B = self.pos_B
        else:

            if new_pos_A == self.goal_A:
                reward_A = 100
                done_A = True

            if new_pos_B == self.goal_B:
                reward_B = 100
                done_B = True

        if not done_A: self.pos_A = new_pos_A
        if not done_B: self.pos_B = new_pos_B

        next_state = self.pos_A + self.pos_B

        done = done_A or done_B

        return next_state, reward_A, reward_B, done

    def _move(self, pos, action):
        r, c = pos
        if action == 0:   r = max(r - 1, 0) 
        elif action == 1: r = min(r + 1, self.grid_size - 1) 
        elif action == 2: c = max(c - 1, 0) 
        elif action == 3: c = min(c + 1, self.grid_size - 1) 
        return (r, c)

def train_marl():
    env = MultiAgentGridWorld()

    def get_state_idx(state_tuple):
        r1, c1, r2, c2 = state_tuple

        return r1*64 + c1*16 + r2*4 + c2

    Q_A = np.zeros((256, 4))
    Q_B = np.zeros((256, 4))

    episodes = 5000
    alpha = 0.1
    gamma = 0.95
    epsilon = 0.1

    print("Training Multi-Agent System (Agents A & B)...")

    for episode in range(episodes):
        state = env.reset()
        state_idx = get_state_idx(state)
        done = False

        while not done:
            if np.random.rand() < epsilon:
                act_A = np.random.choice(env.actions)
            else:
                act_A = np.argmax(Q_A[state_idx])

            if np.random.rand() < epsilon:
                act_B = np.random.choice(env.actions)
            else:
                act_B = np.argmax(Q_B[state_idx])

            next_state, rA, rB, done = env.step(act_A, act_B)
            next_state_idx = get_state_idx(next_state)

            old_val_A = Q_A[state_idx, act_A]
            next_max_A = np.max(Q_A[next_state_idx])
            Q_A[state_idx, act_A] = old_val_A + alpha * (rA + gamma * next_max_A - old_val_A)

            old_val_B = Q_B[state_idx, act_B]
            next_max_B = np.max(Q_B[next_state_idx])
            Q_B[state_idx, act_B] = old_val_B + alpha * (rB + gamma * next_max_B - old_val_B)

            state_idx = next_state_idx

            if rA == 100 or rB == 100:
                break

    return Q_A, Q_B, env

if __name__ == "__main__":
    qa, qb, env = train_marl()

    print("\n--- Testing MARL Interaction ---")
    print("Agent A: (0,0) -> (3,3)")
    print("Agent B: (3,0) -> (0,3)")

    state = env.reset()
    state_idx = 0 

    def print_grid(pos_a, pos_b):
        grid = [[' . ' for _ in range(4)] for _ in range(4)]
        grid[pos_a[0]][pos_a[1]] = ' A '
        grid[pos_b[0]][pos_b[1]] = ' B '
        if pos_a == pos_b: grid[pos_a[0]][pos_a[1]] = ' X ' 
        for row in grid:
            print("".join(row))
        print("-" * 12)

    pos_A = (0,0)
    pos_B = (3,0)

    print_grid(pos_A, pos_B)

    for step in range(8):
        
        idx = pos_A[0]*64 + pos_A[1]*16 + pos_B[0]*4 + pos_B[1]

        act_A = np.argmax(qa[idx])
        act_B = np.argmax(qb[idx])

        move_map = {0:'UP', 1:'DOWN', 2:'LEFT', 3:'RIGHT'}
        print(f"Step {step+1}: Agent A goes {move_map[act_A]}, Agent B goes {move_map[act_B]}")

        new_state, _, _, _ = env.step(act_A, act_B)

        pos_A = (new_state[0], new_state[1])
        pos_B = (new_state[2], new_state[3])

        print_grid(pos_A, pos_B)

        if pos_A == (3,3) and pos_B == (0,3):
            print("Both Agents Reached Goals!")
            break
        ''',
    ]

    @staticmethod
    def text(index):
        """Fetch a specific code based on the index."""
        try:
            return RL.codes[index - 1]
        except IndexError:
            return f"Invalid code index. Please choose a number between 1 and {len(RL.codes)}."
