from textblob import TextBlob
import spacy

# Load the model carefully
try:
    nlp = spacy.load("en_core_web_sm")
except OSError:
    print("Warning: Model not found. Run: python -m spacy download en_core_web_sm")

def analyze_sentiment(text):
    """Returns the polarity score and sentiment label."""
    blob = TextBlob(text)
    polarity = blob.sentiment.polarity
    
    if polarity > 0:
        sentiment = "Positive"
    elif polarity < 0:
        sentiment = "Negative"
    else:
        sentiment = "Neutral"
        
    return {"polarity": polarity, "sentiment": sentiment}

def extract_entities(text):
    """Returns a dictionary of entity counts."""
    doc = nlp(text)
    entity_count = {}
    
    for ent in doc.ents:
        current_count = entity_count.get(ent.label_, 0)
        entity_count[ent.label_] = current_count + 1
        
    return entity_count

import nltk
import spacy

def tokenize_text(text):
    """Breaks text into words and sentences."""
    nltk.download('punkt', quiet=True)
    nltk.download('punkt_tab', quiet=True)
    from nltk.tokenize import word_tokenize, sent_tokenize
    
    return {
        "words": word_tokenize(text),
        "sentences": sent_tokenize(text)
    }

def remove_stopwords(text):
    """Removes common English stopwords from text."""
    nltk.download('punkt', quiet=True)
    nltk.download('stopwords', quiet=True)
    from nltk.tokenize import word_tokenize
    from nltk.corpus import stopwords
    
    stop_words = set(stopwords.words('english'))
    words = word_tokenize(text)
    filtered_words = [word for word in words if word.lower() not in stop_words]
    
    return filtered_words

def lemmatize_text(word_list):
    """Reduces words to their base/root form."""
    nltk.download('wordnet', quiet=True)
    from nltk.stem import WordNetLemmatizer
    
    lemmatizer = WordNetLemmatizer()
    return [lemmatizer.lemmatize(word) for word in word_list]

def get_nltk_pos_tags(text):
    """Returns Part-of-Speech tags using NLTK."""
    nltk.download('punkt', quiet=True)
    nltk.download('averaged_perceptron_tagger_eng', quiet=True)
    from nltk.tokenize import word_tokenize
    from nltk import pos_tag
    
    tokens = word_tokenize(text)
    return pos_tag(tokens)

def spacy_extract_entities(text, model="en_core_web_sm"):
    """Extracts Named Entities using spaCy."""
    # Note: The user must have downloaded the model via CLI first:
    # python -m spacy download en_core_web_sm
    nlp = spacy.load(model)
    doc = nlp(text)
    
    entities = {ent.text: ent.label_ for ent in doc.ents}
    return entities

def train_word2vec_similarity(sentences, word1, word2):
    """Trains a quick Word2Vec model and checks similarity between two words."""
    from gensim.models import Word2Vec
    
    # Example input: sentences = [["cat", "say", "meow"], ["dog", "say", "woof"]]
    model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)
    
    if word1 in model.wv and word2 in model.wv:
        similarity = model.wv.similarity(word1, word2)
        return similarity
    return None

def get_glove_similarity(word1, word2):
    """Downloads GloVe embeddings (Heavy!) and finds similarity."""
    import gensim.downloader as api
    
    print("Loading GloVe model (this may take a while)...")
    word_vectors = api.load("glove-wiki-gigaword-50")
    
    if word1 in word_vectors and word2 in word_vectors:
        return word_vectors.similarity(word1, word2)
    return None

def get_bert_embeddings(sentence):
    """Generates BERT embeddings for a given sentence."""
    from transformers import BertTokenizer, BertModel
    import torch
    
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    model = BertModel.from_pretrained('bert-base-uncased')
    
    inputs = tokenizer(sentence, return_tensors='pt')
    outputs = model(**inputs)
    
    # Returns the hidden state tensor
    return outputs.last_hidden_state

def get_roberta_embeddings(sentence):
    """Generates RoBERTa embeddings for a given sentence."""
    from transformers import RobertaTokenizer, RobertaModel
    
    tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
    model = RobertaModel.from_pretrained("roberta-base")
    
    inputs = tokenizer(sentence, return_tensors="pt")
    outputs = model(**inputs)
    
    return outputs.last_hidden_state

import pandas as pd
from sklearn.tree import DecisionTreeClassifier
from collections import Counter

def find_complex_words(sentence, threshold=50):
    """Identifies complex words in a sentence based on Brown corpus frequency."""
    import nltk
    nltk.download('brown', quiet=True)
    from nltk.corpus import brown
    
    words = brown.words()
    freq_dist = Counter(word.lower() for word in words)
    
    results = {}
    for word in sentence.split():
        # If the word appears less than the threshold, it is considered complex
        is_complex = freq_dist[word.lower()] < threshold
        results[word] = "Complex" if is_complex else "Simple"
        
    return results

def classify_words_decision_tree(train_data, test_words):
    """
    Trains a Decision Tree to classify words as Simple or Complex.
    train_data format: [("dog", 0), ("photosynthesis", 1)]
    """
    import nltk
    nltk.download('brown', quiet=True)
    from nltk.corpus import brown
    
    # Get frequency distribution for feature extraction
    words = brown.words()
    freq_dist = Counter(word.lower() for word in words)
    
    def extract_features(word):
        return {
            "length": len(word),
            "frequency": freq_dist.get(word.lower(), 0)
        }

    # Prepare training data
    df = pd.DataFrame(train_data, columns=["word", "label"])
    features = df["word"].apply(extract_features)
    X = pd.DataFrame(features.tolist())
    y = df["label"]

    # Train model
    model = DecisionTreeClassifier()
    model.fit(X, y)

    # Prepare testing data
    test_features = pd.DataFrame([extract_features(w) for w in test_words])
    predictions = model.predict(test_features)

    # Format results
    results = {}
    for word, label in zip(test_words, predictions):
        results[word] = "Complex" if label == 1 else "Simple"
        
    return results

# ... keep your existing imports and functions above ...

def show_example_code():
    """Prints the source code for a sentiment analysis demo."""
    code_to_print = """
!pip install textblob nltk spacy
!python -m spacy download en_core_web_sm

from textblob import TextBlob

sentences = [
    "The movie was fantastic",
    "The plot was boring", 
    "The music was average",
    "I would not recommend this film",
    "The acting was excellent"]

polarities = []

for sentence in sentences:
    blob = TextBlob(sentence)
    polarity = blob.sentiment.polarity
    polarities.append(polarity)

    if polarity > 0:
        sentiment = "Positive"
    elif polarity < 0:
        sentiment = "Negative"
    else:
        sentiment = "Neutral"

    print(sentence)
    print("Polarity: ", polarity)
    print("Sentiment: ", sentiment)

    Labsheet 1
    import nltk
nltk.download('punkt')#punkt is used for tokenization
nltk.download('punkt_tab')#punkt_tab is used for tab tokenization
from nltk.tokenize import word_tokenize, sent_tokenize#tokenization is the process of breaking down text into smaller units like words or sentences
text = "Hello there! How are you doing today? Let's tokenize this text."#sample text for tokenization

print("Word Tokenization:", word_tokenize(text))#word_tokenize function breaks the text into individual words
print("Sentence Tokenization:", sent_tokenize(text))#sent_tokenize function breaks the text into sentences

nltk.download('stopwords')#stopwords are common words that are usually filtered out in text processing
from nltk.corpus import stopwords#importing stopwords from nltk corpus
stop_words = set(stopwords.words('english'))#defining a set of English stopwords
filtered_words = [word for word in word_tokenize(text) if word.lower() not in stop_words]#filtering out stopwords from the tokenized words
print("Filtered Words:", filtered_words)#printing the filtered words

#wordnet download: wordnet is a lexical database whihc is used to find the base word form
nltk.download('wordnet')#downloading wordnet
from nltk.stem import WordNetLemmatizer#importing WordNetLemmatizer for lemmatization
lemmatizer = WordNetLemmatizer() #creating an instance of WordNetLemmatizer
#lemmatization is the process of reducing words to their base or root form   
lemmatized_words = [lemmatizer.lemmatize(word) for word in filtered_words]#lemmatizing the filtered words
print("Lemmatized Words:", lemmatized_words)#printing the lemmatized words

#POS tagging
nltk.download('averaged_perceptron_tagger_eng')#downloading POS tagger
from nltk import pos_tag #part of speech tagging
tokens = word_tokenize("Ronaldo is the Greatest Of all time")#tokenizing a sample sentence
print(pos_tag(tokens))#printing the POS tags for the tokens

'''import nltk
nltk.download('tagsets_json')
nltk.help.upenn_tagset()'''

#NLP pipeline in spacy
import spacy#importing spacy library
nlp = spacy.load("en_core_web_sm")#loading the small English model in spacy
doc = nlp("Apple is looking at buying U.K. startup for $1 billion")#processing a sample text through the NLP pipeline
for token in doc:#iterating through the tokens in the processed document
    print(token.text,  token.pos_)#printing the token text and its part of speech

#NER in spacy
for ent in doc.ents:#iterating through the named entities in the processed document
    print(ent.text,"->", ent.label_)#printing the entity text and its label

    

Labsheet2:
import nltk
from nltk.corpus import wordnet as wn
from gensim.models import Word2Vec
sentences = [["cat", "say", "meow"], ["dog", "say", "woof"], ["natural," "language", "processing"]]
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)
vector = model.wv['cat']
print(vector)

similarity = model.wv.similarity('cat', 'dog')
print(f"Similarity between 'cat' and 'dog': {similarity}")

import gensim.downloader as api
word_vectors = api.load("glove-wiki-gigaword-50")
print(word_vectors['computer'])
print(word_vectors.similarity('computer', 'laptop'))


import gensim.downloader as api
word_vectors = api.load("glove-wiki-gigaword-50")
print(word_vectors['computer'])
print(word_vectors.similarity('computer', 'laptop'))

from transformers import BertTokenizer, BertModel
import torch
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

sentence = "The Bank is near the river."
inputs = tokenizer(sentence, return_tensors='pt')
outputs = model(**inputs)
embeddings = outputs.last_hidden_state
print(embeddings)

from transformers import RobertaTokenizer, RobertaModel

tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
model = RobertaModel.from_pretrained("roberta-base")
inputs = tokenizer("Word representations are powerful", return_tensors = "pt")

outputs = model(**inputs)

print(outputs.last_hidden_state.shape)


Labsheet 3;
import nltk
from nltk.corpus import brown
from collections import Counter as counter
nltk.download('brown')
words = brown.words()
freq_dist = counter(word.lower() for word in words)

def is_complex(word, threshold=50):
    return freq_dist[word.lower()] < threshold

sentence = "The quick brown fox jumps over the lazy dog"
for word in sentence.split():
    if is_complex(word):
        print(f"Complex word found: {word}")
    else:
        print(f"Simple word: {word}")
        


import pandas as pd
from sklearn.tree import DecisionTreeClassifier

def extract_features(word):
    return {
        "length": len(word),
        "frequency": freq_dist.get(word.lower(), 0)
    }


data = [
    ("dog", 0), 
    ("cat", 0), 
    ("gay", 1),
    ("photosynthesis", 1),
    ("think", 0),
    ("bitch", 1)
]

df = pd.DataFrame(data, columns=["word", "label"])

features = df["word"].apply(extract_features)
X = pd.DataFrame(features.tolist())
y = df["label"]

model = DecisionTreeClassifier()
model.fit(X,y)

test_words = ["flow", "stitch"]

test_features = pd.DataFrame([extract_features(w) for w in test_words])
predictions = model.predict(test_features)

for word, label in zip(test_words, predictions):
    print(word, "->", "Complex" if label == 1 else "Simple")


Labsheet 4:

import nltk
from nltk.corpus import wordnet as wn
import spacy
import gensim.downloader as api
from sklearn.metrics.pairwise import cosine_similarity
nltk.download('wordnet')
vectors = api.load("glove-wiki-gigaword-50")# 50-dimensional GloVe embeddings
def simplify(sentence):
    words = sentence.split()
    out = []
    for word in words:
        best = word
        best_sim  = 0
        if word in vectors and len(word) > 3:
            for syn in wn.synsets(word):
                for lemma in syn.lemmas():
                    s = lemma.name().replace('_', ' ')
                    if lemma.name() in vectors:
                        sim = cosine_similarity([vectors[word]], [vectors[lemma.name()]])
                        if sim > best_sim:
                            best_sim = sim
                            best = lemma.name()
        out.append(best)
    
    return ' '.join(out)import nltk
    """
    print(code_to_print)


def show_example_code_rl():
    """Prints the source code for a sentiment analysis demo."""
    code_to_print = """

Labsheet 1
import gymnasium as gym
import time

env = gym.make("FrozenLake-v1", render_mode="human")
env.reset()
env.render()

print("no.of states:", env.observation_space)
print("no.of actions:", env.action_space)
print("Trans_prob:", env.unwrapped.P[0][2])

env.step(1)
env.render()

a = env.action_space.sample()
env.step(a)
env.render()

env.reset()
ret = 0
num_steps = 10
for i in range(10):
    rnd = env.action_space.sample()
    (ns, r, done, tp, info) = env.step(rnd)
    env.render()
    ret = ret + r
    if done:
        print("return of this episode is", ret)
        break

num_epsds = 20
for n in range(num_epsds):
    env.reset()
    ret = 0
    num_steps = 10
    for i in range(10):
        rnd = env.action_space.sample()
        (ns, r, done, tp, info) = env.step(rnd)
        time.sleep(2)
        env.render()
        ret = ret + r
        if done:
            print("return of this episode{0} is{1}".format(n, ret))
            break

num_epsds = 20
for n in range(num_epsds):
    env.reset()
    ret = 0
    num_steps = 10
    policy = []

    for i in range(num_steps):
        rnd = env.action_space.sample()
        if rnd == 0:
            policy.append('Left')
        elif rnd == 1:
            policy.append('Down')
        elif rnd == 2:
            policy.append('Right')
        else:
            policy.append('Up')

    for i in range(num_steps):
        rnd = env.action_space.sample()
        ns, r, done, tp, info = env.step(rnd)
        time.sleep(2)
        env.render()
        ret += r
        if done:
            print("Episode #{}\tPolicy: {}\tReturn: {}".format(n, policy, ret))
            break
-------------------------------------------------------------------------------------------------------------------
Labsheet 2
import gymnasium as gym
import time

# 2.MAKE THE CARTPOLE ENVIRONMENT
env = gym.make("CartPole-v1", render_mode="human")
env.reset()
env.render()
print(env.observation_space)
print(env.action_space)
env.reset()

# 0 to left and 1 to right
# 5.GENERATE AN EPISODE FOR 50 TIME STEP USING A RANDOM POLICY AND PRINT THE RETURN OF THIS EPISODE
env.step(1)
env.render()
Return = 0
(next_state, reward, done, info, trans_prob) = env.step(1)
Return += reward
print("Reward is", reward)
rnd_action = env.action_space.sample()
(next_state, reward, done, info, trans_prob) = env.step(rnd_action)

# 6.GENERATE 50 SUCH EPISODE AND PRINT THE RETURN AFTER EVERY 10TH EPISODE
n_episode = 50
n_timestep = 50
for m in range(n_episode):
    env.reset()
    for i in range(n_timestep):
        env.render()
        rnd_action = env.action_space.sample()
        next_state, reward, done, trans_prob, info = env.step(rnd_action)
        Return += reward
        if done:
            env.reset()
            break
    if m % 10 == 0:
        print("Episode:{},Return:{}".format(m + 1, Return))
env.close()


-------------------------------------------------------------------------------------------------------------------
Labsheet 3

import gymnasium as gym
import numpy as np


def value_iteration(env):
    num_itns = 1000
    threshold = 1e-20
    gamma = 1.0

    value_table = np.zeros(env.observation_space.n)

    for _ in range(num_itns):
        updated_value_table = np.copy(value_table)

        for s in range(env.observation_space.n):
            Q_values = []

            for a in range(env.action_space.n):
                q = sum(
                    prob * (r + gamma * updated_value_table[s_])
                    for prob, s_, r, _ in env.P[s][a]
                )
                Q_values.append(q)

            value_table[s] = max(Q_values)

        # convergence check
        if np.sum(np.abs(updated_value_table - value_table)) <= threshold:
            break

    return value_table


def extract_policy(env, value_table):
    gamma = 1.0
    policy = np.zeros(env.observation_space.n, dtype=int)

    for s in range(env.observation_space.n):
        Q_values = []

        for a in range(env.action_space.n):
            q = sum(
                prob * (r + gamma * value_table[s_])
                for prob, s_, r, _ in env.P[s][a]
            )
            Q_values.append(q)

        policy[s] = np.argmax(Q_values)

    return policy


# ---------- MAIN PROGRAM ----------

env = gym.make("FrozenLake-v1", render_mode="human")
env.reset()
env.render()

# unwrap to access transition matrix P
env = env.unwrapped

# compute optimal values
optimal_value_function = value_iteration(env)

# compute optimal policy
optimal_policy = extract_policy(env, optimal_value_function)

print("Optimal Policy:")
print(optimal_policy)

# run episode using optimal policy
state, _ = env.reset()
done = False

while not done:
    action = int(optimal_policy[state])
    state, reward, terminated, truncated, info = env.step(action)
    env.render()
    done = terminated or truncated

print("Episode finished")

Labsheet 4:
import gymnasium as gym 
import numpy as np 

# Initialize the environment first so functions can access it
env = gym.make('FrozenLake-v1', render_mode='human') 
env = env.unwrapped 

def compute_value_function(policy): 
    num_iterations = 1000
    threshold = 1e-20
    gamma = 1.0
    value_table = np.zeros(env.observation_space.n)
    
    for i in range(num_iterations): 
        updated_value_table = np.copy(value_table)
        for s in range(env.observation_space.n):
            a = int(policy[s])
            # Fixed the Bellman equation (multiplication for gamma)
            value_table[s] = sum([prob * (r + gamma * updated_value_table[s_])
                                  for prob, s_, r, _ in env.P[s][a]])
            
        # Fixed indentation: check for convergence after updating all states
        if(np.sum(np.fabs(updated_value_table - value_table)) <= threshold): 
            break
            
    return value_table

def extract_policy(value_table):
    gamma = 1.0
    policy = np.zeros(env.observation_space.n)
    for s in range(env.observation_space.n):
        Q_values = [sum([prob * (r + gamma * value_table[s_])
                         for prob, s_, r, _ in env.P[s][a]])
                    for a in range(env.action_space.n)]
        policy[s] = np.argmax(np.array(Q_values))
    return policy

def policy_iteration(env): 
    num_iterations = 1000 
    policy = np.zeros(env.observation_space.n) 
    for i in range(num_iterations): 
        value_function = compute_value_function(policy) 
        new_policy = extract_policy(value_function) 
         
        if (np.all(policy == new_policy)): 
            break 
        policy = new_policy 
    return policy 

# --- Main Execution ---
state, _ = env.reset() # Proper unpacking
env.render()  

optimal_policy = policy_iteration(env) 
print("Optimal Policy:", optimal_policy) 
 
# LEVEL 2: RUN A GAME USING THIS OPTIMAL POLICY 
state, _ = env.reset() 
s = state
tot_reward = 0 

while True:  
    # Select the action according to the policy 
    next_state, reward, terminated, truncated, _ = env.step(int(optimal_policy[s])) 
    env.render() 
    s = next_state 
    tot_reward += reward  
    
    if terminated or truncated: 
        break 
        
print("Return =", tot_reward)
env.close()

Labsheet 6:
import pandas as pd
import numpy as np
from collections import defaultdict
import gymnasium as gym

env = gym.make('Blackjack-v1', render_mode='human')
state, _ = env.reset()
env.render()

def policy(state):
    # state[0] is the player's current hand sum
    return 0 if state[0] > 19 else 1

print("Initial state:", state)
print("Policy action for initial state:", policy(state))

num_timesteps = 100

def generate_episode(policy):
    episode = []
    state, _ = env.reset() # Proper unpacking
    
    for t in range(num_timesteps):
        action = policy(state)
        # Proper 5-value unpacking for Gymnasium
        next_state, reward, terminated, truncated, info = env.step(action)
        episode.append((state, action, reward))
        
        if terminated or truncated: 
            break
            
        state = next_state
        
    return episode

print("Sample Episode:", generate_episode(policy))

total_return = defaultdict(float)
N = defaultdict(int)
num_iterations = 10

# Run Monte Carlo simulation
for i in range(num_iterations):
    episode = generate_episode(policy)
    states, actions, rewards = zip(*episode)
    
    for t, state in enumerate(states):
        # First-visit MC check
        if state not in states[0:t]:
            R = sum(rewards[t:])
            total_return[state] += R
            N[state] += 1

# Process results into DataFrames
total_return_df = pd.DataFrame(
    total_return.items(), columns=['state', 'total_return']
)

N_df = pd.DataFrame(
    N.items(), 
    columns=['state', 'N']
)

# Merge and calculate state values
df = pd.merge(total_return_df, N_df, on='state')
df['value'] = df['total_return'] / df['N']

print("\nValue Table (First 10 rows):")
print(df.head(10))
print("\nShape of DataFrame:", df.shape)

# Query a specific state (using 0 instead of False for usable_ace)
state_query = df[df['state'] == (5, 8, 0)]['value'].values
if len(state_query) > 0:
    print("\nValue of state (5, 8, 0):", state_query)
else:
    print("\nState (5, 8, 0) was not visited in these 10 iterations.")

env.close() # Added missing parentheses

exp5
import pandas as pd
from collections import defaultdict
import gymnasium as gym
env=gym.make('Blackjack-v1',render_mode='human')
env.reset()
env.render()
def policy(state):
    return 0 if state[0]>19 else 1
state = env.reset()
state=state[0]
print(state)
print(policy(state))
num_timesteps = 100
def generate_episode(policy):
    episode = []
    state = env.reset()
    state=state[0]
    for t in range(num_timesteps):
        action = policy(state)
        next_state, reward, done, info,trans_prob = env.step(action)
        episode.append((state, action, reward))
        if done:
            break
        state = next_state
    return episode

print(generate_episode(policy))
total_return = defaultdict(float)
N = defaultdict(int)
num_iterations = 10
for i in range(num_iterations):
    episode = generate_episode(policy)
    states, actions, rewards = zip(*episode)

    for t, state in enumerate(states):
        R = sum(rewards[t:])
        total_return[state] += R
        N[state] += 1
# print last state stats
print(total_return[state])
print(N[state])
total_return = pd.DataFrame(total_return.items(),
                            columns=['state', 'total_return'])
N = pd.DataFrame(N.items(),
                 columns=['state', 'N'])
df = pd.merge(total_return, N, on="state")
print(df.head(10))
df['value'] = df['total_return'] / df['N']
print(df.head(10)) 
print(df.shape)
print(df[df['state'] == (21, 9, False)]['value'].values)
print(df[df['state'] == (5, 8, False)]['value'].values)
env.close()



    """
    print(code_to_print)


def show_example_code():
    """Prints the source code for a sentiment analysis demo."""
    code_to_print = """
!pip install textblob nltk spacy
!python -m spacy download en_core_web_sm

from textblob import TextBlob

sentences = [
    "The movie was fantastic",
    "The plot was boring", 
    "The music was average",
    "I would not recommend this film",
    "The acting was excellent"]

polarities = []

for sentence in sentences:
    blob = TextBlob(sentence)
    polarity = blob.sentiment.polarity
    polarities.append(polarity)

    if polarity > 0:
        sentiment = "Positive"
    elif polarity < 0:
        sentiment = "Negative"
    else:
        sentiment = "Neutral"

    print(sentence)
    print("Polarity: ", polarity)
    print("Sentiment: ", sentiment)

    Labsheet 1
    import nltk
nltk.download('punkt')#punkt is used for tokenization
nltk.download('punkt_tab')#punkt_tab is used for tab tokenization
from nltk.tokenize import word_tokenize, sent_tokenize#tokenization is the process of breaking down text into smaller units like words or sentences
text = "Hello there! How are you doing today? Let's tokenize this text."#sample text for tokenization

print("Word Tokenization:", word_tokenize(text))#word_tokenize function breaks the text into individual words
print("Sentence Tokenization:", sent_tokenize(text))#sent_tokenize function breaks the text into sentences

nltk.download('stopwords')#stopwords are common words that are usually filtered out in text processing
from nltk.corpus import stopwords#importing stopwords from nltk corpus
stop_words = set(stopwords.words('english'))#defining a set of English stopwords
filtered_words = [word for word in word_tokenize(text) if word.lower() not in stop_words]#filtering out stopwords from the tokenized words
print("Filtered Words:", filtered_words)#printing the filtered words

#wordnet download: wordnet is a lexical database whihc is used to find the base word form
nltk.download('wordnet')#downloading wordnet
from nltk.stem import WordNetLemmatizer#importing WordNetLemmatizer for lemmatization
lemmatizer = WordNetLemmatizer() #creating an instance of WordNetLemmatizer
#lemmatization is the process of reducing words to their base or root form   
lemmatized_words = [lemmatizer.lemmatize(word) for word in filtered_words]#lemmatizing the filtered words
print("Lemmatized Words:", lemmatized_words)#printing the lemmatized words

#POS tagging
nltk.download('averaged_perceptron_tagger_eng')#downloading POS tagger
from nltk import pos_tag #part of speech tagging
tokens = word_tokenize("Ronaldo is the Greatest Of all time")#tokenizing a sample sentence
print(pos_tag(tokens))#printing the POS tags for the tokens

'''import nltk
nltk.download('tagsets_json')
nltk.help.upenn_tagset()'''

#NLP pipeline in spacy
import spacy#importing spacy library
nlp = spacy.load("en_core_web_sm")#loading the small English model in spacy
doc = nlp("Apple is looking at buying U.K. startup for $1 billion")#processing a sample text through the NLP pipeline
for token in doc:#iterating through the tokens in the processed document
    print(token.text,  token.pos_)#printing the token text and its part of speech

#NER in spacy
for ent in doc.ents:#iterating through the named entities in the processed document
    print(ent.text,"->", ent.label_)#printing the entity text and its label

    

Labsheet2:
import nltk
from nltk.corpus import wordnet as wn
from gensim.models import Word2Vec
sentences = [["cat", "say", "meow"], ["dog", "say", "woof"], ["natural," "language", "processing"]]
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)
vector = model.wv['cat']
print(vector)

similarity = model.wv.similarity('cat', 'dog')
print(f"Similarity between 'cat' and 'dog': {similarity}")

import gensim.downloader as api
word_vectors = api.load("glove-wiki-gigaword-50")
print(word_vectors['computer'])
print(word_vectors.similarity('computer', 'laptop'))


import gensim.downloader as api
word_vectors = api.load("glove-wiki-gigaword-50")
print(word_vectors['computer'])
print(word_vectors.similarity('computer', 'laptop'))

from transformers import BertTokenizer, BertModel
import torch
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

sentence = "The Bank is near the river."
inputs = tokenizer(sentence, return_tensors='pt')
outputs = model(**inputs)
embeddings = outputs.last_hidden_state
print(embeddings)

from transformers import RobertaTokenizer, RobertaModel

tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
model = RobertaModel.from_pretrained("roberta-base")
inputs = tokenizer("Word representations are powerful", return_tensors = "pt")

outputs = model(**inputs)

print(outputs.last_hidden_state.shape)


Labsheet 3;
import nltk
from nltk.corpus import brown
from collections import Counter as counter
nltk.download('brown')
words = brown.words()
freq_dist = counter(word.lower() for word in words)

def is_complex(word, threshold=50):
    return freq_dist[word.lower()] < threshold

sentence = "The quick brown fox jumps over the lazy dog"
for word in sentence.split():
    if is_complex(word):
        print(f"Complex word found: {word}")
    else:
        print(f"Simple word: {word}")
        


import pandas as pd
from sklearn.tree import DecisionTreeClassifier

def extract_features(word):
    return {
        "length": len(word),
        "frequency": freq_dist.get(word.lower(), 0)
    }


data = [
    ("dog", 0), 
    ("cat", 0), 
    ("gay", 1),
    ("photosynthesis", 1),
    ("think", 0),
    ("bitch", 1)
]

df = pd.DataFrame(data, columns=["word", "label"])

features = df["word"].apply(extract_features)
X = pd.DataFrame(features.tolist())
y = df["label"]

model = DecisionTreeClassifier()
model.fit(X,y)

test_words = ["flow", "stitch"]

test_features = pd.DataFrame([extract_features(w) for w in test_words])
predictions = model.predict(test_features)

for word, label in zip(test_words, predictions):
    print(word, "->", "Complex" if label == 1 else "Simple")


Labsheet 4:

import nltk
from nltk.corpus import wordnet as wn
import spacy
import gensim.downloader as api
from sklearn.metrics.pairwise import cosine_similarity
nltk.download('wordnet')
vectors = api.load("glove-wiki-gigaword-50")# 50-dimensional GloVe embeddings
def simplify(sentence):
    words = sentence.split()
    out = []
    for word in words:
        best = word
        best_sim  = 0
        if word in vectors and len(word) > 3:
            for syn in wn.synsets(word):
                for lemma in syn.lemmas():
                    s = lemma.name().replace('_', ' ')
                    if lemma.name() in vectors:
                        sim = cosine_similarity([vectors[word]], [vectors[lemma.name()]])
                        if sim > best_sim:
                            best_sim = sim
                            best = lemma.name()
        out.append(best)
    
    return ' '.join(out)import nltk
    """
    print(code_to_print)


def show_example_code_rl_1():
    """Prints the source code for a sentiment analysis demo."""
    code_to_print = """
RL Lab Mid-Term Examinations - Solutions

Part A - First Question Paper (Image ending in .45)

1. Implement the reinforcement learning environment namely, Frozen Lake Environment using a random policy...

import gymnasium as gym

env = gym.make("FrozenLake-v1", render_mode="human")
env.reset()
env.render()

print(env.action_space)

print(env.observation_space)

random_action = env.action_space.sample()
next_state, reward, done, trans_prob, info = env.step(random_action)
env.render()

num_steps = 20
total_reward = 0

env.reset()
for k in range(num_steps):
    random_action = env.action_space.sample()
    next_state, reward, done, trans_prob, info = env.step(random_action)
    total_reward += reward
    env.render()
    if done:
        break

print(f"Episode return = {total_reward}")


2. Implement Policy Iteration for a simple RL environment and display the optimal policy.

import gymnasium as gym
import numpy as np

def compute_value_function(policy):
    num_iterations = 1000
    threshold = 1e-20
    gamma = 1.0
    value_table = np.zeros(env.observation_space.n)
    for i in range(num_iterations):
        updated_value_table = np.copy(value_table)
        for s in range(env.observation_space.n):
            a = policy[s]
            value_table[s] = sum([prob * (r + gamma * updated_value_table[s_])
                                  for prob, s_, r, _ in env.P[s][a]])
        if (np.sum(np.fabs(updated_value_table - value_table)) <= threshold):
            break
    return value_table

def extract_policy(value_table):
    gamma = 1.0
    policy = np.zeros(env.observation_space.n)
    for s in range(env.observation_space.n):
        Q_values = [sum([prob * (r + gamma * value_table[s_])
                         for prob, s_, r, _ in env.P[s][a]])
                    for a in range(env.action_space.n)]
        policy[s] = np.argmax(np.array(Q_values))
    return policy

def policy_iteration(env):
    num_iterations = 1000
    policy = np.zeros(env.observation_space.n)
    for i in range(num_iterations):
        value_function = compute_value_function(policy)
        new_policy = extract_policy(value_function)
        if (np.all(policy == new_policy)):
            break
        policy = new_policy
    return policy

env = gym.make('FrozenLake-v1', render_mode='human')
env.reset()
env.render()
env = env.unwrapped

optimal_policy = policy_iteration(env)
print(optimal_policy)


Explanation of the Results: The code iterates between evaluating a given policy and improving it. It first computes the value function for an arbitrary initial policy using the Bellman expectation equation. Then, it acts greedily upon this value function to extract a better policy. This loop repeats until the policy stabilizes (no changes occur between iterations). The resulting optimal_policy array shows the best action to take (0: Left, 1: Down, 2: Right, 3: Up) for each of the 16 states in the FrozenLake grid to safely reach the goal.

3. Implementation of First-visit Monte Carlo Prediction for Blackjack environment.

import pandas as pd
from collections import defaultdict
import gymnasium as gym

env = gym.make('Blackjack-v1', render_mode='human')

def policy(state):
    return 0 if state[0] > 19 else 1

num_timesteps = 100

def generate_episode(policy):
    episode = []
    state = env.reset()
    state = state[0]
    for t in range(num_timesteps):
        action = policy(state)
        next_state, reward, done, info, trans_prob = env.step(action)
        episode.append((state, action, reward))
        if done:
            break
        state = next_state
    return episode

total_return = defaultdict(float)
N = defaultdict(int)
num_iterations = 10

for i in range(num_iterations):
    episode = generate_episode(policy)
    states, actions, rewards = zip(*episode)
    for t, state in enumerate(states):
        if state not in states[0:t]:
            R = sum(rewards[t:])
            total_return[state] += R
            N[state] += 1

total_return_df = pd.DataFrame(total_return.items(), columns=['state', 'total_return'])
N_df = pd.DataFrame(N.items(), columns=['state', 'N'])

df = pd.merge(total_return_df, N_df, on='state')
df['value'] = df['total_return'] / df['N']

print(df.head(10))
env.close()


Part A - Second Question Paper (Image ending in .46)

1. Implementation of Every-visit Monte Carlo Prediction for Blackjack environment.

import pandas as pd
from collections import defaultdict
import gymnasium as gym

env = gym.make('Blackjack-v1', render_mode='human')

def policy(state):
    return 0 if state[0] > 19 else 1

num_timesteps = 100

def generate_episode(policy):
    episode = []
    state = env.reset()
    state = state[0]
    for t in range(num_timesteps):
        action = policy(state)
        next_state, reward, done, info, trans_prob = env.step(action)
        episode.append((state, action, reward))
        if done:
            break
        state = next_state
    return episode

total_return = defaultdict(float)
N = defaultdict(int)
num_iterations = 10

for i in range(num_iterations):
    episode = generate_episode(policy)
    states, actions, rewards = zip(*episode)
    for t, state in enumerate(states):
        R = sum(rewards[t:])
        total_return[state] += R
        N[state] += 1

total_return_df = pd.DataFrame(total_return.items(), columns=['state', 'total_return'])
N_df = pd.DataFrame(N.items(), columns=['state', 'N'])

df = pd.merge(total_return_df, N_df, on='state')
df['value'] = df['total_return'] / df['N']

print(df.head(10))
env.close()


2. Implement the reinforcement learning environment namely, Cartpole Environment using a random policy...

import gymnasium as gym

env = gym.make("CartPole-v1", render_mode="human")
env.reset()
env.render()

print(env.action_space)

print(env.observation_space)

rnd_action = env.action_space.sample()
next_state, reward, done, info, trans_prob = env.step(rnd_action)

Return = 0
env.reset()
n_timesteps = 20

for j in range(1, n_timesteps + 1):
    env.render()
    rnd_action = env.action_space.sample()
    next_state, reward, done, info, trans_prob = env.step(rnd_action)
    Return += reward
    if done:
        env.reset()
        break

print(f"Return : {Return}")
env.close()


3. Implement Value Iteration for a simple RL environment and display the optimal policy. Run a game...

import gymnasium as gym
import numpy as np
import time

def value_iteration(env):
    num_iterations = 1000
    threshold = 1e-20
    gamma = 1
    value_table = np.zeros(env.observation_space.n)
    for i in range(num_iterations):
        updated_value_table = np.copy(value_table)
        for s in range(env.observation_space.n):
            Q_values = [sum([prob * (r + gamma * updated_value_table[s_])
                             for prob, s_, r, _ in env.P[s][a]])
                        for a in range(env.action_space.n)]
            value_table[s] = max(Q_values)
        if (np.sum(np.fabs(updated_value_table - value_table)) <= threshold):
            break
    return value_table

def extract_policy(value_table):
    gamma = 1.0
    policy = np.zeros(env.observation_space.n)
    for s in range(env.observation_space.n):
        Q_values = [sum([prob * (r + gamma * value_table[s_])
                         for prob, s_, r, _ in env.P[s][a]])
                    for a in range(env.action_space.n)]
        policy[s] = np.argmax(np.array(Q_values))
    return policy

env = gym.make("FrozenLake-v1", render_mode="human")
env.reset()
env.render()
env = env.unwrapped

optimal_value_function = value_iteration(env)
optimal_policy = extract_policy(optimal_value_function)
print(optimal_policy)

env.reset()
ret = 0
s = 0

for i in range(50):
    act = int(optimal_policy[s])
    ns, r, done, tp, info = env.step(act)
    ret = ret + r
    env.render()
    time.sleep(1)
    s = ns
    if done:
        break

print(ret)

#Labsheet 7
# Implementation of On-Policy Monte Carlo Control

import gym
import pandas as pd
import random

from collections import defaultdict

# Create Environment
env = gym.make('Blackjack-v1', render_mode='human')

# Initialize Q-table
Q = defaultdict(float)

total_return = defaultdict(float)

N = defaultdict(int)

# Epsilon-Greedy Policy
def epsilon_greedy_policy(state, Q):

    epsilon = 0.5

    if random.uniform(0,1) < epsilon:

        return env.action_space.sample()

    else:

        return max(

            list(range(env.action_space.n)),

            key=lambda x: Q[(state,x)]

        )

# Number of timesteps
num_timesteps = 100

# Generate Episode
def generate_episode(Q):

    episode = []

    state = env.reset()
    state = state[0]

    for t in range(num_timesteps):

        action = epsilon_greedy_policy(state, Q)

        next_state, reward, done, info, trans_prob = env.step(action)

        episode.append((state, action, reward))

        if done:
            break

        state = next_state

    return episode

# Training
num_iterations = 100

for i in range(num_iterations):

    episode = generate_episode(Q)

    all_state_action_pairs = [

        (s,a)

        for (s,a,r) in episode
    ]

    rewards = [

        r

        for (s,a,r) in episode
    ]

    for t, (state, action, _) in enumerate(episode):

        if not (state, action) in all_state_action_pairs[0:t]:

            R = sum(rewards[t:])

            total_return[(state,action)] += R

            N[(state,action)] += 1

            Q[(state,action)] = (

                total_return[(state,action)]

                /

                N[(state,action)]

            )

# Display Q-table
df = pd.DataFrame(

    Q.items(),

    columns=['state_action pair', 'value']

)

print(df.head(11))

print(df[124:126])

env.close()


#Labsheet 8
# TD Prediction

import gymnasium as gym
import pandas as pd

# Create Environment
env = gym.make("FrozenLake-v1", render_mode="human")

env.reset()
env.render()

# Random Policy
def random_policy():

    return env.action_space.sample()

# Initialize V-table
V = {}

for s in range(env.observation_space.n):

    V[s] = 0.0

# Hyperparameters
alpha = 0.85
gamma = 0.90

num_eps = 50
num_steps = 10

# Training Loop
for i in range(num_eps):

    s = env.reset()
    s = s[0]

    for t in range(num_steps):

        a = random_policy()

        s_, r, done, _, _ = env.step(a)

        # TD Update
        V[s] += alpha * (

            r + gamma * V[s_] - V[s]

        )

        s = s_

        if done:
            break

# Display Values
df = pd.DataFrame(

    list(V.items()),

    columns=['state', 'value']

)

print(df)

#LABSHEET 9
# SARSA Algorithm

import gymnasium as gym
import random
import pandas as pd

# Create Environment
env = gym.make("FrozenLake-v1", render_mode="human")

env.reset()
env.render()

# Initialize Q-table
Q = {}

for s in range(env.observation_space.n):

    for a in range(env.action_space.n):

        Q[(s,a)] = 0.0

print("Initial Q Table:", Q)

# Epsilon-Greedy Policy
def epsilon_greedy(state, epsilon):

    if random.uniform(0,1) < epsilon:

        return env.action_space.sample()

    else:

        return max(

            list(range(env.action_space.n)),

            key=lambda x: Q[(state,x)]

        )

# Hyperparameters
alpha = 0.85
gamma = 0.90
epsilon = 0.8

num_eps = 500
num_steps = 50

# Training Loop
for i in range(num_eps):

    s = env.reset()
    s = s[0]

    a = epsilon_greedy(s, epsilon)

    for t in range(num_steps):

        s_, r, done, _, _ = env.step(a)

        a_ = epsilon_greedy(s_, epsilon)

        predict = Q[(s,a)]

        target = r + gamma * Q[(s_,a_)]

        Q[(s,a)] = Q[(s,a)] + alpha * (

            target - predict

        )

        s = s_
        a = a_

        if done:
            break

# Display Q-table
df = pd.DataFrame(

    list(Q.items()),

    columns=['state_action', 'value']

)

print(df)


$LABSHEET 10
# Q-Learning Algorithm

import gymnasium as gym
import numpy as np
import random
import pandas as pd

# Create Environment
env = gym.make("FrozenLake-v1", render_mode="human")

# Initialize Q-table
Q = {}

for s in range(env.observation_space.n):

    for a in range(env.action_space.n):

        Q[(s,a)] = 0.0

# Epsilon-Greedy Policy
def epsilon_greedy(state, epsilon):

    if random.uniform(0,1) < epsilon:

        return env.action_space.sample()

    else:

        return max(

            list(range(env.action_space.n)),

            key=lambda x: Q[(state,x)]

        )

# Hyperparameters
alpha = 0.85
gamma = 0.90
epsilon = 0.8

num_eps = 50
num_steps = 100

# Training Loop
for i in range(num_eps):

    s = env.reset()
    s = s[0]

    for t in range(num_steps):

        a = epsilon_greedy(s, epsilon)

        s_, r, done, _, _ = env.step(a)

        a_ = np.argmax(

            [Q[(s_,a)] for a in range(env.action_space.n)]

        )

        Q[(s,a)] += alpha * (

            r + gamma * Q[(s_,a_)] - Q[(s,a)]

        )

        s = s_

        if done:
            break

# Display Q-table
df = pd.DataFrame(

    list(Q.items()),

    columns=['state_action', 'value']

)

print(df)


#LABSHEET 11
# Epsilon-Greedy Multi-Armed Bandit

import gym
import gym_bandits
import numpy as np

# Create Environment
env = gym.make("BanditTwoArmedHighLowFixed-v0")

print(env.action_space.n)
print(env.p_dist)

# Initialize Arrays
count = np.zeros(2)

sum_rewards = np.zeros(2)

Q = np.zeros(2)

num_rounds = 100

# Epsilon-Greedy Policy
def epsilon_greedy(epsilon):

    if np.random.uniform(0,1) < epsilon:

        return env.action_space.sample()

    else:

        return np.argmax(Q)

# Training
env.reset()

for i in range(num_rounds):

    arm = epsilon_greedy(epsilon=0.5)

    next_state, reward, done, info = env.step(arm)

    count[arm] += 1

    sum_rewards[arm] += reward

    Q[arm] = sum_rewards[arm] / count[arm]

# Display Results
print(Q)

print(

    'The optimal arm is arm {}'.format(

        np.argmax(Q)+1

    )

)


LABSHEET 12

# Softmax Exploration

import gym
import gym_bandits
import numpy as np

# Create Environment
env = gym.make("BanditTwoArmedHighLowFixed-v0")

# Initialize Arrays
count = np.zeros(2)

sum_rewards = np.zeros(2)

Q = np.zeros(2)

num_rounds = 100

# Softmax Policy
def softmax(T):

    denom = sum(

        [np.exp(i/T) for i in Q]

    )

    probs = [

        np.exp(i/T)/denom

        for i in Q
    ]

    arm = np.random.choice(

        env.action_space.n,

        p=probs

    )

    return arm

# Training
env.reset()

T = 50

for i in range(num_rounds):

    arm = softmax(T)

    next_state, reward, done, info = env.step(arm)

    count[arm] += 1

    sum_rewards[arm] += reward

    Q[arm] = sum_rewards[arm] / count[arm]

    T = T * 0.99

# Display Results
print(Q)

print(

    'The optimal arm is arm {}'.format(

        np.argmax(Q)+1

    )

)


#LABSHEET 13
# Upper Confidence Bound (UCB)

import gym
import gym_bandits
import numpy as np

# Create Environment
env = gym.make("BanditTwoArmedHighLowFixed-v0")

# Initialize Arrays
count = np.zeros(2)

sum_rewards = np.zeros(2)

Q = np.zeros(2)

num_rounds = 100

# UCB Policy
def UCB(i):

    ucb = np.zeros(2)

    if i < 2:

        return i

    else:

        for arm in range(2):

            ucb[arm] = (

                Q[arm]

                +

                np.sqrt(

                    (2*np.log(sum(count)))

                    /

                    count[arm]

                )

            )

    return np.argmax(ucb)

# Training
env.reset()

for i in range(num_rounds):

    arm = UCB(i)

    next_state, reward, done, info = env.step(arm)

    count[arm] += 1

    sum_rewards[arm] += reward

    Q[arm] = sum_rewards[arm] / count[arm]

# Display Results
print(Q)

print(

    'The optimal arm is arm {}'.format(

        np.argmax(Q)+1

    )

)


#LABSHEET 14
# Thompson Sampling

import gym
import gym_bandits
import numpy as np

# Create Environment
env = gym.make("BanditTwoArmedHighLowFixed-v0")

# Initialize Arrays
count = np.zeros(2)

sum_rewards = np.zeros(2)

Q = np.zeros(2)

alpha = np.ones(2)

beta = np.ones(2)

num_rounds = 100

# Thompson Sampling Policy
def thompson_sampling(alpha, beta):

    samples = [

        np.random.beta(

            alpha[i]+1,

            beta[i]+1

        )

        for i in range(2)
    ]

    return np.argmax(samples)

# Training
env.reset()

for i in range(num_rounds):

    arm = thompson_sampling(alpha, beta)

    next_state, reward, done, info = env.step(arm)

    count[arm] += 1

    sum_rewards[arm] += reward

    Q[arm] = sum_rewards[arm] / count[arm]

    if reward == 1:

        alpha[arm] += 1

    else:

        beta[arm] += 1

# Display Results
print(Q)

print(

    'The optimal arm is arm {}'.format(

        np.argmax(Q)+1

    )

)


LABSHEET 15
# Banner Advertisement Selection

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

plt.style.use('ggplot')

# Create Dataset
df = pd.DataFrame()

for i in range(5):

    df['Banner_type_'+str(i)] = np.random.randint(0,2,100000)

# Initialize Variables
num_iterations = 100000

num_banner = 5

count = np.zeros(num_banner)

sum_rewards = np.zeros(num_banner)

Q = np.zeros(num_banner)

banner_selected = []

# Epsilon-Greedy Policy
def epsilon_greedy_policy(epsilon):

    if np.random.uniform(0,1) < epsilon:

        return np.random.choice(num_banner)

    else:

        return np.argmax(Q)

# Training Loop
for i in range(num_iterations):

    banner = epsilon_greedy_policy(0.5)

    reward = df.values[i, banner]

    count[banner] += 1

    sum_rewards[banner] += reward

    Q[banner] = sum_rewards[banner] / count[banner]

    banner_selected.append(banner)

# Display Best Banner
print(

    'The best banner is banner {}'.format(

        np.argmax(Q)+1

    )

)

# Visualization
ax = sns.countplot(x=banner_selected)

ax.set(

    xlabel='Banner',

    ylabel='Count'

)

plt.show()


    """
    print(code_to_print)


def montecarlo():
    """Prints and returns the source code for a Monte Carlo control demo."""
    code_to_print = """
#Labsheet 7
# Implementation of On-Policy Monte Carlo Control

import gym
import pandas as pd
import random

from collections import defaultdict

# Create Environment
env = gym.make('Blackjack-v1', render_mode='human')

# Initialize Q-table
Q = defaultdict(float)

total_return = defaultdict(float)

N = defaultdict(int)

# Epsilon-Greedy Policy
def epsilon_greedy_policy(state, Q):

    epsilon = 0.5

    if random.uniform(0,1) < epsilon:

        return env.action_space.sample()

    else:

        return max(

            list(range(env.action_space.n)),

            key=lambda x: Q[(state,x)]

        )

# Number of timesteps
num_timesteps = 100

# Generate Episode
def generate_episode(Q):

    episode = []

    state = env.reset()
    state = state[0]

    for t in range(num_timesteps):

        action = epsilon_greedy_policy(state, Q)

        next_state, reward, done, info, trans_prob = env.step(action)

        episode.append((state, action, reward))

        if done:
            break

        state = next_state

    return episode

# Training
num_iterations = 100

for i in range(num_iterations):

    episode = generate_episode(Q)

    all_state_action_pairs = [

        (s,a)

        for (s,a,r) in episode
    ]

    rewards = [

        r

        for (s,a,r) in episode
    ]

    for t, (state, action, _) in enumerate(episode):

        if not (state, action) in all_state_action_pairs[0:t]:

            R = sum(rewards[t:])

            total_return[(state,action)] += R

            N[(state,action)] += 1

            Q[(state,action)] = (

                total_return[(state,action)]

                /

                N[(state,action)]

            )

# Display Q-table
df = pd.DataFrame(

    Q.items(),

    columns=['state_action pair', 'value']

)

print(df.head(11))

print(df[124:126])

env.close()


#Labsheet 8
# TD Prediction

import gymnasium as gym
import pandas as pd

# Create Environment
env = gym.make("FrozenLake-v1", render_mode="human")

env.reset()
env.render()

# Random Policy
def random_policy():

    return env.action_space.sample()

# Initialize V-table
V = {}

for s in range(env.observation_space.n):

    V[s] = 0.0

# Hyperparameters
alpha = 0.85
gamma = 0.90

num_eps = 50
num_steps = 10

# Training Loop
for i in range(num_eps):

    s = env.reset()
    s = s[0]

    for t in range(num_steps):

        a = random_policy()

        s_, r, done, _, _ = env.step(a)

        # TD Update
        V[s] += alpha * (

            r + gamma * V[s_] - V[s]

        )

        s = s_

        if done:
            break

# Display Values
df = pd.DataFrame(

    list(V.items()),

    columns=['state', 'value']

)

print(df)

#LABSHEET 9
# SARSA Algorithm

import gymnasium as gym
import random
import pandas as pd

# Create Environment
env = gym.make("FrozenLake-v1", render_mode="human")

env.reset()
env.render()

# Initialize Q-table
Q = {}

for s in range(env.observation_space.n):

    for a in range(env.action_space.n):

        Q[(s,a)] = 0.0

print("Initial Q Table:", Q)

# Epsilon-Greedy Policy
def epsilon_greedy(state, epsilon):

    if random.uniform(0,1) < epsilon:

        return env.action_space.sample()

    else:

        return max(

            list(range(env.action_space.n)),

            key=lambda x: Q[(state,x)]

        )

# Hyperparameters
alpha = 0.85
gamma = 0.90
epsilon = 0.8

num_eps = 500
num_steps = 50

# Training Loop
for i in range(num_eps):

    s = env.reset()
    s = s[0]

    a = epsilon_greedy(s, epsilon)

    for t in range(num_steps):

        s_, r, done, _, _ = env.step(a)

        a_ = epsilon_greedy(s_, epsilon)

        predict = Q[(s,a)]

        target = r + gamma * Q[(s_,a_)]

        Q[(s,a)] = Q[(s,a)] + alpha * (

            target - predict

        )

        s = s_
        a = a_

        if done:
            break

# Display Q-table
df = pd.DataFrame(

    list(Q.items()),

    columns=['state_action', 'value']

)

print(df)


$LABSHEET 10
# Q-Learning Algorithm

import gymnasium as gym
import numpy as np
import random
import pandas as pd

# Create Environment
env = gym.make("FrozenLake-v1", render_mode="human")

# Initialize Q-table
Q = {}

for s in range(env.observation_space.n):

    for a in range(env.action_space.n):

        Q[(s,a)] = 0.0

# Epsilon-Greedy Policy
def epsilon_greedy(state, epsilon):

    if random.uniform(0,1) < epsilon:

        return env.action_space.sample()

    else:

        return max(

            list(range(env.action_space.n)),

            key=lambda x: Q[(state,x)]

        )

# Hyperparameters
alpha = 0.85
gamma = 0.90
epsilon = 0.8

num_eps = 50
num_steps = 100

# Training Loop
for i in range(num_eps):

    s = env.reset()
    s = s[0]

    for t in range(num_steps):

        a = epsilon_greedy(s, epsilon)

        s_, r, done, _, _ = env.step(a)

        a_ = np.argmax(

            [Q[(s_,a)] for a in range(env.action_space.n)]

        )

        Q[(s,a)] += alpha * (

            r + gamma * Q[(s_,a_)] - Q[(s,a)]

        )

        s = s_

        if done:
            break

# Display Q-table
df = pd.DataFrame(

    list(Q.items()),

    columns=['state_action', 'value']

)

print(df)


#LABSHEET 11
# Epsilon-Greedy Multi-Armed Bandit

import gym
import gym_bandits
import numpy as np

# Create Environment
env = gym.make("BanditTwoArmedHighLowFixed-v0")

print(env.action_space.n)
print(env.p_dist)

# Initialize Arrays
count = np.zeros(2)

sum_rewards = np.zeros(2)

Q = np.zeros(2)

num_rounds = 100

# Epsilon-Greedy Policy
def epsilon_greedy(epsilon):

    if np.random.uniform(0,1) < epsilon:

        return env.action_space.sample()

    else:

        return np.argmax(Q)

# Training
env.reset()

for i in range(num_rounds):

    arm = epsilon_greedy(epsilon=0.5)

    next_state, reward, done, info = env.step(arm)

    count[arm] += 1

    sum_rewards[arm] += reward

    Q[arm] = sum_rewards[arm] / count[arm]

# Display Results
print(Q)

print(

    'The optimal arm is arm {}'.format(

        np.argmax(Q)+1

    )

)


LABSHEET 12

# Softmax Exploration

import gym
import gym_bandits
import numpy as np

# Create Environment
env = gym.make("BanditTwoArmedHighLowFixed-v0")

# Initialize Arrays
count = np.zeros(2)

sum_rewards = np.zeros(2)

Q = np.zeros(2)

num_rounds = 100

# Softmax Policy
def softmax(T):

    denom = sum(

        [np.exp(i/T) for i in Q]

    )

    probs = [

        np.exp(i/T)/denom

        for i in Q
    ]

    arm = np.random.choice(

        env.action_space.n,

        p=probs

    )

    return arm

# Training
env.reset()

T = 50

for i in range(num_rounds):

    arm = softmax(T)

    next_state, reward, done, info = env.step(arm)

    count[arm] += 1

    sum_rewards[arm] += reward

    Q[arm] = sum_rewards[arm] / count[arm]

    T = T * 0.99

# Display Results
print(Q)

print(

    'The optimal arm is arm {}'.format(

        np.argmax(Q)+1

    )

)


#LABSHEET 13
# Upper Confidence Bound (UCB)

import gym
import gym_bandits
import numpy as np

# Create Environment
env = gym.make("BanditTwoArmedHighLowFixed-v0")

# Initialize Arrays
count = np.zeros(2)

sum_rewards = np.zeros(2)

Q = np.zeros(2)

num_rounds = 100

# UCB Policy
def UCB(i):

    ucb = np.zeros(2)

    if i < 2:

        return i

    else:

        for arm in range(2):

            ucb[arm] = (

                Q[arm]

                +

                np.sqrt(

                    (2*np.log(sum(count)))

                    /

                    count[arm]

                )

            )

    return np.argmax(ucb)

# Training
env.reset()

for i in range(num_rounds):

    arm = UCB(i)

    next_state, reward, done, info = env.step(arm)

    count[arm] += 1

    sum_rewards[arm] += reward

    Q[arm] = sum_rewards[arm] / count[arm]

# Display Results
print(Q)

print(

    'The optimal arm is arm {}'.format(

        np.argmax(Q)+1

    )

)


#LABSHEET 14
# Thompson Sampling

import gym
import gym_bandits
import numpy as np

# Create Environment
env = gym.make("BanditTwoArmedHighLowFixed-v0")

# Initialize Arrays
count = np.zeros(2)

sum_rewards = np.zeros(2)

Q = np.zeros(2)

alpha = np.ones(2)

beta = np.ones(2)

num_rounds = 100

# Thompson Sampling Policy
def thompson_sampling(alpha, beta):

    samples = [

        np.random.beta(

            alpha[i]+1,

            beta[i]+1

        )

        for i in range(2)
    ]

    return np.argmax(samples)

# Training
env.reset()

for i in range(num_rounds):

    arm = thompson_sampling(alpha, beta)

    next_state, reward, done, info = env.step(arm)

    count[arm] += 1

    sum_rewards[arm] += reward

    Q[arm] = sum_rewards[arm] / count[arm]

    if reward == 1:

        alpha[arm] += 1

    else:

        beta[arm] += 1

# Display Results
print(Q)

print(

    'The optimal arm is arm {}'.format(

        np.argmax(Q)+1

    )

)


LABSHEET 15
# Banner Advertisement Selection

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

plt.style.use('ggplot')

# Create Dataset
df = pd.DataFrame()

for i in range(5):

    df['Banner_type_'+str(i)] = np.random.randint(0,2,100000)

# Initialize Variables
num_iterations = 100000

num_banner = 5

count = np.zeros(num_banner)

sum_rewards = np.zeros(num_banner)

Q = np.zeros(num_banner)

banner_selected = []

# Epsilon-Greedy Policy
def epsilon_greedy_policy(epsilon):

    if np.random.uniform(0,1) < epsilon:

        return np.random.choice(num_banner)

    else:

        return np.argmax(Q)

# Training Loop
for i in range(num_iterations):

    banner = epsilon_greedy_policy(0.5)

    reward = df.values[i, banner]

    count[banner] += 1

    sum_rewards[banner] += reward

    Q[banner] = sum_rewards[banner] / count[banner]

    banner_selected.append(banner)

# Display Best Banner
print(

    'The best banner is banner {}'.format(

        np.argmax(Q)+1

    )

)

# Visualization
ax = sns.countplot(x=banner_selected)

ax.set(

    xlabel='Banner',

    ylabel='Count'

)

plt.show()
    """
    print(code_to_print)
    return code_to_print