import sys
import re

# 🔹 Paste your FULL 70-question content inside this string
ALL_CONTENT = """

Question 1: Decision Tree Classifier for Diabetes Dataset

Python

import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeClassifier, plot_tree
import matplotlib.pyplot as plt

# Load the dataset
df = pd.read_csv('diabetes.csv')

# Separate features and target
X = df.drop('label', axis=1)  # All features except label
y = df['label']

# Create Decision Tree Classifier with max_depth=3
dt_classifier = DecisionTreeClassifier(max_depth=3, random_state=42)
dt_classifier.fit(X, y)

# Plot the tree
plt.figure(figsize=(20, 10))
plot_tree(dt_classifier, 
          feature_names=['pregnant', 'glucose', 'bp', 'skin', 'insulin', 'bmi', 'pedigree', 'age'],
          class_names=['No Diabetes', 'Diabetes'],
          filled=True, 
          rounded=True,
          fontsize=10)
plt.title('Decision Tree Classifier for Diabetes Dataset (Depth=3)')
plt.tight_layout()
plt.savefig('decision_tree_diabetes.png', dpi=300)
plt.show()

print("Training Accuracy:", dt_classifier.score(X, y))

Question 2: Decision Tree with Feature Importance

Python

import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeClassifier, plot_tree
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
import matplotlib.pyplot as plt
import seaborn as sns

# Load the dataset
df = pd.read_csv('diabetes.csv')

# Separate features and target
X = df.drop('label', axis=1)
y = df['label']

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Create and train the classifier
dt_classifier = DecisionTreeClassifier(random_state=42)
dt_classifier.fit(X_train, y_train)

# Predictions
y_pred = dt_classifier.predict(X_test)

# Feature Importance
feature_importance = pd.DataFrame({
    'Feature': X.columns,
    'Importance': dt_classifier.feature_importances_
}).sort_values('Importance', ascending=False)

print("Feature Importance:")
print(feature_importance)

# Plot 1: Feature Importance Bar Chart
plt.figure(figsize=(10, 6))
plt.barh(feature_importance['Feature'], feature_importance['Importance'], color='steelblue')
plt.xlabel('Importance')
plt.ylabel('Feature')
plt.title('Feature Importance in Decision Tree Classifier')
plt.gca().invert_yaxis()
plt.tight_layout()
plt.savefig('feature_importance.png')
plt.show()

# Plot 2: Confusion Matrix
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
            xticklabels=['No Diabetes', 'Diabetes'],
            yticklabels=['No Diabetes', 'Diabetes'])
plt.title('Confusion Matrix')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.tight_layout()
plt.savefig('confusion_matrix.png')
plt.show()

# Plot 3: Decision Tree Visualization
plt.figure(figsize=(20, 12))
plot_tree(dt_classifier, 
          feature_names=X.columns.tolist(),
          class_names=['No Diabetes', 'Diabetes'],
          filled=True, 
          rounded=True,
          fontsize=8,
          max_depth=4)
plt.title('Decision Tree Classifier')
plt.tight_layout()
plt.savefig('decision_tree_full.png', dpi=300)
plt.show()

# Plot 4: Accuracy Metrics
print("\nClassification Report:")
print(classification_report(y_test, y_pred))

# Plot 5: Feature Importance Pie Chart
plt.figure(figsize=(10, 8))
plt.pie(feature_importance['Importance'], labels=feature_importance['Feature'], 
        autopct='%1.1f%%', startangle=90)
plt.title('Feature Importance Distribution')
plt.tight_layout()
plt.savefig('feature_importance_pie.png')
plt.show()

print(f"\nModel Accuracy: {accuracy_score(y_test, y_pred):.4f}")

Question 3: Decision Tree vs SVM on Iris Dataset

Python

import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, classification_report
import matplotlib.pyplot as plt

# Load iris dataset
iris = load_iris()
X = iris.data
y = iris.target

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Decision Tree Classifier
dt_classifier = DecisionTreeClassifier(random_state=42)
dt_classifier.fit(X_train, y_train)
dt_pred = dt_classifier.predict(X_test)
dt_accuracy = accuracy_score(y_test, dt_pred)

# SVM Classifier
svm_classifier = SVC(kernel='rbf', random_state=42)
svm_classifier.fit(X_train, y_train)
svm_pred = svm_classifier.predict(X_test)
svm_accuracy = accuracy_score(y_test, svm_pred)

# Display results
print("=" * 50)
print("DECISION TREE CLASSIFIER")
print("=" * 50)
print(f"Accuracy: {dt_accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, dt_pred, target_names=iris.target_names))

print("=" * 50)
print("SVM CLASSIFIER")
print("=" * 50)
print(f"Accuracy: {svm_accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, svm_pred, target_names=iris.target_names))

# Comparison Plot
models = ['Decision Tree', 'SVM']
accuracies = [dt_accuracy, svm_accuracy]

plt.figure(figsize=(8, 6))
bars = plt.bar(models, accuracies, color=['green', 'blue'])
plt.ylim(0, 1.1)
plt.ylabel('Accuracy')
plt.title('Decision Tree vs SVM Classifier Accuracy Comparison')

for bar, acc in zip(bars, accuracies):
    plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, 
             f'{acc:.4f}', ha='center', fontsize=12)

plt.tight_layout()
plt.savefig('dt_vs_svm_comparison.png')
plt.show()

Question 4: Decision Tree with Different Parameters (max_leaf_nodes vs min_samples_leaf)

Python

import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier, plot_tree
from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt

# Load iris dataset
iris = load_iris()
X = iris.data
y = iris.target

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Model 1: max_leaf_nodes = 8
dt_max_leaf = DecisionTreeClassifier(max_leaf_nodes=8, random_state=42)
dt_max_leaf.fit(X_train, y_train)
pred_max_leaf = dt_max_leaf.predict(X_test)
accuracy_max_leaf = accuracy_score(y_test, pred_max_leaf)

# Model 2: min_samples_leaf = 5
dt_min_samples = DecisionTreeClassifier(min_samples_leaf=5, random_state=42)
dt_min_samples.fit(X_train, y_train)
pred_min_samples = dt_min_samples.predict(X_test)
accuracy_min_samples = accuracy_score(y_test, pred_min_samples)

# Display Results
print("=" * 60)
print("MODEL COMPARISON")
print("=" * 60)
print(f"\nModel 1 (max_leaf_nodes=8):")
print(f"  Accuracy: {accuracy_max_leaf:.4f}")
print(f"  Number of Leaves: {dt_max_leaf.get_n_leaves()}")
print(f"  Tree Depth: {dt_max_leaf.get_depth()}")

print(f"\nModel 2 (min_samples_leaf=5):")
print(f"  Accuracy: {accuracy_min_samples:.4f}")
print(f"  Number of Leaves: {dt_min_samples.get_n_leaves()}")
print(f"  Tree Depth: {dt_min_samples.get_depth()}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(20, 8))

# Tree 1
plot_tree(dt_max_leaf, 
          feature_names=iris.feature_names,
          class_names=iris.target_names.tolist(),
          filled=True, 
          rounded=True,
          ax=axes[0],
          fontsize=8)
axes[0].set_title(f'max_leaf_nodes=8\nAccuracy: {accuracy_max_leaf:.4f}')

# Tree 2
plot_tree(dt_min_samples, 
          feature_names=iris.feature_names,
          class_names=iris.target_names.tolist(),
          filled=True, 
          rounded=True,
          ax=axes[1],
          fontsize=8)
axes[1].set_title(f'min_samples_leaf=5\nAccuracy: {accuracy_min_samples:.4f}')

plt.tight_layout()
plt.savefig('dt_comparison.png', dpi=300)
plt.show()

# Accuracy Comparison Bar Chart
plt.figure(figsize=(8, 6))
models = ['max_leaf_nodes=8', 'min_samples_leaf=5']
accuracies = [accuracy_max_leaf, accuracy_min_samples]
bars = plt.bar(models, accuracies, color=['coral', 'teal'])
plt.ylim(0, 1.1)
plt.ylabel('Accuracy')
plt.title('Model Accuracy Comparison')

for bar, acc in zip(bars, accuracies):
    plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, 
             f'{acc:.4f}', ha='center', fontsize=12)

plt.tight_layout()
plt.savefig('accuracy_comparison.png')
plt.show()

Question 5: Decision Tree vs Logistic Regression on Diabetes Dataset

Python

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

# Load the dataset
df = pd.read_csv('diabetes.csv')

# Separate features and target
X = df.drop('label', axis=1)
y = df['label']

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features for Logistic Regression
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Decision Tree Classifier
dt_classifier = DecisionTreeClassifier(random_state=42)
dt_classifier.fit(X_train, y_train)
dt_pred = dt_classifier.predict(X_test)
dt_accuracy = accuracy_score(y_test, dt_pred)

# Logistic Regression
lr_classifier = LogisticRegression(max_iter=1000, random_state=42)
lr_classifier.fit(X_train_scaled, y_train)
lr_pred = lr_classifier.predict(X_test_scaled)
lr_accuracy = accuracy_score(y_test, lr_pred)

# Display Results
print("=" * 60)
print("DECISION TREE CLASSIFIER")
print("=" * 60)
print(f"Accuracy: {dt_accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, dt_pred))

print("=" * 60)
print("LOGISTIC REGRESSION")
print("=" * 60)
print(f"Accuracy: {lr_accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, lr_pred))

# Comparison Plot
fig, axes = plt.subplots(1, 3, figsize=(15, 5))

# Accuracy Comparison
models = ['Decision Tree', 'Logistic Regression']
accuracies = [dt_accuracy, lr_accuracy]
bars = axes[0].bar(models, accuracies, color=['green', 'orange'])
axes[0].set_ylim(0, 1.1)
axes[0].set_ylabel('Accuracy')
axes[0].set_title('Model Accuracy Comparison')
for bar, acc in zip(bars, accuracies):
    axes[0].text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, 
                 f'{acc:.4f}', ha='center', fontsize=11)

# Decision Tree Confusion Matrix
cm_dt = confusion_matrix(y_test, dt_pred)
sns.heatmap(cm_dt, annot=True, fmt='d', cmap='Greens', ax=axes[1])
axes[1].set_title('Decision Tree - Confusion Matrix')
axes[1].set_xlabel('Predicted')
axes[1].set_ylabel('Actual')

# Logistic Regression Confusion Matrix
cm_lr = confusion_matrix(y_test, lr_pred)
sns.heatmap(cm_lr, annot=True, fmt='d', cmap='Oranges', ax=axes[2])
axes[2].set_title('Logistic Regression - Confusion Matrix')
axes[2].set_xlabel('Predicted')
axes[2].set_ylabel('Actual')

plt.tight_layout()
plt.savefig('dt_vs_lr_comparison.png')
plt.show()

Question 6: Random Dataset with Multiple Model Comparison

Python

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt

# Generate random dataset
np.random.seed(42)
n_samples = 350

data = {
    'Feature1': np.random.randn(n_samples) * 10 + 50,
    'Feature2': np.random.randn(n_samples) * 15 + 30,
    'Feature3': np.random.randn(n_samples) * 8 + 25
}

# Create target based on features
df = pd.DataFrame(data)
df['Target'] = ((df['Feature1'] > 50) & (df['Feature2'] > 30) | (df['Feature3'] > 25)).astype(int)

print("Generated Dataset:")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")
print(f"\nClass Distribution:\n{df['Target'].value_counts()}")

# Separate features and target
X = df[['Feature1', 'Feature2', 'Feature3']]
y = df['Target']

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Model 1: Decision Tree Classifier
dt_classifier = DecisionTreeClassifier(random_state=42)
dt_classifier.fit(X_train, y_train)
dt_pred = dt_classifier.predict(X_test)
dt_accuracy = accuracy_score(y_test, dt_pred)

# Model 2: Random Forest Classifier
rf_classifier = RandomForestClassifier(n_estimators=100, random_state=42)
rf_classifier.fit(X_train, y_train)
rf_pred = rf_classifier.predict(X_test)
rf_accuracy = accuracy_score(y_test, rf_pred)

# Model 3: SVM Classifier
svm_classifier = SVC(kernel='rbf', random_state=42)
svm_classifier.fit(X_train, y_train)
svm_pred = svm_classifier.predict(X_test)
svm_accuracy = accuracy_score(y_test, svm_pred)

# Display Results
print("\n" + "=" * 60)
print("MODEL COMPARISON RESULTS")
print("=" * 60)
print(f"Decision Tree Accuracy: {dt_accuracy:.4f}")
print(f"Random Forest Accuracy: {rf_accuracy:.4f}")
print(f"SVM Accuracy: {svm_accuracy:.4f}")

# Visualization
models = ['Decision Tree', 'Random Forest', 'SVM']
accuracies = [dt_accuracy, rf_accuracy, svm_accuracy]
colors = ['green', 'blue', 'red']

plt.figure(figsize=(10, 6))
bars = plt.bar(models, accuracies, color=colors, edgecolor='black', linewidth=1.5)
plt.ylim(0, 1.1)
plt.ylabel('Accuracy', fontsize=12)
plt.xlabel('Model', fontsize=12)
plt.title('Model Accuracy Comparison on Random Dataset', fontsize=14)

for bar, acc in zip(bars, accuracies):
    plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, 
             f'{acc:.4f}', ha='center', fontsize=12, fontweight='bold')

plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.tight_layout()
plt.savefig('model_comparison_random_dataset.png', dpi=300)
plt.show()

Question 7: CCP Alpha Analysis for Decision Tree

Python

import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

# Load the dataset
df = pd.read_csv('diabetes.csv')

# Separate features and target
X = df.drop('label', axis=1)
y = df['label']

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Get the cost complexity pruning path
clf = DecisionTreeClassifier(random_state=42)
path = clf.cost_complexity_pruning_path(X_train, y_train)
ccp_alphas = path.ccp_alphas
impurities = path.impurities

# Build trees for different ccp_alpha values
clfs = []
node_counts = []
max_depths = []

for ccp_alpha in ccp_alphas:
    clf = DecisionTreeClassifier(ccp_alpha=ccp_alpha, random_state=42)
    clf.fit(X_train, y_train)
    clfs.append(clf)
    node_counts.append(clf.tree_.node_count)
    max_depths.append(clf.tree_.max_depth)

# Plot ccp_alpha vs Node Count
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: ccp_alpha vs Node Count
axes[0].plot(ccp_alphas, node_counts, marker='o', drawstyle="steps-post", color='blue')
axes[0].set_xlabel('ccp_alpha', fontsize=12)
axes[0].set_ylabel('Number of Nodes', fontsize=12)
axes[0].set_title('ccp_alpha vs Node Count', fontsize=14)
axes[0].grid(True, linestyle='--', alpha=0.7)

# Plot 2: ccp_alpha vs Max Depth
axes[1].plot(ccp_alphas, max_depths, marker='s', drawstyle="steps-post", color='green')
axes[1].set_xlabel('ccp_alpha', fontsize=12)
axes[1].set_ylabel('Maximum Depth', fontsize=12)
axes[1].set_title('ccp_alpha vs Maximum Depth', fontsize=14)
axes[1].grid(True, linestyle='--', alpha=0.7)

plt.tight_layout()
plt.savefig('ccp_alpha_analysis.png', dpi=300)
plt.show()

# Print summary
print("CCP Alpha Analysis Summary:")
print("-" * 50)
for i in range(0, len(ccp_alphas), max(1, len(ccp_alphas)//10)):
    print(f"ccp_alpha: {ccp_alphas[i]:.6f}, Nodes: {node_counts[i]}, Depth: {max_depths[i]}")

Question 8: Elbow Method for Optimal Clusters

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans

# Generate dataset with 14 points
np.random.seed(42)
X = np.array([
    [1, 2], [1.5, 1.8], [5, 8], [8, 8], [1, 0.6],
    [9, 11], [8, 2], [10, 2], [9, 3], [4, 5],
    [6, 6], [3, 4], [7, 7], [2, 3]
])

print("Dataset (14 points):")
print(X)

# Calculate WCSS (Within-Cluster Sum of Squares) for different k values
wcss = []
k_range = range(1, 11)

for k in k_range:
    kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)
    kmeans.fit(X)
    wcss.append(kmeans.inertia_)

# Plot the Elbow Curve
plt.figure(figsize=(10, 6))
plt.plot(k_range, wcss, 'bo-', linewidth=2, markersize=8)
plt.xlabel('Number of Clusters (k)', fontsize=12)
plt.ylabel('WCSS (Within-Cluster Sum of Squares)', fontsize=12)
plt.title('Elbow Method for Optimal Number of Clusters', fontsize=14)
plt.xticks(k_range)
plt.grid(True, linestyle='--', alpha=0.7)

# Annotate the elbow point
elbow_point = 3  # Usually identified visually
plt.axvline(x=elbow_point, color='red', linestyle='--', label=f'Optimal k = {elbow_point}')
plt.legend()

plt.tight_layout()
plt.savefig('elbow_method.png', dpi=300)
plt.show()

print("\nWCSS values for different k:")
for k, w in zip(k_range, wcss):
    print(f"k = {k}: WCSS = {w:.4f}")

print(f"\nOptimal number of clusters (Elbow Point): {elbow_point}")

Question 9: K-Medoids Clustering for Urban Areas

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn_extra.cluster import KMedoids
from sklearn.preprocessing import StandardScaler

# Generate random dataset for urban areas
np.random.seed(42)
n_samples = 100

data = {
    'Population_Density': np.random.randint(1000, 50000, n_samples),
    'Average_Income': np.random.randint(20000, 150000, n_samples),
    'Amenities_Score': np.random.randint(1, 100, n_samples),
    'Latitude': np.random.uniform(25.0, 45.0, n_samples),
    'Longitude': np.random.uniform(-120.0, -70.0, n_samples)
}

df = pd.DataFrame(data)
print("Urban Areas Dataset:")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")

# Select features for clustering
features = ['Population_Density', 'Average_Income', 'Amenities_Score']
X = df[features]

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply K-Medoids clustering
n_clusters = 4
kmedoids = KMedoids(n_clusters=n_clusters, random_state=42)
df['Cluster'] = kmedoids.fit_predict(X_scaled)

# Visualize clusters on a map
fig, axes = plt.subplots(1, 2, figsize=(16, 6))

# Map visualization
colors = ['red', 'blue', 'green', 'purple']
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    axes[0].scatter(cluster_data['Longitude'], cluster_data['Latitude'], 
                    c=colors[cluster], label=f'Cluster {cluster}', 
                    s=100, alpha=0.7, edgecolors='black')

axes[0].set_xlabel('Longitude', fontsize=12)
axes[0].set_ylabel('Latitude', fontsize=12)
axes[0].set_title('Urban Areas Clusters on Map', fontsize=14)
axes[0].legend()
axes[0].grid(True, linestyle='--', alpha=0.5)

# Feature-based visualization
scatter = axes[1].scatter(df['Population_Density'], df['Average_Income'], 
                          c=df['Cluster'], cmap='viridis', s=df['Amenities_Score']*2, 
                          alpha=0.7, edgecolors='black')
axes[1].set_xlabel('Population Density', fontsize=12)
axes[1].set_ylabel('Average Income', fontsize=12)
axes[1].set_title('Clusters based on Population Density vs Income', fontsize=14)
plt.colorbar(scatter, ax=axes[1], label='Cluster')

plt.tight_layout()
plt.savefig('kmedoids_urban_clusters.png', dpi=300)
plt.show()

# Cluster Summary
print("\nCluster Summary:")
print("-" * 60)
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    print(f"\nCluster {cluster} ({len(cluster_data)} neighborhoods):")
    print(f"  Avg Population Density: {cluster_data['Population_Density'].mean():.2f}")
    print(f"  Avg Income: ${cluster_data['Average_Income'].mean():.2f}")
    print(f"  Avg Amenities Score: {cluster_data['Amenities_Score'].mean():.2f}")

Question 10: K-Medoids for Customer Segmentation

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn_extra.cluster import KMedoids
from sklearn.preprocessing import StandardScaler

# Generate random customer dataset
np.random.seed(42)
n_customers = 100

data = {
    'Customer_ID': range(1, n_customers + 1),
    'Age': np.random.randint(18, 70, n_customers),
    'Spending_Score': np.random.randint(1, 100, n_customers),
    'Annual_Income': np.random.randint(15000, 150000, n_customers)
}

df = pd.DataFrame(data)
print("Customer Dataset:")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")

# Select features for clustering
features = ['Age', 'Spending_Score']
X = df[features]

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply K-Medoids clustering
n_clusters = 4
kmedoids = KMedoids(n_clusters=n_clusters, random_state=42)
df['Cluster'] = kmedoids.fit_predict(X_scaled)

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 6))

# Scatter plot
colors = ['red', 'blue', 'green', 'purple']
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    axes[0].scatter(cluster_data['Age'], cluster_data['Spending_Score'], 
                    c=colors[cluster], label=f'Cluster {cluster}', 
                    s=100, alpha=0.7, edgecolors='black')

# Mark medoids
medoid_indices = kmedoids.medoid_indices_
for idx in medoid_indices:
    axes[0].scatter(df.loc[idx, 'Age'], df.loc[idx, 'Spending_Score'], 
                    c='black', marker='X', s=200, label='Medoid' if idx == medoid_indices[0] else "")

axes[0].set_xlabel('Age', fontsize=12)
axes[0].set_ylabel('Spending Score', fontsize=12)
axes[0].set_title('Customer Segmentation using K-Medoids', fontsize=14)
axes[0].legend()
axes[0].grid(True, linestyle='--', alpha=0.5)

# Cluster distribution
cluster_counts = df['Cluster'].value_counts().sort_index()
axes[1].bar(cluster_counts.index, cluster_counts.values, color=colors, edgecolor='black')
axes[1].set_xlabel('Cluster', fontsize=12)
axes[1].set_ylabel('Number of Customers', fontsize=12)
axes[1].set_title('Customer Distribution per Cluster', fontsize=14)

plt.tight_layout()
plt.savefig('kmedoids_customer_segmentation.png', dpi=300)
plt.show()

# Cluster Summary
print("\n" + "=" * 60)
print("CUSTOMER SEGMENT CHARACTERISTICS")
print("=" * 60)
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    print(f"\nCluster {cluster} ({len(cluster_data)} customers):")
    print(f"  Age Range: {cluster_data['Age'].min()} - {cluster_data['Age'].max()}")
    print(f"  Avg Age: {cluster_data['Age'].mean():.1f}")
    print(f"  Avg Spending Score: {cluster_data['Spending_Score'].mean():.1f}")
    print(f"  Avg Annual Income: ${cluster_data['Annual_Income'].mean():,.2f}")

Question 11: K-Means for E-Commerce Customer Segmentation

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler

# Generate random e-commerce dataset
np.random.seed(42)
n_customers = 200

data = {
    'Customer_ID': range(1, n_customers + 1),
    'Total_Spending': np.random.exponential(500, n_customers) + 100,
    'Purchase_Frequency': np.random.poisson(10, n_customers) + 1,
    'Avg_Basket_Size': np.random.normal(50, 20, n_customers)
}

df = pd.DataFrame(data)
df['Avg_Basket_Size'] = df['Avg_Basket_Size'].clip(lower=10)  # Ensure positive values

print("E-Commerce Customer Dataset:")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")
print(f"\nStatistics:")
print(df.describe())

# Select features for clustering
features = ['Total_Spending', 'Purchase_Frequency', 'Avg_Basket_Size']
X = df[features]

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply K-Means clustering
n_clusters = 4
kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
df['Cluster'] = kmeans.fit_predict(X_scaled)

# Visualization
fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# Plot 1: Total Spending vs Purchase Frequency
colors = ['red', 'blue', 'green', 'purple']
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    axes[0, 0].scatter(cluster_data['Total_Spending'], cluster_data['Purchase_Frequency'], 
                       c=colors[cluster], label=f'Cluster {cluster}', 
                       s=80, alpha=0.7, edgecolors='black')

axes[0, 0].set_xlabel('Total Spending ($)', fontsize=11)
axes[0, 0].set_ylabel('Purchase Frequency', fontsize=11)
axes[0, 0].set_title('Total Spending vs Purchase Frequency', fontsize=12)
axes[0, 0].legend()
axes[0, 0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Total Spending vs Avg Basket Size
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    axes[0, 1].scatter(cluster_data['Total_Spending'], cluster_data['Avg_Basket_Size'], 
                       c=colors[cluster], label=f'Cluster {cluster}', 
                       s=80, alpha=0.7, edgecolors='black')

axes[0, 1].set_xlabel('Total Spending ($)', fontsize=11)
axes[0, 1].set_ylabel('Avg Basket Size ($)', fontsize=11)
axes[0, 1].set_title('Total Spending vs Avg Basket Size', fontsize=12)
axes[0, 1].legend()
axes[0, 1].grid(True, linestyle='--', alpha=0.5)

# Plot 3: 3D Scatter Plot
ax3d = fig.add_subplot(2, 2, 3, projection='3d')
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    ax3d.scatter(cluster_data['Total_Spending'], 
                 cluster_data['Purchase_Frequency'],
                 cluster_data['Avg_Basket_Size'],
                 c=colors[cluster], label=f'Cluster {cluster}', s=50, alpha=0.7)
ax3d.set_xlabel('Total Spending')
ax3d.set_ylabel('Purchase Frequency')
ax3d.set_zlabel('Avg Basket Size')
ax3d.set_title('3D Customer Segments')
axes[1, 0].set_visible(False)

# Plot 4: Cluster Size Distribution
cluster_counts = df['Cluster'].value_counts().sort_index()
axes[1, 1].bar(cluster_counts.index, cluster_counts.values, color=colors, edgecolor='black')
axes[1, 1].set_xlabel('Cluster', fontsize=11)
axes[1, 1].set_ylabel('Number of Customers', fontsize=11)
axes[1, 1].set_title('Customer Distribution per Cluster', fontsize=12)

plt.tight_layout()
plt.savefig('kmeans_ecommerce_segmentation.png', dpi=300)
plt.show()

# Cluster Summary
print("\n" + "=" * 70)
print("E-COMMERCE CUSTOMER SEGMENT CHARACTERISTICS")
print("=" * 70)
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    print(f"\nCluster {cluster} ({len(cluster_data)} customers):")
    print(f"  Avg Total Spending: ${cluster_data['Total_Spending'].mean():,.2f}")
    print(f"  Avg Purchase Frequency: {cluster_data['Purchase_Frequency'].mean():.1f} times")
    print(f"  Avg Basket Size: ${cluster_data['Avg_Basket_Size'].mean():.2f}")
    
    # Segment naming based on characteristics
    if cluster_data['Total_Spending'].mean() > df['Total_Spending'].median():
        if cluster_data['Purchase_Frequency'].mean() > df['Purchase_Frequency'].median():
            segment = "High-Value Frequent Buyers"
        else:
            segment = "High-Value Occasional Buyers"
    else:
        if cluster_data['Purchase_Frequency'].mean() > df['Purchase_Frequency'].median():
            segment = "Low-Value Frequent Buyers"
        else:
            segment = "Low-Value Occasional Buyers"
    print(f"  Segment Type: {segment}")

Question 12: K-Means with make_blobs (Varying n_clusters)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans

# Generate 300 samples using make_blobs
np.random.seed(42)
X, y_true = make_blobs(n_samples=300, centers=4, cluster_std=1.0, random_state=42)

print("Dataset Shape:", X.shape)
print("True Labels:", np.unique(y_true))

# Analyze data
plt.figure(figsize=(8, 6))
plt.scatter(X[:, 0], X[:, 1], c=y_true, cmap='viridis', s=50, alpha=0.7)
plt.title('Original Data (True Labels)', fontsize=14)
plt.xlabel('Feature 1')
plt.ylabel('Feature 2')
plt.colorbar(label='True Cluster')
plt.savefig('original_data.png')
plt.show()

# Apply KMeans with different n_clusters (2 to 7)
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
cluster_range = range(2, 8)

inertias = []

for idx, n_clusters in enumerate(cluster_range):
    row = idx // 3
    col = idx % 3
    
    kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
    labels = kmeans.fit_predict(X)
    inertias.append(kmeans.inertia_)
    
    # Plot
    scatter = axes[row, col].scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', s=50, alpha=0.7)
    centers = kmeans.cluster_centers_
    axes[row, col].scatter(centers[:, 0], centers[:, 1], c='red', marker='X', 
                           s=200, edgecolors='black', linewidths=2)
    axes[row, col].set_title(f'n_clusters = {n_clusters}\nInertia: {kmeans.inertia_:.2f}', fontsize=12)
    axes[row, col].set_xlabel('Feature 1')
    axes[row, col].set_ylabel('Feature 2')

plt.tight_layout()
plt.savefig('kmeans_varying_clusters.png', dpi=300)
plt.show()

# Elbow curve to show optimal clusters
plt.figure(figsize=(10, 6))
plt.plot(list(cluster_range), inertias, 'bo-', linewidth=2, markersize=10)
plt.xlabel('Number of Clusters (k)', fontsize=12)
plt.ylabel('Inertia (WCSS)', fontsize=12)
plt.title('Elbow Method - Impact of Changing n_clusters', fontsize=14)
plt.xticks(list(cluster_range))
plt.grid(True, linestyle='--', alpha=0.7)
plt.savefig('elbow_curve_make_blobs.png')
plt.show()

# Analysis
print("\n" + "=" * 60)
print("ANALYSIS OF VARYING CLUSTER COUNT")
print("=" * 60)
for n, inertia in zip(cluster_range, inertias):
    print(f"n_clusters = {n}: Inertia = {inertia:.2f}")

print("\nObservations:")
print("- As n_clusters increases, inertia (WCSS) decreases")
print("- Optimal clusters can be identified at the 'elbow' point")
print("- Since data was generated with 4 centers, k=4 is optimal")

Question 13: K-Means for Grocery Store Transaction Clustering

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler

# Generate random transaction data
np.random.seed(42)
n_transactions = 200

# Define product categories
products = ['Bread', 'Milk', 'Eggs', 'Cheese', 'Butter', 'Fruits', 'Vegetables', 'Meat', 'Fish', 'Snacks']

# Generate transaction data (1 = purchased, 0 = not purchased)
data = {}
for product in products:
    data[product] = np.random.binomial(1, np.random.uniform(0.2, 0.8), n_transactions)

df = pd.DataFrame(data)
df['Transaction_ID'] = range(1, n_transactions + 1)
df['Total_Items'] = df[products].sum(axis=1)
df['Transaction_Value'] = np.random.uniform(10, 200, n_transactions)

print("Transaction Dataset:")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")

# Features for clustering
X = df[products]

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply K-Means clustering
n_clusters = 4
kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
df['Cluster'] = kmeans.fit_predict(X_scaled)

# Visualization
fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# Plot 1: Cluster distribution
cluster_counts = df['Cluster'].value_counts().sort_index()
colors = ['coral', 'lightblue', 'lightgreen', 'plum']
axes[0, 0].bar(cluster_counts.index, cluster_counts.values, color=colors, edgecolor='black')
axes[0, 0].set_xlabel('Cluster', fontsize=11)
axes[0, 0].set_ylabel('Number of Transactions', fontsize=11)
axes[0, 0].set_title('Transaction Distribution per Cluster', fontsize=12)

# Plot 2: Average products per cluster
cluster_means = df.groupby('Cluster')[products].mean()
cluster_means.T.plot(kind='bar', ax=axes[0, 1], width=0.8, edgecolor='black')
axes[0, 1].set_xlabel('Product', fontsize=11)
axes[0, 1].set_ylabel('Average Purchase Rate', fontsize=11)
axes[0, 1].set_title('Product Purchase Rate by Cluster', fontsize=12)
axes[0, 1].legend(title='Cluster')
axes[0, 1].tick_params(axis='x', rotation=45)

# Plot 3: Heatmap of cluster centroids
import seaborn as sns
sns.heatmap(cluster_means, annot=True, cmap='YlOrRd', fmt='.2f', ax=axes[1, 0])
axes[1, 0].set_title('Cluster Centroids Heatmap', fontsize=12)
axes[1, 0].set_xlabel('Products')
axes[1, 0].set_ylabel('Cluster')

# Plot 4: Total Items vs Transaction Value by Cluster
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    axes[1, 1].scatter(cluster_data['Total_Items'], cluster_data['Transaction_Value'],
                       c=colors[cluster], label=f'Cluster {cluster}', s=80, alpha=0.7, edgecolors='black')
axes[1, 1].set_xlabel('Total Items', fontsize=11)
axes[1, 1].set_ylabel('Transaction Value ($)', fontsize=11)
axes[1, 1].set_title('Total Items vs Transaction Value', fontsize=12)
axes[1, 1].legend()
axes[1, 1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('grocery_transaction_clustering.png', dpi=300)
plt.show()

# Cluster Summary for Product Placement Recommendations
print("\n" + "=" * 70)
print("GROCERY TRANSACTION CLUSTERS - PRODUCT PLACEMENT INSIGHTS")
print("=" * 70)
for cluster in range(n_clusters):
    cluster_data = df[df['Cluster'] == cluster]
    top_products = cluster_means.loc[cluster].nlargest(3)
    print(f"\nCluster {cluster} ({len(cluster_data)} transactions):")
    print(f"  Avg Items per Transaction: {cluster_data['Total_Items'].mean():.1f}")
    print(f"  Avg Transaction Value: ${cluster_data['Transaction_Value'].mean():.2f}")
    print(f"  Top Products: {', '.join(top_products.index.tolist())}")

Question 14: K-Means with 35 Points - Impact of Varying Cluster Count

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans

# Generate random dataset with 35 points
np.random.seed(42)
X = np.vstack([
    np.random.randn(10, 2) + [0, 0],
    np.random.randn(10, 2) + [5, 5],
    np.random.randn(10, 2) + [10, 0],
    np.random.randn(5, 2) + [5, 10]
])

print(f"Dataset Shape: {X.shape}")

# Apply KMeans with different cluster counts
cluster_range = range(2, 8)
fig, axes = plt.subplots(2, 3, figsize=(15, 10))

inertias = []
silhouette_scores = []

from sklearn.metrics import silhouette_score

for idx, n_clusters in enumerate(cluster_range):
    row = idx // 3
    col = idx % 3
    
    kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
    labels = kmeans.fit_predict(X)
    inertias.append(kmeans.inertia_)
    
    if n_clusters > 1:
        sil_score = silhouette_score(X, labels)
        silhouette_scores.append(sil_score)
    
    # Plot with different colors for each cluster
    colors = plt.cm.viridis(np.linspace(0, 1, n_clusters))
    for cluster_id in range(n_clusters):
        cluster_points = X[labels == cluster_id]
        axes[row, col].scatter(cluster_points[:, 0], cluster_points[:, 1], 
                               c=[colors[cluster_id]], s=100, alpha=0.7, 
                               edgecolors='black', label=f'Cluster {cluster_id}')
    
    # Mark centroids
    centers = kmeans.cluster_centers_
    axes[row, col].scatter(centers[:, 0], centers[:, 1], c='red', marker='X', 
                           s=200, edgecolors='black', linewidths=2)
    axes[row, col].set_title(f'k = {n_clusters}\nInertia: {kmeans.inertia_:.2f}', fontsize=12)
    axes[row, col].set_xlabel('Feature 1')
    axes[row, col].set_ylabel('Feature 2')
    axes[row, col].legend(loc='best', fontsize=8)
    axes[row, col].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('kmeans_35_points_clusters.png', dpi=300)
plt.show()

# Impact Analysis Graphs
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Elbow Curve
axes[0].plot(list(cluster_range), inertias, 'bo-', linewidth=2, markersize=10)
axes[0].set_xlabel('Number of Clusters (k)', fontsize=12)
axes[0].set_ylabel('Inertia (WCSS)', fontsize=12)
axes[0].set_title('Elbow Method', fontsize=14)
axes[0].set_xticks(list(cluster_range))
axes[0].grid(True, linestyle='--', alpha=0.7)

# Silhouette Score
axes[1].plot(list(cluster_range), silhouette_scores, 'go-', linewidth=2, markersize=10)
axes[1].set_xlabel('Number of Clusters (k)', fontsize=12)
axes[1].set_ylabel('Silhouette Score', fontsize=12)
axes[1].set_title('Silhouette Score Analysis', fontsize=14)
axes[1].set_xticks(list(cluster_range))
axes[1].grid(True, linestyle='--', alpha=0.7)

plt.tight_layout()
plt.savefig('cluster_impact_analysis.png', dpi=300)
plt.show()

# Summary
print("\n" + "=" * 60)
print("IMPACT OF VARYING CLUSTER COUNT")
print("=" * 60)
print("\nCluster Analysis:")
for i, (k, inertia, sil) in enumerate(zip(cluster_range, inertias, silhouette_scores)):
    print(f"k = {k}: Inertia = {inertia:.2f}, Silhouette = {sil:.4f}")

print("\nJustification:")
print("- Increasing k reduces inertia but may lead to overfitting")
print("- Silhouette score helps identify the optimal number of clusters")
print("- The elbow point suggests optimal k where adding more clusters")
print("  provides diminishing returns")

Question 15: DBSCAN with make_moons

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_moons
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler

# Generate make_moons dataset
X, y_true = make_moons(n_samples=300, noise=0.1, random_state=42)

print("Dataset Shape:", X.shape)

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply DBSCAN
dbscan = DBSCAN(eps=0.3, min_samples=5)
labels = dbscan.fit_predict(X_scaled)

# Identify core samples and noise
core_samples_mask = np.zeros_like(labels, dtype=bool)
core_samples_mask[dbscan.core_sample_indices_] = True
n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
n_noise = list(labels).count(-1)

print(f"Number of clusters: {n_clusters}")
print(f"Number of noise points: {n_noise}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 6))

# Plot 1: True labels
axes[0].scatter(X[:, 0], X[:, 1], c=y_true, cmap='viridis', s=50, alpha=0.7)
axes[0].set_title('True Labels', fontsize=14)
axes[0].set_xlabel('Feature 1')
axes[0].set_ylabel('Feature 2')

# Plot 2: DBSCAN clusters
unique_labels = set(labels)
colors = plt.cm.Spectral(np.linspace(0, 1, len(unique_labels)))

for k, col in zip(unique_labels, colors):
    if k == -1:
        col = 'black'  # Noise points in black
        marker = 'x'
        size = 30
    else:
        marker = 'o'
        size = 50
    
    class_member_mask = (labels == k)
    xy = X[class_member_mask]
    axes[1].scatter(xy[:, 0], xy[:, 1], c=[col], marker=marker, s=size, 
                    alpha=0.7, label=f'Cluster {k}' if k != -1 else 'Noise')

axes[1].set_title(f'DBSCAN Clustering\n(eps=0.3, min_samples=5)', fontsize=14)
axes[1].set_xlabel('Feature 1')
axes[1].set_ylabel('Feature 2')
axes[1].legend()

plt.tight_layout()
plt.savefig('dbscan_make_moons.png', dpi=300)
plt.show()

print(f"\nDBSCAN successfully identified the two moons as separate clusters!")

Question 16: DBSCAN with Random Dataset (3rd and 4th columns)

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN

# Generate random dataset with 5 columns
np.random.seed(42)
n_samples = 200

data = np.random.randn(n_samples, 5) * 10

# Create clusters in 3rd and 4th columns
data[:50, 2:4] += [0, 0]
data[50:100, 2:4] += [15, 15]
data[100:150, 2:4] += [30, 0]
data[150:, 2:4] += [15, 30]

df = pd.DataFrame(data, columns=['Col1', 'Col2', 'Col3', 'Col4', 'Col5'])

print("Random Dataset (5 columns):")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")

# Use 3rd and 4th column for DBSCAN
X = df[['Col3', 'Col4']].values

# Apply DBSCAN with eps=5 and min_samples=5
dbscan = DBSCAN(eps=5, min_samples=5)
labels = dbscan.fit_predict(X)

df['Cluster'] = labels

# Analysis
n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
n_noise = list(labels).count(-1)

print(f"\nDBSCAN Results (eps=5, min_samples=5):")
print(f"Number of clusters: {n_clusters}")
print(f"Number of noise points: {n_noise}")

# Visualization
plt.figure(figsize=(10, 8))
unique_labels = set(labels)
colors = plt.cm.Spectral(np.linspace(0, 1, len(unique_labels)))

for k, col in zip(unique_labels, colors):
    if k == -1:
        col = 'black'
        marker = 'x'
        label = 'Noise'
    else:
        marker = 'o'
        label = f'Cluster {k}'
    
    mask = (labels == k)
    plt.scatter(X[mask, 0], X[mask, 1], c=[col], marker=marker, s=60, 
                alpha=0.7, edgecolors='black', label=label)

plt.xlabel('Column 3', fontsize=12)
plt.ylabel('Column 4', fontsize=12)
plt.title(f'DBSCAN Clustering (eps=5, min_samples=5)\n{n_clusters} Clusters, {n_noise} Noise Points', fontsize=14)
plt.legend()
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('dbscan_random_dataset.png', dpi=300)
plt.show()

Question 17: DBSCAN for Customer Purchasing Behavior

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler

# Generate random customer dataset
np.random.seed(42)
n_customers = 150

data = {
    'Customer_ID': range(1, n_customers + 1),
    'Purchase_Frequency': np.concatenate([
        np.random.poisson(5, 50),      # Low frequency customers
        np.random.poisson(15, 50),     # Medium frequency customers
        np.random.poisson(30, 50)      # High frequency customers
    ]),
    'Avg_Transaction_Amount': np.concatenate([
        np.random.normal(50, 10, 50),   # Low spenders
        np.random.normal(150, 30, 50),  # Medium spenders
        np.random.normal(300, 50, 50)   # High spenders
    ])
}

df = pd.DataFrame(data)
df['Avg_Transaction_Amount'] = df['Avg_Transaction_Amount'].clip(lower=10)

print("Customer Purchasing Behavior Dataset:")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")
print(f"\nStatistics:")
print(df.describe())

# Features for clustering
X = df[['Purchase_Frequency', 'Avg_Transaction_Amount']].values

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply DBSCAN
dbscan = DBSCAN(eps=0.5, min_samples=5)
labels = dbscan.fit_predict(X_scaled)

df['Cluster'] = labels

# Analysis
n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
n_noise = list(labels).count(-1)

print(f"\nDBSCAN Results:")
print(f"Number of customer segments: {n_clusters}")
print(f"Number of outliers: {n_noise}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 6))

# Plot 1: DBSCAN Clusters
unique_labels = set(labels)
colors = plt.cm.Spectral(np.linspace(0, 1, len(unique_labels)))

for k, col in zip(unique_labels, colors):
    if k == -1:
        col = 'black'
        marker = 'x'
        label = 'Outliers'
    else:
        marker = 'o'
        label = f'Segment {k}'
    
    mask = (labels == k)
    axes[0].scatter(df.loc[mask, 'Purchase_Frequency'], 
                    df.loc[mask, 'Avg_Transaction_Amount'], 
                    c=[col], marker=marker, s=80, alpha=0.7, 
                    edgecolors='black', label=label)

axes[0].set_xlabel('Purchase Frequency', fontsize=12)
axes[0].set_ylabel('Avg Transaction Amount ($)', fontsize=12)
axes[0].set_title('Customer Segments - DBSCAN', fontsize=14)
axes[0].legend()
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Cluster Distribution
cluster_counts = df['Cluster'].value_counts().sort_index()
colors_bar = ['black' if c == -1 else plt.cm.Spectral(c / max(cluster_counts.index.max(), 1)) 
              for c in cluster_counts.index]
axes[1].bar(cluster_counts.index.astype(str), cluster_counts.values, color=colors_bar, edgecolor='black')
axes[1].set_xlabel('Cluster', fontsize=12)
axes[1].set_ylabel('Number of Customers', fontsize=12)
axes[1].set_title('Customer Distribution per Segment', fontsize=14)

plt.tight_layout()
plt.savefig('dbscan_customer_behavior.png', dpi=300)
plt.show()

# Customer Segment Analysis
print("\n" + "=" * 70)
print("CUSTOMER SEGMENT ANALYSIS FOR TARGETED MARKETING")
print("=" * 70)

for cluster in sorted(df['Cluster'].unique()):
    cluster_data = df[df['Cluster'] == cluster]
    
    if cluster == -1:
        segment_name = "Outliers/Unusual Customers"
    else:
        avg_freq = cluster_data['Purchase_Frequency'].mean()
        avg_amount = cluster_data['Avg_Transaction_Amount'].mean()
        
        if avg_freq > 20 and avg_amount > 200:
            segment_name = "High-Value Loyal Customers"
        elif avg_freq > 10 and avg_amount > 100:
            segment_name = "Regular Mid-Tier Customers"
        elif avg_freq <= 10 and avg_amount > 150:
            segment_name = "Low-Frequency High-Value"
        else:
            segment_name = "Casual/Low-Value Customers"
    
    print(f"\nCluster {cluster}: {segment_name}")
    print(f"  Number of Customers: {len(cluster_data)}")
    print(f"  Avg Purchase Frequency: {cluster_data['Purchase_Frequency'].mean():.1f}")
    print(f"  Avg Transaction Amount: ${cluster_data['Avg_Transaction_Amount'].mean():.2f}")
    
    # Marketing Recommendations
    if cluster != -1:
        if segment_name == "High-Value Loyal Customers":
            print("  Marketing Strategy: VIP programs, exclusive offers, early access")
        elif segment_name == "Regular Mid-Tier Customers":
            print("  Marketing Strategy: Loyalty rewards, upselling, bundle deals")
        elif segment_name == "Low-Frequency High-Value":
            print("  Marketing Strategy: Re-engagement campaigns, premium products")
        else:
            print("  Marketing Strategy: Discounts, promotions, email campaigns")

Question 18: DBSCAN Parametric Analysis for eps

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import silhouette_score

# Generate dataset with Spending and Features
np.random.seed(42)
n_samples = 100

data = {
    'Spending': np.concatenate([
        np.random.normal(100, 20, 30),
        np.random.normal(300, 40, 40),
        np.random.normal(500, 30, 30)
    ]),
    'Features': np.concatenate([
        np.random.normal(20, 5, 30),
        np.random.normal(50, 10, 40),
        np.random.normal(80, 8, 30)
    ])
}

df = pd.DataFrame(data)

print("Dataset (Spending, Features):")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")

# Standardize features
X = df[['Spending', 'Features']].values
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Parametric analysis for eps
eps_values = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1.0, 1.5, 2.0]

results = []
for eps in eps_values:
    dbscan = DBSCAN(eps=eps, min_samples=5)
    labels = dbscan.fit_predict(X_scaled)
    
    n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
    n_noise = list(labels).count(-1)
    
    # Calculate silhouette score if more than 1 cluster
    if n_clusters > 1 and n_noise < len(X) - 1:
        sil_score = silhouette_score(X_scaled[labels != -1], labels[labels != -1])
    else:
        sil_score = -1
    
    results.append({
        'eps': eps,
        'n_clusters': n_clusters,
        'n_noise': n_noise,
        'silhouette': sil_score
    })

results_df = pd.DataFrame(results)
print("\nParametric Analysis Results:")
print(results_df)

# Visualization
fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# Plot 1: eps vs Number of Clusters
axes[0, 0].plot(results_df['eps'], results_df['n_clusters'], 'bo-', linewidth=2, markersize=8)
axes[0, 0].set_xlabel('eps', fontsize=12)
axes[0, 0].set_ylabel('Number of Clusters', fontsize=12)
axes[0, 0].set_title('eps vs Number of Clusters', fontsize=14)
axes[0, 0].grid(True, linestyle='--', alpha=0.7)

# Plot 2: eps vs Number of Noise Points
axes[0, 1].plot(results_df['eps'], results_df['n_noise'], 'ro-', linewidth=2, markersize=8)
axes[0, 1].set_xlabel('eps', fontsize=12)
axes[0, 1].set_ylabel('Number of Noise Points', fontsize=12)
axes[0, 1].set_title('eps vs Number of Noise Points', fontsize=14)
axes[0, 1].grid(True, linestyle='--', alpha=0.7)

# Plot 3: eps vs Silhouette Score
valid_sil = results_df[results_df['silhouette'] != -1]
axes[1, 0].plot(valid_sil['eps'], valid_sil['silhouette'], 'go-', linewidth=2, markersize=8)
axes[1, 0].set_xlabel('eps', fontsize=12)
axes[1, 0].set_ylabel('Silhouette Score', fontsize=12)
axes[1, 0].set_title('eps vs Silhouette Score', fontsize=14)
axes[1, 0].grid(True, linestyle='--', alpha=0.7)

# Plot 4: Clustering at different eps values (subplot with 4 examples)
eps_examples = [0.2, 0.5, 1.0, 2.0]
colors = ['red', 'blue', 'green', 'purple', 'orange']

for i, eps in enumerate(eps_examples):
    ax = fig.add_subplot(2, 4, 5 + i)
    dbscan = DBSCAN(eps=eps, min_samples=5)
    labels = dbscan.fit_predict(X_scaled)
    
    unique_labels = set(labels)
    for k in unique_labels:
        if k == -1:
            col = 'black'
            marker = 'x'
        else:
            col = colors[k % len(colors)]
            marker = 'o'
        mask = (labels == k)
        ax.scatter(df.loc[mask, 'Spending'], df.loc[mask, 'Features'], 
                   c=col, marker=marker, s=40, alpha=0.7)
    ax.set_title(f'eps={eps}', fontsize=10)
    ax.set_xlabel('Spending')
    ax.set_ylabel('Features')

axes[1, 1].set_visible(False)

plt.tight_layout()
plt.savefig('dbscan_eps_analysis.png', dpi=300)
plt.show()

# Summary
print("\n" + "=" * 60)
print("IMPACT OF EPS ON MODEL PERFORMANCE")
print("=" * 60)
print("\nObservations:")
print("- Small eps: More clusters, more noise points")
print("- Large eps: Fewer clusters, points merge together")
print("- Optimal eps: Balance between cluster definition and noise")
print(f"\nBest eps (highest silhouette): {valid_sil.loc[valid_sil['silhouette'].idxmax(), 'eps']}")

Question 19: DBSCAN Parameter Impact (eps and minPts)

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import silhouette_score

# Generate random dataset with 4 columns
np.random.seed(42)
n_samples = 200

data = np.random.randn(n_samples, 4) * 10

# Create clusters in 1st and 3rd columns
data[:60, [0, 2]] += [0, 0]
data[60:120, [0, 2]] += [20, 20]
data[120:180, [0, 2]] += [40, 0]
data[180:, [0, 2]] += [20, 40]

df = pd.DataFrame(data, columns=['Col1', 'Col2', 'Col3', 'Col4'])

print("Random Dataset (4 columns):")
print(df.head(10))

# Use 1st and 3rd column for DBSCAN
X = df[['Col1', 'Col3']].values

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Part 1: Impact of changing eps from 1 to 5
eps_values = [0.2, 0.4, 0.6, 0.8, 1.0, 1.5, 2.0, 3.0]
fig, axes = plt.subplots(2, 4, figsize=(16, 8))

eps_results = []
for idx, eps in enumerate(eps_values):
    row = idx // 4
    col = idx % 4
    
    dbscan = DBSCAN(eps=eps, min_samples=5)
    labels = dbscan.fit_predict(X_scaled)
    
    n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
    n_noise = list(labels).count(-1)
    
    eps_results.append({'eps': eps, 'n_clusters': n_clusters, 'n_noise': n_noise})
    
    # Plot
    unique_labels = set(labels)
    colors = plt.cm.Spectral(np.linspace(0, 1, max(len(unique_labels), 1)))
    
    for k, c in zip(unique_labels, colors):
        if k == -1:
            c = 'black'
            marker = 'x'
        else:
            marker = 'o'
        mask = (labels == k)
        axes[row, col].scatter(X[mask, 0], X[mask, 1], c=[c], marker=marker, s=30, alpha=0.7)
    
    axes[row, col].set_title(f'eps={eps}\nClusters:{n_clusters}, Noise:{n_noise}', fontsize=10)
    axes[row, col].set_xlabel('Col1')
    axes[row, col].set_ylabel('Col3')

plt.suptitle('Impact of Changing eps (min_samples=5)', fontsize=14)
plt.tight_layout()
plt.savefig('dbscan_eps_impact.png', dpi=300)
plt.show()

# Part 2: Impact of changing minPts
minPts_values = [2, 3, 5, 7, 10, 15, 20, 30]
fig, axes = plt.subplots(2, 4, figsize=(16, 8))

minPts_results = []
for idx, minPts in enumerate(minPts_values):
    row = idx // 4
    col = idx % 4
    
    dbscan = DBSCAN(eps=0.5, min_samples=minPts)
    labels = dbscan.fit_predict(X_scaled)
    
    n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
    n_noise = list(labels).count(-1)
    
    minPts_results.append({'minPts': minPts, 'n_clusters': n_clusters, 'n_noise': n_noise})
    
    # Plot
    unique_labels = set(labels)
    colors = plt.cm.Spectral(np.linspace(0, 1, max(len(unique_labels), 1)))
    
    for k, c in zip(unique_labels, colors):
        if k == -1:
            c = 'black'
            marker = 'x'
        else:
            marker = 'o'
        mask = (labels == k)
        axes[row, col].scatter(X[mask, 0], X[mask, 1], c=[c], marker=marker, s=30, alpha=0.7)
    
    axes[row, col].set_title(f'minPts={minPts}\nClusters:{n_clusters}, Noise:{n_noise}', fontsize=10)
    axes[row, col].set_xlabel('Col1')
    axes[row, col].set_ylabel('Col3')

plt.suptitle('Impact of Changing minPts (eps=0.5)', fontsize=14)
plt.tight_layout()
plt.savefig('dbscan_minPts_impact.png', dpi=300)
plt.show()

# Summary graphs
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# eps impact
eps_df = pd.DataFrame(eps_results)
axes[0].plot(eps_df['eps'], eps_df['n_clusters'], 'bo-', label='Clusters', linewidth=2, markersize=8)
axes[0].plot(eps_df['eps'], eps_df['n_noise'], 'ro-', label='Noise', linewidth=2, markersize=8)
axes[0].set_xlabel('eps', fontsize=12)
axes[0].set_ylabel('Count', fontsize=12)
axes[0].set_title('Impact of eps on DBSCAN', fontsize=14)
axes[0].legend()
axes[0].grid(True, linestyle='--', alpha=0.7)

# minPts impact
minPts_df = pd.DataFrame(minPts_results)
axes[1].plot(minPts_df['minPts'], minPts_df['n_clusters'], 'go-', label='Clusters', linewidth=2, markersize=8)
axes[1].plot(minPts_df['minPts'], minPts_df['n_noise'], 'mo-', label='Noise', linewidth=2, markersize=8)
axes[1].set_xlabel('minPts', fontsize=12)
axes[1].set_ylabel('Count', fontsize=12)
axes[1].set_title('Impact of minPts on DBSCAN', fontsize=14)
axes[1].legend()
axes[1].grid(True, linestyle='--', alpha=0.7)

plt.tight_layout()
plt.savefig('dbscan_parameter_impact_summary.png', dpi=300)
plt.show()

print("\n" + "=" * 60)
print("PARAMETER IMPACT ANALYSIS")
print("=" * 60)
print("\neps Impact:")
print(eps_df.to_string(index=False))
print("\nminPts Impact:")
print(minPts_df.to_string(index=False))

Question 20: DBSCAN and Agglomerative Clustering for Mall Customers

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN, AgglomerativeClustering
from sklearn.preprocessing import StandardScaler
from scipy.cluster.hierarchy import dendrogram, linkage

# Generate random mall customer dataset
np.random.seed(42)
n_samples = 200

data = {
    'Customer_ID': range(1, n_samples + 1),
    'Genre': np.random.choice(['Male', 'Female'], n_samples),
    'Age': np.random.randint(18, 70, n_samples),
    'Annual_Income': np.random.randint(15, 140, n_samples),  # in thousands
    'Spending_Score': np.random.randint(1, 100, n_samples)
}

df = pd.DataFrame(data)

print("Mall Customer Dataset:")
print(df.head(10))
print(f"\nDataset Shape: {df.shape}")
print(f"\nGenre Distribution:\n{df['Genre'].value_counts()}")

# Features for clustering
X = df[['Annual_Income', 'Spending_Score']].values

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Part 1: DBSCAN Clustering
dbscan = DBSCAN(eps=0.5, min_samples=5)
df['DBSCAN_Cluster'] = dbscan.fit_predict(X_scaled)

n_clusters_dbscan = len(set(df['DBSCAN_Cluster'])) - (1 if -1 in df['DBSCAN_Cluster'].values else 0)
n_noise = list(df['DBSCAN_Cluster']).count(-1)

print(f"\nDBSCAN Results:")
print(f"Number of clusters: {n_clusters_dbscan}")
print(f"Number of noise points: {n_noise}")

# Part 2: Agglomerative Clustering
n_clusters_agg = 5
agg_clustering = AgglomerativeClustering(n_clusters=n_clusters_agg)
df['Agg_Cluster'] = agg_clustering.fit_predict(X_scaled)

# Generate linkage matrix for dendrogram
linkage_matrix = linkage(X_scaled, method='ward')

# Visualization
fig = plt.figure(figsize=(18, 12))

# Plot 1: DBSCAN Clusters
ax1 = fig.add_subplot(2, 2, 1)
unique_labels = sorted(df['DBSCAN_Cluster'].unique())
colors = plt.cm.Spectral(np.linspace(0, 1, len(unique_labels)))

for k, col in zip(unique_labels, colors):
    if k == -1:
        col = 'black'
        marker = 'x'
        label = 'Noise'
    else:
        marker = 'o'
        label = f'Cluster {k}'
    
    mask = (df['DBSCAN_Cluster'] == k)
    ax1.scatter(df.loc[mask, 'Annual_Income'], df.loc[mask, 'Spending_Score'],
                c=[col], marker=marker, s=80, alpha=0.7, edgecolors='black', label=label)

ax1.set_xlabel('Annual Income (k$)', fontsize=12)
ax1.set_ylabel('Spending Score', fontsize=12)
ax1.set_title(f'DBSCAN Clustering\n{n_clusters_dbscan} Clusters, {n_noise} Noise Points', fontsize=14)
ax1.legend()
ax1.grid(True, linestyle='--', alpha=0.5)

# Plot 2: Agglomerative Clustering
ax2 = fig.add_subplot(2, 2, 2)
colors_agg = plt.cm.viridis(np.linspace(0, 1, n_clusters_agg))

for k in range(n_clusters_agg):
    mask = (df['Agg_Cluster'] == k)
    ax2.scatter(df.loc[mask, 'Annual_Income'], df.loc[mask, 'Spending_Score'],
                c=[colors_agg[k]], s=80, alpha=0.7, edgecolors='black', label=f'Cluster {k}')

ax2.set_xlabel('Annual Income (k$)', fontsize=12)
ax2.set_ylabel('Spending Score', fontsize=12)
ax2.set_title(f'Agglomerative Clustering\n{n_clusters_agg} Clusters', fontsize=14)
ax2.legend()
ax2.grid(True, linestyle='--', alpha=0.5)

# Plot 3: Dendrogram
ax3 = fig.add_subplot(2, 1, 2)
dendrogram(linkage_matrix, truncate_mode='lastp', p=30, leaf_rotation=90, 
           leaf_font_size=8, show_contracted=True, ax=ax3)
ax3.set_xlabel('Sample Index or Cluster Size', fontsize=12)
ax3.set_ylabel('Distance', fontsize=12)
ax3.set_title('Hierarchical Clustering Dendrogram', fontsize=14)
ax3.axhline(y=7, color='r', linestyle='--', label='Optimal cut')
ax3.legend()

plt.tight_layout()
plt.savefig('mall_customer_clustering.png', dpi=300)
plt.show()

# Cluster Summary
print("\n" + "=" * 70)
print("CLUSTER ANALYSIS SUMMARY")
print("=" * 70)

print("\nDBSCAN Clusters:")
for cluster in sorted(df['DBSCAN_Cluster'].unique()):
    cluster_data = df[df['DBSCAN_Cluster'] == cluster]
    print(f"\nCluster {cluster} ({len(cluster_data)} customers):")
    print(f"  Avg Age: {cluster_data['Age'].mean():.1f}")
    print(f"  Avg Annual Income: ${cluster_data['Annual_Income'].mean():.1f}k")
    print(f"  Avg Spending Score: {cluster_data['Spending_Score'].mean():.1f}")

print("\nAgglomerative Clusters:")
for cluster in range(n_clusters_agg):
    cluster_data = df[df['Agg_Cluster'] == cluster]
    print(f"\nCluster {cluster} ({len(cluster_data)} customers):")
    print(f"  Avg Age: {cluster_data['Age'].mean():.1f}")
    print(f"  Avg Annual Income: ${cluster_data['Annual_Income'].mean():.1f}k")
    print(f"  Avg Spending Score: {cluster_data['Spending_Score'].mean():.1f}")

Question 21: K-Fold Cross-Validation with SVM

Python

import numpy as np
from sklearn.datasets import make_classification
from sklearn.svm import SVC
from sklearn.model_selection import KFold, cross_val_score
from sklearn.preprocessing import StandardScaler

# Generate synthetic dataset
X, y = make_classification(
    n_samples=100,
    n_features=20,
    n_informative=15,
    n_redundant=5,
    n_classes=2,
    random_state=42
)

print("Dataset Shape:", X.shape)
print("Target Distribution:", np.bincount(y))

# Preprocess data (scaling)
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Initialize SVM classifier
svm_classifier = SVC(kernel='rbf', random_state=42)

# Implement 10-fold cross-validation
kfold = KFold(n_splits=10, shuffle=True, random_state=42)

# Manual cross-validation
print("\n" + "=" * 50)
print("K-FOLD CROSS-VALIDATION (k=10)")
print("=" * 50)

fold_accuracies = []
for fold, (train_idx, test_idx) in enumerate(kfold.split(X_scaled), 1):
    X_train, X_test = X_scaled[train_idx], X_scaled[test_idx]
    y_train, y_test = y[train_idx], y[test_idx]
    
    svm_classifier.fit(X_train, y_train)
    accuracy = svm_classifier.score(X_test, y_test)
    fold_accuracies.append(accuracy)
    
    print(f"Fold {fold:2d}: Accuracy = {accuracy:.4f}")

print("-" * 50)
print(f"Average Accuracy: {np.mean(fold_accuracies):.4f}")
print(f"Standard Deviation: {np.std(fold_accuracies):.4f}")

# Using cross_val_score for verification
cv_scores = cross_val_score(svm_classifier, X_scaled, y, cv=10)
print(f"\nVerification using cross_val_score:")
print(f"Average Accuracy: {cv_scores.mean():.4f} (+/- {cv_scores.std() * 2:.4f})")

Question 22: Bootstrap Sampling with Random Forest

Python

import numpy as np
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.utils import resample
from sklearn.metrics import accuracy_score

# Generate synthetic dataset
X, y = make_classification(
    n_samples=150,
    n_features=10,
    n_informative=7,
    n_redundant=3,
    n_classes=3,
    n_clusters_per_class=1,
    random_state=42
)

print("Dataset Shape:", X.shape)
print("Target Distribution:", np.bincount(y))

# Number of bootstrap samples
n_bootstrap = 10

print("\n" + "=" * 50)
print("BOOTSTRAP SAMPLING WITH RANDOM FOREST")
print("=" * 50)

bootstrap_accuracies = []

for i in range(n_bootstrap):
    # Create bootstrap sample
    X_boot, y_boot = resample(X, y, n_samples=len(X), random_state=i)
    
    # Train Random Forest on bootstrap sample
    rf_classifier = RandomForestClassifier(n_estimators=100, random_state=i)
    rf_classifier.fit(X_boot, y_boot)
    
    # Evaluate on original dataset
    y_pred = rf_classifier.predict(X)
    accuracy = accuracy_score(y, y_pred)
    bootstrap_accuracies.append(accuracy)
    
    print(f"Bootstrap Sample {i+1:2d}: Accuracy = {accuracy:.4f}")

print("-" * 50)
print(f"Average Accuracy: {np.mean(bootstrap_accuracies):.4f}")
print(f"Standard Deviation: {np.std(bootstrap_accuracies):.4f}")
print(f"95% Confidence Interval: [{np.percentile(bootstrap_accuracies, 2.5):.4f}, {np.percentile(bootstrap_accuracies, 97.5):.4f}]")

Question 23: Leave-One-Out Cross-Validation (LOOCV)

Python

import numpy as np
from sklearn.datasets import make_classification
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import LeaveOneOut

# Generate synthetic dataset
X, y = make_classification(
    n_samples=100,
    n_features=5,
    n_informative=4,
    n_redundant=1,
    n_classes=2,
    random_state=42
)

print("Dataset Shape:", X.shape)
print("Target Distribution:", np.bincount(y))

# Initialize Decision Tree classifier
dt_classifier = DecisionTreeClassifier(random_state=42)

# Implement LOOCV
loo = LeaveOneOut()

print("\n" + "=" * 50)
print("LEAVE-ONE-OUT CROSS-VALIDATION")
print("=" * 50)

accuracies = []
correct_predictions = 0

for train_idx, test_idx in loo.split(X):
    X_train, X_test = X[train_idx], X[test_idx]
    y_train, y_test = y[train_idx], y[test_idx]
    
    dt_classifier.fit(X_train, y_train)
    y_pred = dt_classifier.predict(X_test)
    
    is_correct = (y_pred[0] == y_test[0])
    correct_predictions += int(is_correct)
    accuracies.append(int(is_correct))

# Print some sample iterations
print("\nSample Iterations (first 10):")
for i in range(min(10, len(accuracies))):
    print(f"Iteration {i+1:3d}: {'Correct' if accuracies[i] else 'Incorrect'}")

print(f"\n... ({len(accuracies) - 10} more iterations)")

print("-" * 50)
print(f"Total Correct Predictions: {correct_predictions}/{len(X)}")
print(f"Average Accuracy: {np.mean(accuracies):.4f}")

Question 24: LOOCV with Noise (Confusion Matrix and Classification Report)

Python

import numpy as np
from sklearn.datasets import make_classification
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import LeaveOneOut
from sklearn.metrics import confusion_matrix, classification_report
import seaborn as sns
import matplotlib.pyplot as plt

# Generate synthetic dataset with noise
X, y = make_classification(
    n_samples=150,
    n_features=10,
    n_informative=5,
    n_redundant=3,
    n_classes=3,
    n_clusters_per_class=1,
    flip_y=0.1,  # Add noise
    random_state=42
)

print("Dataset Shape:", X.shape)
print("Target Distribution:", np.bincount(y))

# Initialize Decision Tree classifier
dt_classifier = DecisionTreeClassifier(random_state=42)

# Implement LOOCV
loo = LeaveOneOut()

print("\n" + "=" * 50)
print("LEAVE-ONE-OUT CROSS-VALIDATION WITH NOISE")
print("=" * 50)

y_true_all = []
y_pred_all = []
accuracies = []

for train_idx, test_idx in loo.split(X):
    X_train, X_test = X[train_idx], X[test_idx]
    y_train, y_test = y[train_idx], y[test_idx]
    
    dt_classifier.fit(X_train, y_train)
    y_pred = dt_classifier.predict(X_test)
    
    y_true_all.append(y_test[0])
    y_pred_all.append(y_pred[0])
    accuracies.append(int(y_pred[0] == y_test[0]))

y_true_all = np.array(y_true_all)
y_pred_all = np.array(y_pred_all)

# Print sample iterations
print("\nSample Iterations (first 10):")
for i in range(min(10, len(accuracies))):
    status = 'Correct' if accuracies[i] else 'Incorrect'
    print(f"Iteration {i+1:3d}: True={y_true_all[i]}, Pred={y_pred_all[i]} - {status}")

print(f"\n... ({len(accuracies) - 10} more iterations)")

# Average Accuracy
print("-" * 50)
print(f"Average Accuracy: {np.mean(accuracies):.4f}")

# Confusion Matrix
print("\nConfusion Matrix:")
cm = confusion_matrix(y_true_all, y_pred_all)
print(cm)

# Classification Report
print("\nClassification Report:")
print(classification_report(y_true_all, y_pred_all, target_names=['Class 0', 'Class 1', 'Class 2']))

# Visualization
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
            xticklabels=['Class 0', 'Class 1', 'Class 2'],
            yticklabels=['Class 0', 'Class 1', 'Class 2'])
plt.title('Confusion Matrix - LOOCV with Decision Tree', fontsize=14)
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.tight_layout()
plt.savefig('loocv_confusion_matrix.png')
plt.show()

Question 25: Stratified K-Fold Cross-Validation

Python

import numpy as np
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

# Generate synthetic dataset with class imbalance
X, y = make_classification(
    n_samples=200,
    n_features=15,
    n_informative=10,
    n_redundant=5,
    n_classes=4,
    n_clusters_per_class=1,
    weights=[0.4, 0.3, 0.2, 0.1],  # Class imbalance
    random_state=42
)

print("Dataset Shape:", X.shape)
print("Target Distribution:", np.bincount(y))

# Initialize Random Forest classifier
rf_classifier = RandomForestClassifier(n_estimators=100, random_state=42)

# Implement Stratified K-Fold Cross-Validation
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

print("\n" + "=" * 50)
print("STRATIFIED K-FOLD CROSS-VALIDATION (k=5)")
print("=" * 50)

fold_accuracies = []
y_true_all = []
y_pred_all = []

for fold, (train_idx, test_idx) in enumerate(skf.split(X, y), 1):
    X_train, X_test = X[train_idx], X[test_idx]
    y_train, y_test = y[train_idx], y[test_idx]
    
    rf_classifier.fit(X_train, y_train)
    y_pred = rf_classifier.predict(X_test)
    
    accuracy = accuracy_score(y_test, y_pred)
    fold_accuracies.append(accuracy)
    
    # Store for final evaluation
    y_true_all.extend(y_test)
    y_pred_all.extend(y_pred)
    
    print(f"Fold {fold}: Accuracy = {accuracy:.4f}")
    print(f"         Train class distribution: {np.bincount(y_train)}")
    print(f"         Test class distribution:  {np.bincount(y_test)}")

print("-" * 50)
print(f"Average Accuracy: {np.mean(fold_accuracies):.4f}")
print(f"Standard Deviation: {np.std(fold_accuracies):.4f}")

# Classification Report for final model
print("\nClassification Report (Overall):")
print(classification_report(y_true_all, y_pred_all, 
                          target_names=['Class 0', 'Class 1', 'Class 2', 'Class 3']))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Confusion Matrix
cm = confusion_matrix(y_true_all, y_pred_all)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
            xticklabels=['Class 0', 'Class 1', 'Class 2', 'Class 3'],
            yticklabels=['Class 0', 'Class 1', 'Class 2', 'Class 3'])
axes[0].set_title('Confusion Matrix', fontsize=14)
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# Fold Accuracies
axes[1].bar(range(1, 6), fold_accuracies, color='steelblue', edgecolor='black')
axes[1].axhline(y=np.mean(fold_accuracies), color='red', linestyle='--', label=f'Mean: {np.mean(fold_accuracies):.4f}')
axes[1].set_xlabel('Fold', fontsize=12)
axes[1].set_ylabel('Accuracy', fontsize=12)
axes[1].set_title('Accuracy per Fold', fontsize=14)
axes[1].set_ylim(0, 1.1)
axes[1].legend()

plt.tight_layout()
plt.savefig('stratified_kfold.png')
plt.show()

Question 26: Grid Search with SVM on Iris Dataset

Python

import numpy as np
from sklearn.datasets import load_iris
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

# Load Iris dataset
iris = load_iris()
X = iris.data
y = iris.target

print("Iris Dataset Shape:", X.shape)
print("Classes:", iris.target_names)

# Define hyperparameter grid
param_grid = {
    'C': [0.1, 1, 10, 100],
    'kernel': ['linear', 'rbf'],
    'gamma': ['scale', 'auto']
}

# Initialize SVM classifier
svm = SVC()

# Implement Grid Search with 5-fold CV
grid_search = GridSearchCV(
    estimator=svm,
    param_grid=param_grid,
    cv=5,
    scoring='accuracy',
    verbose=1,
    return_train_score=True
)

print("\n" + "=" * 50)
print("GRID SEARCH WITH 5-FOLD CROSS-VALIDATION")
print("=" * 50)

grid_search.fit(X, y)

# Best hyperparameters
print("\nBest Hyperparameters:")
print(grid_search.best_params_)
print(f"\nBest Cross-Validation Score: {grid_search.best_score_:.4f}")

# Evaluate best model on entire dataset
best_model = grid_search.best_estimator_
y_pred = best_model.predict(X)
accuracy = accuracy_score(y, y_pred)

print(f"\nAccuracy on Entire Dataset: {accuracy:.4f}")

# Confusion Matrix
cm = confusion_matrix(y, y_pred)
print("\nConfusion Matrix:")
print(cm)

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Confusion Matrix Heatmap
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
            xticklabels=iris.target_names,
            yticklabels=iris.target_names)
axes[0].set_title('Confusion Matrix - Best SVM Model', fontsize=14)
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# Grid Search Results
results = grid_search.cv_results_
scores = results['mean_test_score']
params = results['params']

# Create a heatmap for C and kernel (for gamma='scale')
c_values = [0.1, 1, 10, 100]
kernels = ['linear', 'rbf']

score_matrix = np.zeros((len(kernels), len(c_values)))
for i, kernel in enumerate(kernels):
    for j, c in enumerate(c_values):
        for k, param in enumerate(params):
            if param['C'] == c and param['kernel'] == kernel and param['gamma'] == 'scale':
                score_matrix[i, j] = scores[k]

sns.heatmap(score_matrix, annot=True, fmt='.4f', cmap='YlGn', ax=axes[1],
            xticklabels=c_values, yticklabels=kernels)
axes[1].set_title('Grid Search Scores (gamma=scale)', fontsize=14)
axes[1].set_xlabel('C')
axes[1].set_ylabel('Kernel')

plt.tight_layout()
plt.savefig('grid_search_svm.png')
plt.show()

Question 27: PCA on Iris Dataset (2D Visualization)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

# Load Iris dataset
iris = load_iris()
X = iris.data
y = iris.target

print("Original Dataset Shape:", X.shape)
print("Features:", iris.feature_names)
print("Classes:", iris.target_names)

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

print("\nStandardized Data:")
print(f"Mean: {X_scaled.mean(axis=0)}")
print(f"Std: {X_scaled.std(axis=0)}")

# Apply PCA to reduce to 2 dimensions
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_scaled)

print("\nPCA Results:")
print(f"Transformed Shape: {X_pca.shape}")
print(f"Explained Variance Ratio: {pca.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(pca.explained_variance_ratio_):.4f}")

# Visualization
plt.figure(figsize=(10, 8))

colors = ['red', 'green', 'blue']
markers = ['o', 's', '^']

for i, (target, color, marker) in enumerate(zip(iris.target_names, colors, markers)):
    mask = (y == i)
    plt.scatter(X_pca[mask, 0], X_pca[mask, 1], 
                c=color, marker=marker, s=80, 
                alpha=0.7, edgecolors='black', label=target)

plt.xlabel(f'PC1 ({pca.explained_variance_ratio_[0]*100:.2f}% variance)', fontsize=12)
plt.ylabel(f'PC2 ({pca.explained_variance_ratio_[1]*100:.2f}% variance)', fontsize=12)
plt.title('PCA of Iris Dataset (2D)', fontsize=14)
plt.legend(title='Species')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('pca_iris_2d.png', dpi=300)
plt.show()

# Print component loadings
print("\nPCA Component Loadings:")
components_df = np.abs(pca.components_)
for i, feature in enumerate(iris.feature_names):
    print(f"{feature}: PC1={pca.components_[0, i]:.4f}, PC2={pca.components_[1, i]:.4f}")

Question 28: Curse of Dimensionality with k-NN

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import time

# Fixed parameters
n_samples = 1000
k = 5

# Varying dimensions
dimensions = [2, 5, 10, 20, 50, 100]

print("=" * 60)
print("CURSE OF DIMENSIONALITY DEMONSTRATION")
print("=" * 60)

accuracies = []
training_times = []

for n_features in dimensions:
    # Generate synthetic dataset
    X, y = make_classification(
        n_samples=n_samples,
        n_features=n_features,
        n_informative=min(n_features, 10),
        n_redundant=0,
        n_classes=2,
        random_state=42
    )
    
    # Split data
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
    
    # Scale features
    scaler = StandardScaler()
    X_train_scaled = scaler.fit_transform(X_train)
    X_test_scaled = scaler.transform(X_test)
    
    # Train k-NN classifier
    knn = KNeighborsClassifier(n_neighbors=k)
    
    start_time = time.time()
    knn.fit(X_train_scaled, y_train)
    training_time = time.time() - start_time
    
    accuracy = knn.score(X_test_scaled, y_test)
    
    accuracies.append(accuracy)
    training_times.append(training_time)
    
    print(f"Dimensions: {n_features:3d} | Accuracy: {accuracy:.4f} | Time: {training_time:.4f}s")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Accuracy vs Dimensionality
axes[0].plot(dimensions, accuracies, 'bo-', linewidth=2, markersize=10)
axes[0].set_xlabel('Number of Dimensions', fontsize=12)
axes[0].set_ylabel('Accuracy', fontsize=12)
axes[0].set_title('k-NN Accuracy vs Dimensionality', fontsize=14)
axes[0].grid(True, linestyle='--', alpha=0.7)

# Training Time vs Dimensionality
axes[1].plot(dimensions, training_times, 'ro-', linewidth=2, markersize=10)
axes[1].set_xlabel('Number of Dimensions', fontsize=12)
axes[1].set_ylabel('Training Time (seconds)', fontsize=12)
axes[1].set_title('k-NN Training Time vs Dimensionality', fontsize=14)
axes[1].grid(True, linestyle='--', alpha=0.7)

plt.tight_layout()
plt.savefig('curse_of_dimensionality.png', dpi=300)
plt.show()

print("\n" + "=" * 60)
print("OBSERVATIONS")
print("=" * 60)
print("1. As dimensionality increases, accuracy may decrease")
print("2. Training time increases with dimensionality")
print("3. In high dimensions, data becomes sparse")
print("4. Distance metrics become less meaningful in high dimensions")

Question 29: Dimensionality Impact Explanation (SVM)

Python


import numpy as np
import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from math import gamma, pi

# Demonstrate volume change
def hypersphere_volume(d, r=1):
    
    return (pi ** (d/2) * r**d) / gamma(d/2 + 1)

dimensions = range(1, 15)
volumes = [hypersphere_volume(d) for d in dimensions]

plt.figure(figsize=(10, 6))
plt.plot(dimensions, volumes, 'bo-', linewidth=2, markersize=8)
plt.xlabel('Dimensions', fontsize=12)
plt.ylabel('Volume of Unit Hypersphere', fontsize=12)
plt.title('Volume of Unit Hypersphere vs Dimensionality', fontsize=14)
plt.axvline(x=3, color='green', linestyle='--', label='3D')
plt.axvline(x=12, color='red', linestyle='--', label='12D')
plt.legend()
plt.grid(True, linestyle='--', alpha=0.7)
plt.tight_layout()
plt.savefig('hypersphere_volume.png')
plt.show()

print("Volume of Unit Hypersphere:")
print(f"3D: {hypersphere_volume(3):.4f}")
print(f"12D: {hypersphere_volume(12):.6f}")
print(f"\nVolume ratio (3D/12D): {hypersphere_volume(3)/hypersphere_volume(12):.0f}x")

# Demonstrate SVM performance degradation
dimensions_test = [3, 6, 9, 12]
accuracies = []

for d in dimensions_test:
    X, y = make_classification(n_samples=500, n_features=d, 
                               n_informative=min(d, 5), random_state=42)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
    
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)
    
    svm = SVC(kernel='rbf')
    svm.fit(X_train, y_train)
    accuracies.append(svm.score(X_test, y_test))

plt.figure(figsize=(8, 6))
plt.bar(range(len(dimensions_test)), accuracies, tick_label=dimensions_test, color='steelblue', edgecolor='black')
plt.xlabel('Number of Features', fontsize=12)
plt.ylabel('Accuracy', fontsize=12)
plt.title('SVM (RBF) Accuracy vs Dimensionality', fontsize=14)
plt.ylim(0, 1.1)
for i, acc in enumerate(accuracies):
    plt.text(i, acc + 0.02, f'{acc:.3f}', ha='center')
plt.tight_layout()
plt.savefig('svm_dimensionality.png')
plt.show()

Question 30: PCA on make_blobs (4D to 3D)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs
from sklearn.decomposition import PCA
from mpl_toolkits.mplot3d import Axes3D

# Generate synthetic dataset with 4 features and 4 centers
X, y = make_blobs(n_samples=300, n_features=4, centers=4, cluster_std=1.5, random_state=42)

print("Original Dataset Shape:", X.shape)
print("Number of clusters:", len(np.unique(y)))

# Apply PCA to reduce to 3 dimensions
pca = PCA(n_components=3)
X_pca = pca.fit_transform(X)

print("\nPCA Results:")
print(f"Transformed Shape: {X_pca.shape}")
print(f"Explained Variance Ratio: {pca.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(pca.explained_variance_ratio_):.4f}")

# Visualization - Side by side 3D plots
fig = plt.figure(figsize=(16, 6))

# Plot 1: Original 3D representation (using first 3 features)
ax1 = fig.add_subplot(121, projection='3d')
scatter1 = ax1.scatter(X[:, 0], X[:, 1], X[:, 2], c=y, cmap='viridis', s=50, alpha=0.7)
ax1.set_xlabel('Feature 1')
ax1.set_ylabel('Feature 2')
ax1.set_zlabel('Feature 3')
ax1.set_title('Original Data (First 3 of 4 Features)', fontsize=14)
plt.colorbar(scatter1, ax=ax1, label='Cluster')

# Plot 2: PCA-reduced 3D representation
ax2 = fig.add_subplot(122, projection='3d')
scatter2 = ax2.scatter(X_pca[:, 0], X_pca[:, 1], X_pca[:, 2], c=y, cmap='viridis', s=50, alpha=0.7)
ax2.set_xlabel(f'PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)')
ax2.set_ylabel(f'PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)')
ax2.set_zlabel(f'PC3 ({pca.explained_variance_ratio_[2]*100:.1f}%)')
ax2.set_title('PCA-Reduced 3D Data', fontsize=14)
plt.colorbar(scatter2, ax=ax2, label='Cluster')

plt.tight_layout()
plt.savefig('pca_make_blobs_3d.png', dpi=300)
plt.show()

# Print variance explained
print("\nVariance Explained by Each Component:")
for i, var in enumerate(pca.explained_variance_ratio_):
    print(f"PC{i+1}: {var*100:.2f}%")

Question 31: Random Forest on Breast Cancer Dataset

Python

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.metrics import confusion_matrix, classification_report
import matplotlib.pyplot as plt
import seaborn as sns

# Load Breast Cancer dataset
data = load_breast_cancer()
X = data.data
y = data.target

print("=" * 60)
print("BREAST CANCER CLASSIFICATION USING RANDOM FOREST")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Features: {data.feature_names[:5]}... (total {len(data.feature_names)})")
print(f"Classes: {data.target_names}")
print(f"\nClass Distribution:")
print(f"  Malignant (0): {sum(y == 0)}")
print(f"  Benign (1): {sum(y == 1)}")

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Train Random Forest
rf_classifier = RandomForestClassifier(n_estimators=100, random_state=42)
rf_classifier.fit(X_train, y_train)

# Predictions
y_pred = rf_classifier.predict(X_test)

# Evaluation metrics
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE METRICS")
print("-" * 40)
print(f"Accuracy:  {accuracy:.4f}")
print(f"Precision: {precision:.4f}")
print(f"Recall:    {recall:.4f}")
print(f"F1-Score:  {f1:.4f}")

print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=data.target_names))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
            xticklabels=data.target_names, yticklabels=data.target_names)
axes[0].set_title('Confusion Matrix', fontsize=14)
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# Feature Importance (Top 10)
feature_importance = rf_classifier.feature_importances_
indices = np.argsort(feature_importance)[-10:]
axes[1].barh(range(10), feature_importance[indices], color='steelblue', edgecolor='black')
axes[1].set_yticks(range(10))
axes[1].set_yticklabels([data.feature_names[i] for i in indices])
axes[1].set_xlabel('Importance')
axes[1].set_title('Top 10 Feature Importances', fontsize=14)

plt.tight_layout()
plt.savefig('breast_cancer_rf.png', dpi=300)
plt.show()

Question 32: k-NN on Wine Dataset

Python

import numpy as np
import pandas as pd
from sklearn.datasets import load_wine
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.metrics import confusion_matrix, classification_report
import matplotlib.pyplot as plt
import seaborn as sns

# Load Wine dataset
wine = load_wine()
X = wine.data
y = wine.target

# Create DataFrame for exploration
df = pd.DataFrame(X, columns=wine.feature_names)
df['target'] = y

print("=" * 60)
print("WINE CLASSIFICATION USING k-NN")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"\nFirst few rows:")
print(df.head())

print(f"\nClass Distribution:")
print(df['target'].value_counts().sort_index())

# Check for missing values
print(f"\nMissing Values: {df.isnull().sum().sum()}")

# Visualize feature distributions
fig, axes = plt.subplots(2, 2, figsize=(12, 10))

# Feature histograms (sample features)
features_to_plot = ['alcohol', 'malic_acid', 'flavanoids', 'color_intensity']
for idx, feature in enumerate(features_to_plot):
    row, col = idx // 2, idx % 2
    for target in range(3):
        subset = df[df['target'] == target][feature]
        axes[row, col].hist(subset, alpha=0.5, label=f'Class {target}', bins=15)
    axes[row, col].set_xlabel(feature)
    axes[row, col].set_ylabel('Frequency')
    axes[row, col].legend()
    axes[row, col].set_title(f'Distribution of {feature}')

plt.tight_layout()
plt.savefig('wine_feature_distributions.png')
plt.show()

# Normalize/Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Split data (80% train, 20% test)
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)

# Train k-NN classifier
knn = KNeighborsClassifier(n_neighbors=5)
knn.fit(X_train, y_train)

# Predictions
y_pred = knn.predict(X_test)

# Evaluation metrics
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, average=None)
recall = recall_score(y_test, y_pred, average=None)
f1 = f1_score(y_test, y_pred, average=None)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE METRICS")
print("-" * 40)
print(f"Overall Accuracy: {accuracy:.4f}")

print(f"\nPer-Class Metrics:")
for i in range(3):
    print(f"  Class {i}: Precision={precision[i]:.4f}, Recall={recall[i]:.4f}, F1={f1[i]:.4f}")

print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=['Class 0', 'Class 1', 'Class 2']))

# Confusion Matrix
plt.figure(figsize=(8, 6))
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Greens',
            xticklabels=['Class 0', 'Class 1', 'Class 2'],
            yticklabels=['Class 0', 'Class 1', 'Class 2'])
plt.title('Confusion Matrix - k-NN Wine Classification', fontsize=14)
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.tight_layout()
plt.savefig('wine_knn_confusion.png')
plt.show()

Question 33: PCA on Iris Dataset (Detailed)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

# Load Iris dataset
iris = load_iris()
X = iris.data
y = iris.target

print("=" * 60)
print("PCA ON IRIS DATASET")
print("=" * 60)

# Explore dataset structure
print(f"\nDataset Shape: {X.shape}")
print(f"Features: {iris.feature_names}")
print(f"Target Classes: {iris.target_names}")

print(f"\nFirst 5 rows:")
for i in range(5):
    print(f"  Sample {i+1}: {X[i]} -> {iris.target_names[y[i]]}")

# Standardize the data
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Initialize PCA with 2 components
pca = PCA(n_components=2)

# Fit and transform
X_pca = pca.fit_transform(X_scaled)

print(f"\nPCA Results:")
print(f"Transformed Shape: {X_pca.shape}")
print(f"Explained Variance Ratio: {pca.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(pca.explained_variance_ratio_)*100:.2f}%")

# Component loadings
print(f"\nPCA Component Loadings:")
for i, feature in enumerate(iris.feature_names):
    print(f"  {feature}: PC1={pca.components_[0, i]:.4f}, PC2={pca.components_[1, i]:.4f}")

# Visualization
plt.figure(figsize=(10, 8))

colors = ['red', 'green', 'blue']
markers = ['o', 's', '^']

for i, (species, color, marker) in enumerate(zip(iris.target_names, colors, markers)):
    mask = (y == i)
    plt.scatter(X_pca[mask, 0], X_pca[mask, 1], 
                c=color, marker=marker, s=100, 
                alpha=0.7, edgecolors='black', label=species)

plt.xlabel(f'Principal Component 1 ({pca.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
plt.ylabel(f'Principal Component 2 ({pca.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
plt.title('PCA of Iris Dataset', fontsize=14)
plt.legend(title='Species', fontsize=10)
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('pca_iris_detailed.png', dpi=300)
plt.show()

Question 34: LDA on Iris Dataset

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.preprocessing import StandardScaler

# Load Iris dataset
iris = load_iris()
X = iris.data
y = iris.target

print("=" * 60)
print("LDA ON IRIS DATASET")
print("=" * 60)

# Explore dataset structure
print(f"\nDataset Shape: {X.shape}")
print(f"Features: {iris.feature_names}")
print(f"Target Classes: {iris.target_names}")

print(f"\nFirst 5 rows:")
for i in range(5):
    print(f"  Sample {i+1}: {X[i]} -> {iris.target_names[y[i]]}")

# Initialize LDA with 2 components
lda = LinearDiscriminantAnalysis(n_components=2)

# Fit and transform
X_lda = lda.fit_transform(X, y)

print(f"\nLDA Results:")
print(f"Transformed Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(lda.explained_variance_ratio_)*100:.2f}%")

# Visualization
plt.figure(figsize=(10, 8))

colors = ['red', 'green', 'blue']
markers = ['o', 's', '^']

for i, (species, color, marker) in enumerate(zip(iris.target_names, colors, markers)):
    mask = (y == i)
    plt.scatter(X_lda[mask, 0], X_lda[mask, 1], 
                c=color, marker=marker, s=100, 
                alpha=0.7, edgecolors='black', label=species)

plt.xlabel(f'LD1 ({lda.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
plt.ylabel(f'LD2 ({lda.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
plt.title('LDA of Iris Dataset', fontsize=14)
plt.legend(title='Species', fontsize=10)
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('lda_iris.png', dpi=300)
plt.show()

print("\nLDA shows better class separation compared to PCA because it")
print("maximizes between-class variance while minimizing within-class variance.")

Question 35: LDA on Wine Dataset

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_wine
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report

# Load Wine dataset
wine = load_wine()
X = wine.data
y = wine.target

print("=" * 60)
print("LDA ON WINE DATASET")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Number of Features: {len(wine.feature_names)}")
print(f"Number of Classes: {len(np.unique(y))}")

# Apply LDA for dimensionality reduction (to 2D)
lda = LinearDiscriminantAnalysis(n_components=2)
X_lda = lda.fit_transform(X, y)

print(f"\nLDA Results:")
print(f"Transformed Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")

# Visualization
plt.figure(figsize=(10, 8))

colors = ['red', 'green', 'blue']
for i, color in enumerate(colors):
    mask = (y == i)
    plt.scatter(X_lda[mask, 0], X_lda[mask, 1], 
                c=color, s=80, alpha=0.7, 
                edgecolors='black', label=f'Wine Class {i}')

plt.xlabel(f'LD1 ({lda.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
plt.ylabel(f'LD2 ({lda.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
plt.title('LDA of Wine Dataset', fontsize=14)
plt.legend(title='Wine Type')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('lda_wine.png', dpi=300)
plt.show()

# Classification performance using LDA
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

lda_classifier = LinearDiscriminantAnalysis()
lda_classifier.fit(X_train, y_train)
y_pred = lda_classifier.predict(X_test)

print("\nClassification Performance:")
print(f"Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred))

Question 36: LDA with Label Encoding on Iris

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.preprocessing import LabelEncoder

# Load Iris dataset
iris = load_iris()
X = iris.data
y_original = iris.target_names[iris.target]  # Convert to string labels

print("=" * 60)
print("LDA WITH LABEL ENCODING ON IRIS DATASET")
print("=" * 60)

print(f"\nOriginal Labels (first 10): {y_original[:10]}")

# Apply Label Encoding
label_encoder = LabelEncoder()
y_encoded = label_encoder.fit_transform(y_original)

print(f"Encoded Labels (first 10): {y_encoded[:10]}")
print(f"Label Mapping: {dict(zip(label_encoder.classes_, label_encoder.transform(label_encoder.classes_)))}")

# Apply LDA
lda = LinearDiscriminantAnalysis(n_components=2)
X_lda = lda.fit_transform(X, y_encoded)

print(f"\nLDA Results:")
print(f"Transformed Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")

# Visualization
plt.figure(figsize=(10, 8))

colors = ['red', 'green', 'blue']
for i, (species, color) in enumerate(zip(label_encoder.classes_, colors)):
    mask = (y_encoded == i)
    plt.scatter(X_lda[mask, 0], X_lda[mask, 1], 
                c=color, s=100, alpha=0.7, 
                edgecolors='black', label=species)

plt.xlabel(f'LD1 ({lda.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
plt.ylabel(f'LD2 ({lda.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
plt.title('LDA of Iris Dataset (with Label Encoding)', fontsize=14)
plt.legend(title='Species')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('lda_iris_label_encoding.png', dpi=300)
plt.show()

Question 37: LDA on Breast Cancer (1D)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

# Load Breast Cancer dataset
data = load_breast_cancer()
X = data.data
y = data.target

print("=" * 60)
print("LDA ON BREAST CANCER DATASET (1D REDUCTION)")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Classes: {data.target_names}")
print(f"Class Distribution: Malignant={sum(y==0)}, Benign={sum(y==1)}")

# Initialize LDA with 1 component
lda = LinearDiscriminantAnalysis(n_components=1)

# Fit and transform
X_lda = lda.fit_transform(X, y)

print(f"\nLDA Results:")
print(f"Transformed Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")

# Visualization
plt.figure(figsize=(12, 5))

# 1D scatter plot
plt.subplot(1, 2, 1)
colors = ['red', 'green']
for i, (target_name, color) in enumerate(zip(data.target_names, colors)):
    mask = (y == i)
    plt.scatter(X_lda[mask], np.zeros_like(X_lda[mask]) + i*0.1, 
                c=color, s=50, alpha=0.5, label=target_name)
plt.xlabel('LD1', fontsize=12)
plt.title('LDA of Breast Cancer (1D)', fontsize=14)
plt.legend()
plt.yticks([])

# Histogram
plt.subplot(1, 2, 2)
for i, (target_name, color) in enumerate(zip(data.target_names, colors)):
    mask = (y == i)
    plt.hist(X_lda[mask], bins=30, alpha=0.6, color=color, label=target_name, edgecolor='black')
plt.xlabel('LD1', fontsize=12)
plt.ylabel('Frequency', fontsize=12)
plt.title('Distribution of Classes along LD1', fontsize=14)
plt.legend()

plt.tight_layout()
plt.savefig('lda_breast_cancer_1d.png', dpi=300)
plt.show()

Question 38: LDA vs PCA Comparison on Iris Dataset

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.decomposition import PCA
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.preprocessing import StandardScaler

# Load Iris dataset
iris = load_iris()
X = iris.data
y = iris.target

print("=" * 60)
print("LDA vs PCA COMPARISON ON IRIS DATASET")
print("=" * 60)

# Standardize features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply PCA
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_scaled)

# Apply LDA
lda = LinearDiscriminantAnalysis(n_components=2)
X_lda = lda.fit_transform(X_scaled, y)

print(f"\nPCA Results:")
print(f"Explained Variance Ratio: {pca.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(pca.explained_variance_ratio_)*100:.2f}%")

print(f"\nLDA Results:")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(lda.explained_variance_ratio_)*100:.2f}%")

# Visualization - Side by side comparison
fig, axes = plt.subplots(1, 2, figsize=(14, 6))

colors = ['red', 'green', 'blue']
markers = ['o', 's', '^']

# PCA Plot
for i, (species, color, marker) in enumerate(zip(iris.target_names, colors, markers)):
    mask = (y == i)
    axes[0].scatter(X_pca[mask, 0], X_pca[mask, 1], 
                    c=color, marker=marker, s=80, 
                    alpha=0.7, edgecolors='black', label=species)

axes[0].set_xlabel(f'PC1 ({pca.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
axes[0].set_ylabel(f'PC2 ({pca.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
axes[0].set_title('PCA of Iris Dataset', fontsize=14)
axes[0].legend(title='Species')
axes[0].grid(True, linestyle='--', alpha=0.5)

# LDA Plot
for i, (species, color, marker) in enumerate(zip(iris.target_names, colors, markers)):
    mask = (y == i)
    axes[1].scatter(X_lda[mask, 0], X_lda[mask, 1], 
                    c=color, marker=marker, s=80, 
                    alpha=0.7, edgecolors='black', label=species)

axes[1].set_xlabel(f'LD1 ({lda.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
axes[1].set_ylabel(f'LD2 ({lda.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
axes[1].set_title('LDA of Iris Dataset', fontsize=14)
axes[1].legend(title='Species')
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('lda_vs_pca_iris.png', dpi=300)
plt.show()

# Effectiveness Comparison
print("\n" + "=" * 60)
print("EFFECTIVENESS COMPARISON")
print("=" * 60)
print("\nPCA:")
print("  - Unsupervised method (doesn't use class labels)")
print("  - Maximizes total variance")
print("  - Good for general dimensionality reduction")

print("\nLDA:")
print("  - Supervised method (uses class labels)")
print("  - Maximizes class separability")
print("  - Better for classification tasks")

# Calculate class separability (silhouette score)
from sklearn.metrics import silhouette_score

sil_pca = silhouette_score(X_pca, y)
sil_lda = silhouette_score(X_lda, y)

print(f"\nClass Separability (Silhouette Score):")
print(f"  PCA: {sil_pca:.4f}")
print(f"  LDA: {sil_lda:.4f}")
print(f"\nLDA provides {'better' if sil_lda > sil_pca else 'worse'} class separation than PCA")

Question 39: Linear Regression for House Price Prediction with PCA Comparison

Python

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Generate synthetic house price dataset
np.random.seed(42)
n_samples = 500

data = {
    'bedrooms': np.random.randint(1, 6, n_samples),
    'bathrooms': np.random.randint(1, 4, n_samples),
    'square_feet': np.random.randint(800, 4000, n_samples),
    'lot_size': np.random.randint(2000, 15000, n_samples),
    'age': np.random.randint(0, 50, n_samples),
    'garage': np.random.randint(0, 3, n_samples),
    'location_score': np.random.uniform(1, 10, n_samples)
}

df = pd.DataFrame(data)

# Create target variable (price) based on features
df['price'] = (
    df['bedrooms'] * 25000 +
    df['bathrooms'] * 15000 +
    df['square_feet'] * 150 +
    df['lot_size'] * 10 +
    df['location_score'] * 20000 -
    df['age'] * 1000 +
    df['garage'] * 10000 +
    np.random.normal(0, 50000, n_samples)
)

print("=" * 60)
print("HOUSE PRICE PREDICTION WITH LINEAR REGRESSION")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 5 rows:")
print(df.head())

# Prepare features and target
X = df.drop('price', axis=1)
y = df['price']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Standardize features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Model 1: Linear Regression on original features
lr_original = LinearRegression()
lr_original.fit(X_train_scaled, y_train)
y_pred_original = lr_original.predict(X_test_scaled)

# Evaluate Model 1
mae_original = mean_absolute_error(y_test, y_pred_original)
mse_original = mean_squared_error(y_test, y_pred_original)
r2_original = r2_score(y_test, y_pred_original)

print("\n" + "-" * 40)
print("MODEL 1: Original Features")
print("-" * 40)
print(f"MAE:  ${mae_original:,.2f}")
print(f"MSE:  ${mse_original:,.2f}")
print(f"RMSE: ${np.sqrt(mse_original):,.2f}")
print(f"R²:   {r2_original:.4f}")

# Model 2: Linear Regression with PCA reduced features
# Apply PCA to reduce to 4 components
pca = PCA(n_components=4)
X_train_pca = pca.fit_transform(X_train_scaled)
X_test_pca = pca.transform(X_test_scaled)

lr_pca = LinearRegression()
lr_pca.fit(X_train_pca, y_train)
y_pred_pca = lr_pca.predict(X_test_pca)

# Evaluate Model 2
mae_pca = mean_absolute_error(y_test, y_pred_pca)
mse_pca = mean_squared_error(y_test, y_pred_pca)
r2_pca = r2_score(y_test, y_pred_pca)

print("\n" + "-" * 40)
print("MODEL 2: PCA Reduced Features (4 components)")
print("-" * 40)
print(f"MAE:  ${mae_pca:,.2f}")
print(f"MSE:  ${mse_pca:,.2f}")
print(f"RMSE: ${np.sqrt(mse_pca):,.2f}")
print(f"R²:   {r2_pca:.4f}")
print(f"\nVariance Explained by PCA: {sum(pca.explained_variance_ratio_)*100:.2f}%")

# Visualization
fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# Plot 1: Actual vs Predicted (Original)
axes[0, 0].scatter(y_test, y_pred_original, alpha=0.5, edgecolors='black')
axes[0, 0].plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)
axes[0, 0].set_xlabel('Actual Price ($)')
axes[0, 0].set_ylabel('Predicted Price ($)')
axes[0, 0].set_title(f'Original Features (R² = {r2_original:.4f})')

# Plot 2: Actual vs Predicted (PCA)
axes[0, 1].scatter(y_test, y_pred_pca, alpha=0.5, edgecolors='black', color='green')
axes[0, 1].plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)
axes[0, 1].set_xlabel('Actual Price ($)')
axes[0, 1].set_ylabel('Predicted Price ($)')
axes[0, 1].set_title(f'PCA Features (R² = {r2_pca:.4f})')

# Plot 3: Residuals (Original)
residuals_original = y_test - y_pred_original
axes[1, 0].scatter(y_pred_original, residuals_original, alpha=0.5, edgecolors='black')
axes[1, 0].axhline(y=0, color='r', linestyle='--')
axes[1, 0].set_xlabel('Predicted Price ($)')
axes[1, 0].set_ylabel('Residuals ($)')
axes[1, 0].set_title('Residual Plot (Original Features)')

# Plot 4: Residuals (PCA)
residuals_pca = y_test - y_pred_pca
axes[1, 1].scatter(y_pred_pca, residuals_pca, alpha=0.5, edgecolors='black', color='green')
axes[1, 1].axhline(y=0, color='r', linestyle='--')
axes[1, 1].set_xlabel('Predicted Price ($)')
axes[1, 1].set_ylabel('Residuals ($)')
axes[1, 1].set_title('Residual Plot (PCA Features)')

plt.tight_layout()
plt.savefig('house_price_prediction.png', dpi=300)
plt.show()

# Comparison
print("\n" + "=" * 60)
print("PERFORMANCE COMPARISON")
print("=" * 60)
metrics = ['MAE', 'MSE', 'R²']
original_values = [mae_original, mse_original, r2_original]
pca_values = [mae_pca, mse_pca, r2_pca]

comparison_df = pd.DataFrame({
    'Metric': metrics,
    'Original': original_values,
    'PCA': pca_values
})
print(comparison_df.to_string(index=False))

Question 40: PCA for House Price Prediction

Python

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Generate synthetic house price dataset
np.random.seed(42)
n_samples = 400

data = {
    'size_sqft': np.random.randint(500, 5000, n_samples),
    'bedrooms': np.random.randint(1, 7, n_samples),
    'bathrooms': np.random.randint(1, 5, n_samples),
    'floors': np.random.randint(1, 4, n_samples),
    'waterfront': np.random.randint(0, 2, n_samples),
    'view': np.random.randint(0, 5, n_samples),
    'condition': np.random.randint(1, 6, n_samples),
    'grade': np.random.randint(1, 13, n_samples),
    'yr_built': np.random.randint(1900, 2020, n_samples),
    'yr_renovated': np.random.randint(0, 2020, n_samples)
}

df = pd.DataFrame(data)

# Create target (price)
df['price'] = (
    df['size_sqft'] * 200 +
    df['bedrooms'] * 30000 +
    df['bathrooms'] * 20000 +
    df['grade'] * 50000 +
    df['waterfront'] * 200000 +
    df['view'] * 30000 +
    np.random.normal(0, 100000, n_samples)
)

print("=" * 60)
print("PCA FOR HOUSE PRICE PREDICTION")
print("=" * 60)
print(f"\nOriginal Dataset Shape: {df.shape}")

# Prepare data
X = df.drop('price', axis=1)
y = df['price']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Standardize
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Apply PCA
pca = PCA(n_components=5)  # Reduce from 10 to 5 features
X_train_pca = pca.fit_transform(X_train_scaled)
X_test_pca = pca.transform(X_test_scaled)

print(f"\nPCA Results:")
print(f"Original Features: {X.shape[1]}")
print(f"Reduced Features: {X_train_pca.shape[1]}")
print(f"Explained Variance Ratio: {pca.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(pca.explained_variance_ratio_)*100:.2f}%")

# Train regression model
lr = LinearRegression()
lr.fit(X_train_pca, y_train)
y_pred = lr.predict(X_test_pca)

# Evaluate
mae = mean_absolute_error(y_test, y_pred)
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE")
print("-" * 40)
print(f"MAE:  ${mae:,.2f}")
print(f"MSE:  ${mse:,.2f}")
print(f"RMSE: ${np.sqrt(mse):,.2f}")
print(f"R²:   {r2:.4f}")

# Visualization
fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# Plot 1: Explained Variance
axes[0, 0].bar(range(1, len(pca.explained_variance_ratio_) + 1), 
               pca.explained_variance_ratio_, color='steelblue', edgecolor='black')
axes[0, 0].set_xlabel('Principal Component')
axes[0, 0].set_ylabel('Explained Variance Ratio')
axes[0, 0].set_title('Variance Explained by Each Component')

# Plot 2: Cumulative Variance
cumulative_var = np.cumsum(pca.explained_variance_ratio_)
axes[0, 1].plot(range(1, len(cumulative_var) + 1), cumulative_var, 'go-', linewidth=2, markersize=10)
axes[0, 1].axhline(y=0.95, color='r', linestyle='--', label='95% threshold')
axes[0, 1].set_xlabel('Number of Components')
axes[0, 1].set_ylabel('Cumulative Explained Variance')
axes[0, 1].set_title('Cumulative Variance Explained')
axes[0, 1].legend()

# Plot 3: Actual vs Predicted
axes[1, 0].scatter(y_test, y_pred, alpha=0.6, edgecolors='black')
axes[1, 0].plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)
axes[1, 0].set_xlabel('Actual Price ($)')
axes[1, 0].set_ylabel('Predicted Price ($)')
axes[1, 0].set_title(f'Actual vs Predicted (R² = {r2:.4f})')

# Plot 4: Residuals
residuals = y_test - y_pred
axes[1, 1].hist(residuals, bins=30, color='steelblue', edgecolor='black', alpha=0.7)
axes[1, 1].axvline(x=0, color='r', linestyle='--')
axes[1, 1].set_xlabel('Residual ($)')
axes[1, 1].set_ylabel('Frequency')
axes[1, 1].set_title('Distribution of Residuals')

plt.tight_layout()
plt.savefig('pca_house_price.png', dpi=300)
plt.show()

Question 41: SVM Classifier on Social Network Ads Dataset

Python

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
import matplotlib.pyplot as plt
import seaborn as sns

# Generate synthetic Social_Network_Ads dataset
np.random.seed(42)
n_samples = 400

data = {
    'UserID': range(1, n_samples + 1),
    'Gender': np.random.choice(['Male', 'Female'], n_samples),
    'Age': np.random.randint(18, 65, n_samples),
    'EstimatedSalary': np.random.randint(15000, 150000, n_samples)
}

df = pd.DataFrame(data)

# Create Purchased column based on Age and Salary
df['Purchased'] = ((df['Age'] > 35) & (df['EstimatedSalary'] > 60000) | 
                   (df['Age'] > 45) | 
                   (df['EstimatedSalary'] > 100000)).astype(int)
# Add some noise
noise_idx = np.random.choice(n_samples, size=int(n_samples * 0.1), replace=False)
df.loc[noise_idx, 'Purchased'] = 1 - df.loc[noise_idx, 'Purchased']

print("=" * 60)
print("SVM CLASSIFIER ON SOCIAL NETWORK ADS DATASET")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare features (Age and EstimatedSalary)
X = df[['Age', 'EstimatedSalary']].values
y = df['Purchased'].values

# Split data (30% test)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Model 1: Linear Kernel SVM
svm_linear = SVC(kernel='linear', C=1.0, random_state=42)
svm_linear.fit(X_train_scaled, y_train)
y_pred_linear = svm_linear.predict(X_test_scaled)
accuracy_linear = accuracy_score(y_test, y_pred_linear)

# Model 2: RBF Kernel SVM
svm_rbf = SVC(kernel='rbf', C=1.0, gamma=0.5, random_state=42)
svm_rbf.fit(X_train_scaled, y_train)
y_pred_rbf = svm_rbf.predict(X_test_scaled)
accuracy_rbf = accuracy_score(y_test, y_pred_rbf)

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"\nLinear Kernel SVM:")
print(f"  Test Accuracy: {accuracy_linear:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred_linear))

print(f"\nRBF Kernel SVM (C=1.0, gamma=0.5):")
print(f"  Test Accuracy: {accuracy_rbf:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred_rbf))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Linear Kernel Confusion Matrix
cm_linear = confusion_matrix(y_test, y_pred_linear)
sns.heatmap(cm_linear, annot=True, fmt='d', cmap='Blues', ax=axes[0])
axes[0].set_title(f'Linear SVM (Accuracy: {accuracy_linear:.4f})')
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# RBF Kernel Confusion Matrix
cm_rbf = confusion_matrix(y_test, y_pred_rbf)
sns.heatmap(cm_rbf, annot=True, fmt='d', cmap='Greens', ax=axes[1])
axes[1].set_title(f'RBF SVM (Accuracy: {accuracy_rbf:.4f})')
axes[1].set_xlabel('Predicted')
axes[1].set_ylabel('Actual')

plt.tight_layout()
plt.savefig('svm_social_network_ads.png', dpi=300)
plt.show()

Question 42: SVM on Breast Cancer Dataset

Python

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

# Load Breast Cancer dataset
data = load_breast_cancer()
X = data.data
y = data.target

print("=" * 60)
print("SVM CLASSIFIER FOR BREAST CANCER PREDICTION")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Features (30): {data.feature_names[:5]}...")
print(f"Classes: {data.target_names}")
print(f"\nClass Distribution:")
print(f"  Malignant (0): {sum(y == 0)}")
print(f"  Benign (1): {sum(y == 1)}")

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Train SVC Model
svm_classifier = SVC(kernel='rbf', C=1.0, gamma='scale', random_state=42)
svm_classifier.fit(X_train_scaled, y_train)

# Predictions
y_pred = svm_classifier.predict(X_test_scaled)

# Evaluation
accuracy = accuracy_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE")
print("-" * 40)
print(f"Accuracy: {accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=data.target_names))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
            xticklabels=data.target_names, yticklabels=data.target_names)
axes[0].set_title('Confusion Matrix', fontsize=14)
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# PCA for visualization
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
X_test_pca = pca.fit_transform(X_test_scaled)

# Plot decision results
colors = ['red' if pred == 0 else 'green' for pred in y_pred]
axes[1].scatter(X_test_pca[:, 0], X_test_pca[:, 1], c=colors, s=50, alpha=0.7, edgecolors='black')
axes[1].set_xlabel('PC1')
axes[1].set_ylabel('PC2')
axes[1].set_title('SVM Predictions (PCA visualization)')

# Add legend
from matplotlib.patches import Patch
legend_elements = [Patch(facecolor='red', label='Malignant'),
                   Patch(facecolor='green', label='Benign')]
axes[1].legend(handles=legend_elements)

plt.tight_layout()
plt.savefig('svm_breast_cancer.png', dpi=300)
plt.show()

Question 43: SVM with StandardScaler and Kernel Tuning

Python

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, classification_report
import matplotlib.pyplot as plt

# Load dataset
data = load_breast_cancer()
X = data.data
y = data.target

print("=" * 60)
print("SVM WITH PREPROCESSING AND KERNEL TUNING")
print("=" * 60)

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Preprocessing using StandardScaler
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

print("\nData Preprocessing:")
print(f"  Original mean: {X_train.mean():.4f}")
print(f"  Scaled mean: {X_train_scaled.mean():.6f}")
print(f"  Scaled std: {X_train_scaled.std():.4f}")

# Kernel parameter tuning
param_grid = {
    'kernel': ['linear', 'rbf', 'poly'],
    'C': [0.1, 1, 10],
    'gamma': ['scale', 'auto']
}

# Grid Search
svm = SVC(random_state=42)
grid_search = GridSearchCV(svm, param_grid, cv=5, scoring='accuracy', verbose=1)
grid_search.fit(X_train_scaled, y_train)

print("\n" + "-" * 40)
print("GRID SEARCH RESULTS")
print("-" * 40)
print(f"Best Parameters: {grid_search.best_params_}")
print(f"Best CV Score: {grid_search.best_score_:.4f}")

# Evaluate best model
best_model = grid_search.best_estimator_
y_pred = best_model.predict(X_test_scaled)
accuracy = accuracy_score(y_test, y_pred)

print(f"\nTest Accuracy: {accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=data.target_names))

# Compare different kernels
kernels = ['linear', 'rbf', 'poly']
kernel_accuracies = []

for kernel in kernels:
    svm_temp = SVC(kernel=kernel, random_state=42)
    svm_temp.fit(X_train_scaled, y_train)
    acc = svm_temp.score(X_test_scaled, y_test)
    kernel_accuracies.append(acc)
    print(f"{kernel.upper()} Kernel Accuracy: {acc:.4f}")

# Visualization
plt.figure(figsize=(10, 6))
bars = plt.bar(kernels, kernel_accuracies, color=['blue', 'green', 'orange'], edgecolor='black')
plt.xlabel('Kernel Type', fontsize=12)
plt.ylabel('Accuracy', fontsize=12)
plt.title('SVM Performance with Different Kernels', fontsize=14)
plt.ylim(0, 1.1)

for bar, acc in zip(bars, kernel_accuracies):
    plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, 
             f'{acc:.4f}', ha='center', fontsize=11)

plt.tight_layout()
plt.savefig('svm_kernel_comparison.png', dpi=300)
plt.show()

Question 44: Naive Bayes on Breast Cancer Dataset

Python

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

# Load Breast Cancer dataset
data = load_breast_cancer()
X = data.data
y = data.target

print("=" * 60)
print("NAIVE BAYES CLASSIFIER ON BREAST CANCER DATASET")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"All features are continuous variables")
print(f"\nFeatures: {data.feature_names[:5]}... (total: {len(data.feature_names)})")

# Split data for training and testing
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

print(f"\nTraining samples: {len(X_train)}")
print(f"Testing samples: {len(X_test)}")

# Build Gaussian Naive Bayes model
gnb = GaussianNB()
gnb.fit(X_train, y_train)

# Predict labels of test data
y_pred = gnb.predict(X_test)

# Evaluation
accuracy = accuracy_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE")
print("-" * 40)
print(f"Accuracy: {accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=data.target_names))

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
print("\nConfusion Matrix:")
print(cm)

# Visualization
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Purples',
            xticklabels=data.target_names, yticklabels=data.target_names)
plt.title('Gaussian Naive Bayes - Confusion Matrix', fontsize=14)
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.tight_layout()
plt.savefig('naive_bayes_breast_cancer.png', dpi=300)
plt.show()

Question 45: Gaussian Naive Bayes for Diabetes Prediction (PIMA Dataset)

Python

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

# Generate PIMA diabetes-like dataset
np.random.seed(42)
n_samples = 768

data = {
    'Pregnancies': np.random.randint(0, 17, n_samples),
    'OGTT': np.random.randint(0, 200, n_samples),
    'BloodPressure': np.random.randint(0, 122, n_samples),
    'SkinThickness': np.random.randint(0, 99, n_samples),
    'Insulin': np.random.randint(0, 846, n_samples),
    'BMI': np.random.uniform(0, 67.1, n_samples),
    'PedigreeFunction': np.random.uniform(0.078, 2.42, n_samples),
    'Age': np.random.randint(21, 81, n_samples)
}

df = pd.DataFrame(data)

# Create target variable
df['Diabetes'] = ((df['OGTT'] > 140) | (df['BMI'] > 30) | 
                  (df['Age'] > 45) & (df['Pregnancies'] > 3)).astype(int)
# Add noise
noise_idx = np.random.choice(n_samples, size=int(n_samples * 0.15), replace=False)
df.loc[noise_idx, 'Diabetes'] = 1 - df.loc[noise_idx, 'Diabetes']

print("=" * 60)
print("GAUSSIAN NAIVE BAYES FOR DIABETES PREDICTION")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")

# Summary statistics
print("\nSummary Statistics:")
print(df.describe())

print(f"\nTarget Distribution:")
print(df['Diabetes'].value_counts())

# Prepare data
X = df.drop('Diabetes', axis=1)
y = df['Diabetes']

# Split data (80% train, 20% test)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Build Gaussian Naive Bayes model
gnb = GaussianNB()
gnb.fit(X_train, y_train)

# Predict
y_pred = gnb.predict(X_test)

# Evaluation
accuracy = accuracy_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE")
print("-" * 40)
print(f"Accuracy: {accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=['No Diabetes', 'Diabetes']))

# Discussion on biases
print("\n" + "-" * 40)
print("DATASET BIASES")
print("-" * 40)
print("1. Gender Bias: Dataset only includes females (Pima Indians)")
print("2. Ethnic Bias: Only Pima Indian population represented")
print("3. Age Bias: Only includes individuals 21+ years old")
print("4. Missing Values: Some features may have missing values coded as 0")
print("\nThese biases may affect model generalization to other populations")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
            xticklabels=['No Diabetes', 'Diabetes'],
            yticklabels=['No Diabetes', 'Diabetes'])
axes[0].set_title('Confusion Matrix', fontsize=14)
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# Feature distributions by class
feature_to_plot = 'OGTT'
for label in [0, 1]:
    subset = df[df['Diabetes'] == label][feature_to_plot]
    axes[1].hist(subset, alpha=0.5, bins=30, 
                 label='No Diabetes' if label == 0 else 'Diabetes')
axes[1].set_xlabel(feature_to_plot)
axes[1].set_ylabel('Frequency')
axes[1].set_title(f'Distribution of {feature_to_plot} by Class')
axes[1].legend()

plt.tight_layout()
plt.savefig('naive_bayes_diabetes.png', dpi=300)
plt.show()

Question 46: Naive Bayes for Sentiment Analysis (IMDB Dataset)

Python

import numpy as np
import pandas as pd
import re
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

# Generate synthetic IMDB-like dataset
np.random.seed(42)

positive_reviews = [
    "This movie was amazing! Great acting and storyline.",
    "Absolutely loved it! Best film I've seen this year.",
    "Wonderful performances by the entire cast.",
    "A masterpiece of cinema. Highly recommended!",
    "Incredible movie with stunning visuals.",
    "Brilliant screenplay and direction.",
    "One of the best movies ever made!",
    "Outstanding film that exceeded expectations.",
    "A beautiful and touching story.",
    "Perfect entertainment from start to finish."
] * 50

negative_reviews = [
    "Terrible movie. Waste of time and money.",
    "The worst film I have ever seen.",
    "Boring and predictable storyline.",
    "Poor acting and weak script.",
    "Completely disappointing experience.",
    "Avoid this movie at all costs.",
    "Dreadful performances throughout.",
    "A complete disaster of a film.",
    "Unwatchable and tedious.",
    "Horrible movie with no redeeming qualities."
] * 50

reviews = positive_reviews + negative_reviews
sentiments = ['positive'] * 500 + ['negative'] * 500

# Shuffle
indices = np.random.permutation(len(reviews))
reviews = [reviews[i] for i in indices]
sentiments = [sentiments[i] for i in indices]

df = pd.DataFrame({'Review': reviews, 'Sentiment': sentiments})

print("=" * 60)
print("NAIVE BAYES FOR SENTIMENT ANALYSIS")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nSample Reviews:")
print(df.head())

print(f"\nSentiment Distribution:")
print(df['Sentiment'].value_counts())

# Text preprocessing function
def clean_text(text):
    text = text.lower()
    text = re.sub(r'[^\w\s]', '', text)
    return text

df['Cleaned_Review'] = df['Review'].apply(clean_text)

# Feature extraction using TfidfVectorizer
tfidf = TfidfVectorizer(max_features=1000, stop_words='english')
X = tfidf.fit_transform(df['Cleaned_Review'])
y = (df['Sentiment'] == 'positive').astype(int)

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Build Multinomial Naive Bayes model
mnb = MultinomialNB()
mnb.fit(X_train, y_train)

# Predict
y_pred = mnb.predict(X_test)

# Evaluation
accuracy = accuracy_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE")
print("-" * 40)
print(f"Accuracy: {accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=['Negative', 'Positive']))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='RdYlGn', ax=axes[0],
            xticklabels=['Negative', 'Positive'],
            yticklabels=['Negative', 'Positive'])
axes[0].set_title('Confusion Matrix', fontsize=14)
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# Top features
feature_names = tfidf.get_feature_names_out()
log_prob_diff = mnb.feature_log_prob_[1] - mnb.feature_log_prob_[0]
top_positive_idx = np.argsort(log_prob_diff)[-10:]
top_negative_idx = np.argsort(log_prob_diff)[:10]

top_words = [feature_names[i] for i in np.concatenate([top_negative_idx, top_positive_idx])]
top_scores = np.concatenate([log_prob_diff[top_negative_idx], log_prob_diff[top_positive_idx]])

colors = ['red']*10 + ['green']*10
axes[1].barh(top_words, top_scores, color=colors)
axes[1].set_xlabel('Log Probability Difference')
axes[1].set_title('Top Predictive Words')
axes[1].axvline(x=0, color='black', linestyle='--')

plt.tight_layout()
plt.savefig('naive_bayes_sentiment.png', dpi=300)
plt.show()

Question 47: Naive Bayes for 20 Newsgroups Classification

Python

import numpy as np
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report
import matplotlib.pyplot as plt

# Load 20 newsgroups dataset (all categories)
print("Loading 20 Newsgroups dataset...")
newsgroups = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))

X = newsgroups.data
y = newsgroups.target
target_names = newsgroups.target_names

print("=" * 60)
print("NAIVE BAYES FOR 20 NEWSGROUPS CLASSIFICATION")
print("=" * 60)
print(f"\nTotal Documents: {len(X)}")
print(f"Number of Categories: {len(target_names)}")
print(f"\nCategories:")
for i, name in enumerate(target_names):
    print(f"  {i}: {name}")

# Preprocess text using CountVectorizer
vectorizer = CountVectorizer(max_features=10000, stop_words='english')
X_counts = vectorizer.fit_transform(X)

print(f"\nVocabulary Size: {len(vectorizer.get_feature_names_out())}")
print(f"Feature Matrix Shape: {X_counts.shape}")

# Split data
X_train, X_test, y_train, y_test = train_test_split(X_counts, y, test_size=0.2, random_state=42)

# Build Multinomial Naive Bayes model
mnb = MultinomialNB(alpha=0.1)
mnb.fit(X_train, y_train)

# Predict
y_pred = mnb.predict(X_test)

# Evaluation
accuracy = accuracy_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE")
print("-" * 40)
print(f"Accuracy: {accuracy:.4f}")
print("\nClassification Report (sample categories):")
print(classification_report(y_test, y_pred, target_names=target_names, zero_division=0))

# Visualization
fig, ax = plt.subplots(figsize=(12, 6))

# Per-class accuracy
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)
per_class_accuracy = cm.diagonal() / cm.sum(axis=1)

# Sort by accuracy
sorted_idx = np.argsort(per_class_accuracy)
sorted_names = [target_names[i][:15] for i in sorted_idx]
sorted_acc = per_class_accuracy[sorted_idx]

colors = plt.cm.RdYlGn(sorted_acc)
ax.barh(sorted_names, sorted_acc, color=colors, edgecolor='black')
ax.set_xlabel('Accuracy', fontsize=12)
ax.set_title('Per-Category Classification Accuracy', fontsize=14)
ax.axvline(x=accuracy, color='red', linestyle='--', label=f'Overall: {accuracy:.4f}')
ax.legend()

plt.tight_layout()
plt.savefig('naive_bayes_20newsgroups.png', dpi=300)
plt.show()

Question 48: Logistic Regression for Customer Churn

Python

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, classification_report
import matplotlib.pyplot as plt

# Create dataset
data = {
    'Age': [25, 34, 45, 29, 50, 38, 42, 35, 48, 55],
    'Account_Length': [12, 24, 36, 18, 48, 30, 42, 24, 36, 60],
    'Monthly_Charges': [70, 90, 80, 100, 60, 80, 90, 70, 80, 100],
    'Churn': [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]
}

df = pd.DataFrame(data)

print("=" * 60)
print("LOGISTIC REGRESSION FOR CUSTOMER CHURN PREDICTION")
print("=" * 60)
print(f"\nDataset:")
print(df)

# Prepare features and target
X = df[['Age', 'Account_Length', 'Monthly_Charges']]
y = df['Churn']

# Split data (80% train)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Build Logistic Regression model
lr = LogisticRegression(random_state=42)
lr.fit(X_train_scaled, y_train)

# Predict
y_pred = lr.predict(X_test_scaled)
y_prob = lr.predict_proba(X_test_scaled)

# Evaluation
accuracy = accuracy_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"Accuracy: {accuracy:.4f}")

print("\nPredictions on Test Data:")
for i, (true, pred, prob) in enumerate(zip(y_test.values, y_pred, y_prob[:, 1])):
    print(f"  Sample {i+1}: True={true}, Predicted={pred}, P(Churn)={prob:.4f}")

print("\nCoefficients:")
for feature, coef in zip(X.columns, lr.coef_[0]):
    print(f"  {feature}: {coef:.4f}")
print(f"  Intercept: {lr.intercept_[0]:.4f}")

Question 49: Logistic Regression for Customer Churn (CSV Dataset)

Python

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

# Generate customer churn dataset
np.random.seed(42)
n_samples = 500

data = {
    'customer_id': range(1, n_samples + 1),
    'age': np.random.randint(18, 70, n_samples),
    'gender': np.random.choice(['male', 'female'], n_samples),
    'account_length': np.random.randint(1, 72, n_samples),
    'international_plan': np.random.choice(['yes', 'no'], n_samples),
    'voice_mail_plan': np.random.choice(['yes', 'no'], n_samples),
    'number_vmail_messages': np.random.randint(0, 50, n_samples),
    'total_day_calls': np.random.randint(50, 200, n_samples),
    'total_night_calls': np.random.randint(50, 200, n_samples),
    'total_intl_calls': np.random.randint(0, 20, n_samples)
}

df = pd.DataFrame(data)

# Create churn target
df['churn'] = np.where(
    (df['international_plan'] == 'yes') & (df['total_intl_calls'] < 5) |
    (df['account_length'] < 12) |
    (df['total_day_calls'] < 80),
    'yes', 'no'
)
# Add noise
noise_idx = np.random.choice(n_samples, size=int(n_samples * 0.1), replace=False)
df.loc[noise_idx, 'churn'] = df.loc[noise_idx, 'churn'].apply(lambda x: 'no' if x == 'yes' else 'yes')

print("=" * 60)
print("LOGISTIC REGRESSION FOR CUSTOMER CHURN (CSV DATASET)")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 5 rows:")
print(df.head())

print(f"\nChurn Distribution:")
print(df['churn'].value_counts())

# Encode categorical variables
le = LabelEncoder()
df['gender_encoded'] = le.fit_transform(df['gender'])
df['international_plan_encoded'] = le.fit_transform(df['international_plan'])
df['voice_mail_plan_encoded'] = le.fit_transform(df['voice_mail_plan'])
df['churn_encoded'] = le.fit_transform(df['churn'])

# Prepare features
features = ['age', 'account_length', 'gender_encoded', 'international_plan_encoded',
            'voice_mail_plan_encoded', 'number_vmail_messages', 'total_day_calls',
            'total_night_calls', 'total_intl_calls']

X = df[features]
y = df['churn_encoded']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Build model
lr = LogisticRegression(max_iter=1000, random_state=42)
lr.fit(X_train_scaled, y_train)

# Predict
y_pred = lr.predict(X_test_scaled)

# Evaluation
accuracy = accuracy_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE")
print("-" * 40)
print(f"Accuracy: {accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=['No Churn', 'Churn']))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
            xticklabels=['No Churn', 'Churn'], yticklabels=['No Churn', 'Churn'])
axes[0].set_title('Confusion Matrix', fontsize=14)
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# Feature Importance
feature_importance = np.abs(lr.coef_[0])
sorted_idx = np.argsort(feature_importance)
axes[1].barh([features[i] for i in sorted_idx], feature_importance[sorted_idx], 
             color='steelblue', edgecolor='black')
axes[1].set_xlabel('Absolute Coefficient')
axes[1].set_title('Feature Importance')

plt.tight_layout()
plt.savefig('logistic_regression_churn.png', dpi=300)
plt.show()

Question 50: Binary Classification on Iris (Iris-Virginica Detection)

Python

import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, classification_report
import matplotlib.pyplot as plt

# Load Iris dataset
iris = load_iris()
X = iris.data
y = (iris.target == 2).astype(int)  # Binary: Virginica (1) vs Others (0)

print("=" * 60)
print("BINARY CLASSIFICATION: IRIS-VIRGINICA DETECTION")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Features: {iris.feature_names}")
print(f"\nTarget Distribution:")
print(f"  Not Virginica (0): {sum(y == 0)}")
print(f"  Virginica (1): {sum(y == 1)}")

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Build Logistic Regression model
lr = LogisticRegression(random_state=42)
lr.fit(X_train_scaled, y_train)

# Predict with default threshold (0.5)
y_pred_default = lr.predict(X_test_scaled)
y_prob = lr.predict_proba(X_test_scaled)[:, 1]

# Predict with altered threshold (0.6)
threshold = 0.6
y_pred_altered = (y_prob >= threshold).astype(int)

print("\n" + "-" * 40)
print("PREDICTIONS WITH DEFAULT THRESHOLD (0.5)")
print("-" * 40)
print(f"Accuracy: {accuracy_score(y_test, y_pred_default):.4f}")
print("\nPredictions with Probabilities:")
for i in range(min(10, len(y_test))):
    print(f"  Sample {i+1}: P(Virginica)={y_prob[i]:.4f}, Predicted={y_pred_default[i]}, Actual={y_test.iloc[i] if hasattr(y_test, 'iloc') else y_test[i]}")

print("\n" + "-" * 40)
print("PREDICTIONS WITH ALTERED THRESHOLD (0.6)")
print("-" * 40)
print(f"Accuracy: {accuracy_score(y_test, y_pred_altered):.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred_altered, target_names=['Not Virginica', 'Virginica']))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Probability Distribution
axes[0].hist(y_prob[y_test == 0], bins=15, alpha=0.6, label='Not Virginica', color='blue')
axes[0].hist(y_prob[y_test == 1], bins=15, alpha=0.6, label='Virginica', color='green')
axes[0].axvline(x=0.5, color='red', linestyle='--', label='Default Threshold (0.5)')
axes[0].axvline(x=0.6, color='orange', linestyle='--', label='Altered Threshold (0.6)')
axes[0].set_xlabel('Probability of Virginica')
axes[0].set_ylabel('Frequency')
axes[0].set_title('Predicted Probability Distribution')
axes[0].legend()

# Threshold Comparison
thresholds = [0.5, 0.6]
accuracies = [accuracy_score(y_test, y_pred_default), accuracy_score(y_test, y_pred_altered)]
axes[1].bar(thresholds, accuracies, width=0.05, color=['red', 'orange'], edgecolor='black')
axes[1].set_xlabel('Threshold')
axes[1].set_ylabel('Accuracy')
axes[1].set_title('Accuracy at Different Thresholds')
axes[1].set_xticks(thresholds)

plt.tight_layout()
plt.savefig('iris_virginica_classification.png', dpi=300)
plt.show()

Question 51: Simple Linear Regression (Salary Prediction)

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt

# Generate Salary dataset
np.random.seed(42)
years_exp = np.array([1.1, 1.3, 1.5, 2.0, 2.2, 2.9, 3.0, 3.2, 3.2, 3.7, 
                      3.9, 4.0, 4.0, 4.1, 4.5, 4.9, 5.1, 5.3, 5.9, 6.0,
                      6.8, 7.1, 7.9, 8.2, 8.7, 9.0, 9.5, 9.6, 10.3, 10.5])
salary = 25000 + 9000 * years_exp + np.random.normal(0, 5000, len(years_exp))

df = pd.DataFrame({'years_of_experience': years_exp, 'salary': salary})

print("=" * 60)
print("SIMPLE LINEAR REGRESSION - SALARY PREDICTION")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare data
X = df[['years_of_experience']].values
y = df['salary'].values

# Fit Linear Regression
lr = LinearRegression()
lr.fit(X, y)

# Predictions
y_pred = lr.predict(X)

# Residuals
residuals = y - y_pred

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"Intercept (b0): ${lr.intercept_:,.2f}")
print(f"Coefficient (b1): ${lr.coef_[0]:,.2f}")
print(f"R² Score: {lr.score(X, y):.4f}")
print(f"\nRegression Equation: Salary = {lr.intercept_:.2f} + {lr.coef_[0]:.2f} × Years")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: Fitted model with data points
axes[0].scatter(X, y, color='blue', alpha=0.7, edgecolors='black', label='Data Points')
axes[0].plot(X, y_pred, color='red', linewidth=2, label='Best Fit Line')
axes[0].set_xlabel('Years of Experience', fontsize=12)
axes[0].set_ylabel('Salary ($)', fontsize=12)
axes[0].set_title('Simple Linear Regression: Salary vs Experience', fontsize=14)
axes[0].legend()
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Residual Plot
axes[1].scatter(y_pred, residuals, color='green', alpha=0.7, edgecolors='black')
axes[1].axhline(y=0, color='red', linestyle='--', linewidth=2)
axes[1].set_xlabel('Predicted Salary ($)', fontsize=12)
axes[1].set_ylabel('Residuals ($)', fontsize=12)
axes[1].set_title('Residual Plot', fontsize=14)
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('simple_linear_regression_salary.png', dpi=300)
plt.show()

Question 52: Polynomial Regression (House Prices)

Python

import numpy as np
import pandas as pd
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Generate HousePrices dataset
np.random.seed(42)
square_feet = np.linspace(500, 4000, 100)
price = 50000 + 100 * square_feet + 0.01 * square_feet**2 + np.random.normal(0, 30000, len(square_feet))

df = pd.DataFrame({'square_feet': square_feet, 'price': price})

print("=" * 60)
print("POLYNOMIAL REGRESSION - HOUSE PRICES")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare data
X = df[['square_feet']].values
y = df['price'].values

# Create polynomial features (degree 2)
poly = PolynomialFeatures(degree=2)
X_poly = poly.fit_transform(X)

# Fit Polynomial Regression
lr = LinearRegression()
lr.fit(X_poly, y)

# Predictions
y_pred = lr.predict(X_poly)
residuals = y - y_pred

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"Coefficients: {lr.coef_}")
print(f"Intercept: ${lr.intercept_:,.2f}")
print(f"R² Score: {r2_score(y, y_pred):.4f}")

# For plotting smooth curve
X_smooth = np.linspace(X.min(), X.max(), 200).reshape(-1, 1)
X_smooth_poly = poly.transform(X_smooth)
y_smooth = lr.predict(X_smooth_poly)

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: Fitted polynomial model
axes[0].scatter(X, y, color='blue', alpha=0.6, edgecolors='black', label='Data Points')
axes[0].plot(X_smooth, y_smooth, color='red', linewidth=2, label='Polynomial Fit (degree=2)')
axes[0].set_xlabel('Square Feet', fontsize=12)
axes[0].set_ylabel('Price ($)', fontsize=12)
axes[0].set_title('Polynomial Regression: House Price vs Square Feet', fontsize=14)
axes[0].legend()
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Residual Plot
axes[1].scatter(y_pred, residuals, color='green', alpha=0.6, edgecolors='black')
axes[1].axhline(y=0, color='red', linestyle='--', linewidth=2)
axes[1].set_xlabel('Predicted Price ($)', fontsize=12)
axes[1].set_ylabel('Residuals ($)', fontsize=12)
axes[1].set_title('Residual Plot', fontsize=14)
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('polynomial_regression_house.png', dpi=300)
plt.show()

Question 53: Multiple Linear Regression (Car Prices)

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Generate CarPrices dataset
np.random.seed(42)
n_samples = 100

age = np.random.randint(0, 15, n_samples)
mileage = np.random.randint(5000, 200000, n_samples)

# Price decreases with age and mileage
price = 35000 - 2000 * age - 0.05 * mileage + np.random.normal(0, 3000, n_samples)
price = np.clip(price, 2000, 50000)

df = pd.DataFrame({'age': age, 'mileage': mileage, 'price': price})

print("=" * 60)
print("MULTIPLE LINEAR REGRESSION - CAR PRICES")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare data
X = df[['age', 'mileage']].values
y = df['price'].values

# Fit Multiple Linear Regression
lr = LinearRegression()
lr.fit(X, y)

# Predictions
y_pred = lr.predict(X)
residuals = y - y_pred

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"Intercept: ${lr.intercept_:,.2f}")
print(f"Coefficient (Age): ${lr.coef_[0]:,.2f}")
print(f"Coefficient (Mileage): ${lr.coef_[1]:.4f}")
print(f"R² Score: {r2_score(y, y_pred):.4f}")
print(f"\nRegression Equation: Price = {lr.intercept_:.2f} + {lr.coef_[0]:.2f}×Age + {lr.coef_[1]:.4f}×Mileage")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: Actual vs Predicted
axes[0].scatter(y, y_pred, color='blue', alpha=0.6, edgecolors='black')
axes[0].plot([y.min(), y.max()], [y.min(), y.max()], 'r--', linewidth=2)
axes[0].set_xlabel('Actual Price ($)', fontsize=12)
axes[0].set_ylabel('Predicted Price ($)', fontsize=12)
axes[0].set_title('Actual vs Predicted Prices', fontsize=14)
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Residual Plot
axes[1].scatter(y_pred, residuals, color='green', alpha=0.6, edgecolors='black')
axes[1].axhline(y=0, color='red', linestyle='--', linewidth=2)
axes[1].set_xlabel('Predicted Price ($)', fontsize=12)
axes[1].set_ylabel('Residuals ($)', fontsize=12)
axes[1].set_title('Residual Plot', fontsize=14)
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('multiple_linear_regression_car.png', dpi=300)
plt.show()

Question 54: Linear Regression using Gradient Descent

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Generate HousePrices dataset
np.random.seed(42)
n_samples = 100

size = np.random.randint(500, 3500, n_samples)
price = 50000 + 100 * size + np.random.normal(0, 50000, n_samples)

df = pd.DataFrame({'size': size, 'price': price})

print("=" * 60)
print("LINEAR REGRESSION USING GRADIENT DESCENT")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")

# Prepare data
X = df['size'].values
y = df['price'].values

# Normalize features
X_mean = X.mean()
X_std = X.std()
X_norm = (X - X_mean) / X_std

# Initialize parameters
theta_0 = 0  # intercept
theta_1 = 0  # coefficient
learning_rate = 0.01
iterations = 1000
m = len(X)

# Gradient Descent
cost_history = []

for i in range(iterations):
    # Predictions
    y_pred = theta_0 + theta_1 * X_norm
    
    # Cost (MSE)
    cost = (1/(2*m)) * np.sum((y_pred - y)**2)
    cost_history.append(cost)
    
    # Gradients
    d_theta_0 = (1/m) * np.sum(y_pred - y)
    d_theta_1 = (1/m) * np.sum((y_pred - y) * X_norm)
    
    # Update parameters
    theta_0 = theta_0 - learning_rate * d_theta_0
    theta_1 = theta_1 - learning_rate * d_theta_1
    
    if i % 200 == 0:
        print(f"Iteration {i}: Cost = {cost:.2f}, theta_0 = {theta_0:.2f}, theta_1 = {theta_1:.2f}")

# Final predictions
y_pred_final = theta_0 + theta_1 * X_norm

# Convert back to original scale
theta_1_original = theta_1 / X_std
theta_0_original = theta_0 - theta_1_original * X_mean

print("\n" + "-" * 40)
print("FINAL RESULTS")
print("-" * 40)
print(f"Intercept (theta_0): ${theta_0_original:,.2f}")
print(f"Coefficient (theta_1): ${theta_1_original:,.2f}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: Regression Line
X_line = np.linspace(X.min(), X.max(), 100)
y_line = theta_0_original + theta_1_original * X_line

axes[0].scatter(X, y, color='blue', alpha=0.6, edgecolors='black', label='Data Points')
axes[0].plot(X_line, y_line, color='red', linewidth=2, label='Regression Line')
axes[0].set_xlabel('Size (sq ft)', fontsize=12)
axes[0].set_ylabel('Price ($)', fontsize=12)
axes[0].set_title('Linear Regression using Gradient Descent', fontsize=14)
axes[0].legend()
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Cost vs Iterations
axes[1].plot(cost_history, color='green', linewidth=1)
axes[1].set_xlabel('Iteration', fontsize=12)
axes[1].set_ylabel('Cost (MSE)', fontsize=12)
axes[1].set_title('Cost Function Convergence', fontsize=14)
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('gradient_descent_regression.png', dpi=300)
plt.show()

Question 55: Linear Regression with MSE, R², MAE (Car Prices)

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
import matplotlib.pyplot as plt

# Generate CarPrices dataset
np.random.seed(42)
n_samples = 200

horsepower = np.random.randint(100, 400, n_samples)
age = np.random.randint(0, 20, n_samples)
price = 5000 + 100 * horsepower - 1500 * age + np.random.normal(0, 5000, n_samples)
price = np.clip(price, 1000, 80000)

df = pd.DataFrame({'horsepower': horsepower, 'age': age, 'price': price})

print("=" * 60)
print("LINEAR REGRESSION WITH EVALUATION METRICS")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare data
X = df[['horsepower', 'age']]
y = df['price']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Fit model
lr = LinearRegression()
lr.fit(X_train, y_train)

# Predictions
y_pred = lr.predict(X_test)

# Evaluation metrics
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
mae = mean_absolute_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL EVALUATION")
print("-" * 40)
print(f"Mean Squared Error (MSE): ${mse:,.2f}")
print(f"Root Mean Squared Error (RMSE): ${rmse:,.2f}")
print(f"Mean Absolute Error (MAE): ${mae:,.2f}")
print(f"R² Score: {r2:.4f}")

print(f"\nCoefficients:")
print(f"  Intercept: ${lr.intercept_:,.2f}")
print(f"  Horsepower: ${lr.coef_[0]:.2f}")
print(f"  Age: ${lr.coef_[1]:.2f}")

# Visualization
fig, axes = plt.subplots(1, 3, figsize=(15, 5))

# Actual vs Predicted
axes[0].scatter(y_test, y_pred, alpha=0.6, edgecolors='black')
axes[0].plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)
axes[0].set_xlabel('Actual Price ($)')
axes[0].set_ylabel('Predicted Price ($)')
axes[0].set_title(f'Actual vs Predicted (R² = {r2:.4f})')
axes[0].grid(True, linestyle='--', alpha=0.5)

# Residual Distribution
residuals = y_test - y_pred
axes[1].hist(residuals, bins=20, color='steelblue', edgecolor='black', alpha=0.7)
axes[1].axvline(x=0, color='red', linestyle='--', lw=2)
axes[1].set_xlabel('Residual ($)')
axes[1].set_ylabel('Frequency')
axes[1].set_title('Residual Distribution')

# Metrics Comparison
metrics = ['MSE', 'RMSE', 'MAE']
values = [mse/1000, rmse/1000, mae/1000]  # Scaled for visualization
axes[2].bar(metrics, values, color=['coral', 'lightgreen', 'skyblue'], edgecolor='black')
axes[2].set_ylabel('Value (in thousands $)')
axes[2].set_title('Error Metrics Comparison')
for i, v in enumerate(values):
    axes[2].text(i, v + 0.5, f'{v*1000:.0f}', ha='center')

plt.tight_layout()
plt.savefig('linear_regression_metrics.png', dpi=300)
plt.show()

Question 56: Linear Regression for House Data with Evaluation Metrics

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
import matplotlib.pyplot as plt
import seaborn as sns

# Generate HouseData dataset
np.random.seed(42)
n_samples = 300

num_rooms = np.random.randint(1, 10, n_samples)
square_feet = np.random.randint(500, 5000, n_samples)
location = np.random.choice(['Urban', 'Suburban', 'Rural'], n_samples)

# Encode location
location_score = {'Urban': 1.2, 'Suburban': 1.0, 'Rural': 0.8}
location_multiplier = np.array([location_score[loc] for loc in location])

# Generate house price
house_price = (50000 + num_rooms * 30000 + square_feet * 100) * location_multiplier
house_price += np.random.normal(0, 30000, n_samples)
house_price = np.clip(house_price, 50000, 2000000)

df = pd.DataFrame({
    'num_rooms': num_rooms,
    'square_feet': square_feet,
    'location': location,
    'house_price': house_price
})

# Encode location for modeling
df['location_encoded'] = df['location'].map({'Rural': 0, 'Suburban': 1, 'Urban': 2})

print("=" * 60)
print("LINEAR REGRESSION FOR HOUSE PRICE PREDICTION")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

print(f"\nLocation Distribution:")
print(df['location'].value_counts())

# Prepare features
X = df[['num_rooms', 'square_feet', 'location_encoded']]
y = df['house_price']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Fit model
lr = LinearRegression()
lr.fit(X_train, y_train)

# Predictions
y_pred = lr.predict(X_test)

# Evaluation metrics
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
mae = mean_absolute_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL EVALUATION")
print("-" * 40)
print(f"Mean Squared Error (MSE): ${mse:,.2f}")
print(f"Root Mean Squared Error (RMSE): ${rmse:,.2f}")
print(f"Mean Absolute Error (MAE): ${mae:,.2f}")
print(f"R² Score: {r2:.4f}")

print(f"\nRegression Coefficients:")
print(f"  Intercept: ${lr.intercept_:,.2f}")
for feature, coef in zip(X.columns, lr.coef_):
    print(f"  {feature}: ${coef:,.2f}")

# Visualization
fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# Actual vs Predicted
axes[0, 0].scatter(y_test, y_pred, alpha=0.6, edgecolors='black')
axes[0, 0].plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)
axes[0, 0].set_xlabel('Actual House Price ($)')
axes[0, 0].set_ylabel('Predicted House Price ($)')
axes[0, 0].set_title(f'Actual vs Predicted (R² = {r2:.4f})')
axes[0, 0].grid(True, linestyle='--', alpha=0.5)

# Residual Plot
residuals = y_test - y_pred
axes[0, 1].scatter(y_pred, residuals, alpha=0.6, edgecolors='black', color='green')
axes[0, 1].axhline(y=0, color='red', linestyle='--', lw=2)
axes[0, 1].set_xlabel('Predicted Price ($)')
axes[0, 1].set_ylabel('Residuals ($)')
axes[0, 1].set_title('Residual Plot')
axes[0, 1].grid(True, linestyle='--', alpha=0.5)

# Feature Importance
importance = np.abs(lr.coef_)
axes[1, 0].barh(X.columns, importance, color='steelblue', edgecolor='black')
axes[1, 0].set_xlabel('Absolute Coefficient Value')
axes[1, 0].set_title('Feature Importance')

# Residual Histogram
axes[1, 1].hist(residuals, bins=25, color='coral', edgecolor='black', alpha=0.7)
axes[1, 1].axvline(x=0, color='red', linestyle='--', lw=2)
axes[1, 1].set_xlabel('Residual ($)')
axes[1, 1].set_ylabel('Frequency')
axes[1, 1].set_title('Residual Distribution')

plt.tight_layout()
plt.savefig('house_price_regression.png', dpi=300)
plt.show()

Question 57: Linear Regression for Employee Salary Prediction

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt

# Generate EmployeeData dataset
np.random.seed(42)
n_samples = 250

years_of_experience = np.random.uniform(0, 30, n_samples)
education_level = np.random.choice(['High School', 'Bachelor', 'Master', 'PhD'], n_samples)

# Education level multiplier
edu_multiplier = {'High School': 0.8, 'Bachelor': 1.0, 'Master': 1.3, 'PhD': 1.6}
edu_factor = np.array([edu_multiplier[e] for e in education_level])

# Generate salary
base_salary = 30000
salary = base_salary + (2500 * years_of_experience) * edu_factor
salary += np.random.normal(0, 8000, n_samples)
salary = np.clip(salary, 25000, 250000)

df = pd.DataFrame({
    'years_of_experience': years_of_experience,
    'education_level': education_level,
    'salary': salary
})

# Encode education level
le = LabelEncoder()
df['education_encoded'] = le.fit_transform(df['education_level'])

print("=" * 60)
print("LINEAR REGRESSION FOR EMPLOYEE SALARY PREDICTION")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

print(f"\nEducation Level Distribution:")
print(df['education_level'].value_counts())

print(f"\nEducation Level Encoding:")
for edu, code in zip(le.classes_, le.transform(le.classes_)):
    print(f"  {edu}: {code}")

# Prepare features
X = df[['years_of_experience', 'education_encoded']]
y = df['salary']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Fit model
lr = LinearRegression()
lr.fit(X_train, y_train)

# Predictions
y_pred = lr.predict(X_test)

# Evaluation metrics
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
mae = mean_absolute_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL EVALUATION")
print("-" * 40)
print(f"Mean Squared Error (MSE): ${mse:,.2f}")
print(f"Root Mean Squared Error (RMSE): ${rmse:,.2f}")
print(f"Mean Absolute Error (MAE): ${mae:,.2f}")
print(f"R² Score: {r2:.4f}")

print(f"\nRegression Coefficients:")
print(f"  Intercept: ${lr.intercept_:,.2f}")
print(f"  Years of Experience: ${lr.coef_[0]:,.2f} per year")
print(f"  Education Level: ${lr.coef_[1]:,.2f} per level")

# Visualization
fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# Actual vs Predicted
axes[0, 0].scatter(y_test, y_pred, alpha=0.6, edgecolors='black')
axes[0, 0].plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)
axes[0, 0].set_xlabel('Actual Salary ($)')
axes[0, 0].set_ylabel('Predicted Salary ($)')
axes[0, 0].set_title(f'Actual vs Predicted (R² = {r2:.4f})')
axes[0, 0].grid(True, linestyle='--', alpha=0.5)

# Salary vs Experience by Education
colors = {'High School': 'red', 'Bachelor': 'blue', 'Master': 'green', 'PhD': 'purple'}
for edu in df['education_level'].unique():
    subset = df[df['education_level'] == edu]
    axes[0, 1].scatter(subset['years_of_experience'], subset['salary'], 
                       alpha=0.5, label=edu, c=colors[edu])
axes[0, 1].set_xlabel('Years of Experience')
axes[0, 1].set_ylabel('Salary ($)')
axes[0, 1].set_title('Salary vs Experience by Education')
axes[0, 1].legend()
axes[0, 1].grid(True, linestyle='--', alpha=0.5)

# Residual Plot
residuals = y_test - y_pred
axes[1, 0].scatter(y_pred, residuals, alpha=0.6, edgecolors='black', color='green')
axes[1, 0].axhline(y=0, color='red', linestyle='--', lw=2)
axes[1, 0].set_xlabel('Predicted Salary ($)')
axes[1, 0].set_ylabel('Residuals ($)')
axes[1, 0].set_title('Residual Plot')
axes[1, 0].grid(True, linestyle='--', alpha=0.5)

# Average Salary by Education
avg_salary_by_edu = df.groupby('education_level')['salary'].mean().reindex(['High School', 'Bachelor', 'Master', 'PhD'])
axes[1, 1].bar(avg_salary_by_edu.index, avg_salary_by_edu.values, color='steelblue', edgecolor='black')
axes[1, 1].set_xlabel('Education Level')
axes[1, 1].set_ylabel('Average Salary ($)')
axes[1, 1].set_title('Average Salary by Education Level')
axes[1, 1].tick_params(axis='x', rotation=45)

plt.tight_layout()
plt.savefig('employee_salary_regression.png', dpi=300)
plt.show()

Question 58: Lasso Regression for Housing Data

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import Lasso
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Generate HousingData dataset
np.random.seed(42)
n_samples = 300

num_rooms = np.random.randint(1, 8, n_samples)
area = np.random.randint(500, 4000, n_samples)
age = np.random.randint(0, 50, n_samples)

# Generate price
price = 50000 + num_rooms * 25000 + area * 80 - age * 1000
price += np.random.normal(0, 30000, n_samples)
price = np.clip(price, 50000, 1000000)

df = pd.DataFrame({
    'num_rooms': num_rooms,
    'area': area,
    'age': age,
    'price': price
})

print("=" * 60)
print("LASSO REGRESSION FOR HOUSING DATA")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare data
X = df[['num_rooms', 'area', 'age']]
y = df['price']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Fit Lasso Regression
lasso = Lasso(alpha=1000, random_state=42)
lasso.fit(X_train_scaled, y_train)

# Predictions
y_pred = lasso.predict(X_test_scaled)
residuals = y_test - y_pred

# Evaluation
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"Mean Squared Error: ${mse:,.2f}")
print(f"R² Score: {r2:.4f}")

print(f"\nLasso Coefficients (scaled features):")
print(f"  Intercept: ${lasso.intercept_:,.2f}")
for feature, coef in zip(X.columns, lasso.coef_):
    print(f"  {feature}: {coef:,.2f}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: Coefficient Plot
coef_df = pd.DataFrame({
    'Feature': X.columns,
    'Coefficient': lasso.coef_
}).sort_values('Coefficient')

colors = ['red' if c < 0 else 'green' for c in coef_df['Coefficient']]
axes[0].barh(coef_df['Feature'], coef_df['Coefficient'], color=colors, edgecolor='black')
axes[0].axvline(x=0, color='black', linestyle='-', lw=1)
axes[0].set_xlabel('Coefficient Value')
axes[0].set_title('Lasso Regression Coefficients')
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Residual Plot
axes[1].scatter(y_pred, residuals, alpha=0.6, edgecolors='black', color='steelblue')
axes[1].axhline(y=0, color='red', linestyle='--', lw=2)
axes[1].set_xlabel('Predicted Price ($)')
axes[1].set_ylabel('Residuals ($)')
axes[1].set_title('Residual Plot')
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('lasso_regression_housing.png', dpi=300)
plt.show()

# Compare different alpha values
alphas = [0.1, 1, 10, 100, 1000, 10000]
coefs = []

for alpha in alphas:
    lasso_temp = Lasso(alpha=alpha, random_state=42)
    lasso_temp.fit(X_train_scaled, y_train)
    coefs.append(lasso_temp.coef_)

# Plot coefficient paths
plt.figure(figsize=(10, 6))
coefs = np.array(coefs)
for i, feature in enumerate(X.columns):
    plt.plot(alphas, coefs[:, i], 'o-', label=feature, linewidth=2)

plt.xscale('log')
plt.xlabel('Alpha (Regularization Strength)')
plt.ylabel('Coefficient Value')
plt.title('Lasso Coefficient Path')
plt.legend()
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('lasso_coefficient_path.png', dpi=300)
plt.show()

Question 59: Ridge Regression for Diabetes Dataset

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import Ridge
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Generate Diabetes-like dataset
np.random.seed(42)
n_samples = 442

# Features
age = np.random.uniform(20, 80, n_samples)
sex = np.random.randint(0, 2, n_samples)
bmi = np.random.uniform(18, 40, n_samples)
bp = np.random.uniform(60, 120, n_samples)
s1 = np.random.uniform(100, 300, n_samples)
s2 = np.random.uniform(50, 200, n_samples)
s3 = np.random.uniform(20, 100, n_samples)
s4 = np.random.uniform(2, 10, n_samples)
s5 = np.random.uniform(3, 7, n_samples)
s6 = np.random.uniform(50, 150, n_samples)

# Target: diabetes progression
diabetes_progression = (
    25 + 0.5 * age + 5 * bmi + 0.3 * bp + 
    0.1 * s1 - 0.05 * s2 + 0.2 * s3 + 
    10 * s4 + 20 * s5 + 0.1 * s6 +
    np.random.normal(0, 50, n_samples)
)

df = pd.DataFrame({
    'age': age, 'sex': sex, 'bmi': bmi, 'bp': bp,
    's1': s1, 's2': s2, 's3': s3, 's4': s4, 's5': s5, 's6': s6,
    'diabetes_progression': diabetes_progression
})

print("=" * 60)
print("RIDGE REGRESSION FOR DIABETES PROGRESSION")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare data
X = df.drop('diabetes_progression', axis=1)
y = df['diabetes_progression']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Fit Ridge Regression
ridge = Ridge(alpha=1.0, random_state=42)
ridge.fit(X_train_scaled, y_train)

# Predictions
y_pred = ridge.predict(X_test_scaled)
residuals = y_test - y_pred

# Evaluation
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"Mean Squared Error: {mse:.2f}")
print(f"R² Score: {r2:.4f}")

print(f"\nRidge Coefficients:")
print(f"  Intercept: {ridge.intercept_:.2f}")
for feature, coef in zip(X.columns, ridge.coef_):
    print(f"  {feature}: {coef:.4f}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: Coefficient Plot
coef_df = pd.DataFrame({
    'Feature': X.columns,
    'Coefficient': ridge.coef_
}).sort_values('Coefficient')

colors = ['red' if c < 0 else 'green' for c in coef_df['Coefficient']]
axes[0].barh(coef_df['Feature'], coef_df['Coefficient'], color=colors, edgecolor='black')
axes[0].axvline(x=0, color='black', linestyle='-', lw=1)
axes[0].set_xlabel('Coefficient Value')
axes[0].set_title('Ridge Regression Coefficients')
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Residual Plot
axes[1].scatter(y_pred, residuals, alpha=0.6, edgecolors='black', color='steelblue')
axes[1].axhline(y=0, color='red', linestyle='--', lw=2)
axes[1].set_xlabel('Predicted Diabetes Progression')
axes[1].set_ylabel('Residuals')
axes[1].set_title('Residual Plot')
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('ridge_regression_diabetes.png', dpi=300)
plt.show()

Question 60: Lasso Regression for Wine Quality

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import Lasso
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Generate Wine Quality dataset
np.random.seed(42)
n_samples = 1600

# Chemical properties
fixed_acidity = np.random.uniform(4, 16, n_samples)
volatile_acidity = np.random.uniform(0.1, 1.6, n_samples)
citric_acid = np.random.uniform(0, 1, n_samples)
residual_sugar = np.random.uniform(0.9, 16, n_samples)
chlorides = np.random.uniform(0.01, 0.6, n_samples)
free_sulfur_dioxide = np.random.uniform(1, 72, n_samples)
total_sulfur_dioxide = np.random.uniform(6, 289, n_samples)
density = np.random.uniform(0.99, 1.04, n_samples)
pH = np.random.uniform(2.7, 4.0, n_samples)
sulphates = np.random.uniform(0.3, 2.0, n_samples)
alcohol = np.random.uniform(8, 15, n_samples)

# Quality (target)
quality = (
    3 + 0.1 * alcohol - 2 * volatile_acidity + 
    0.5 * citric_acid + 0.3 * sulphates - 
    0.01 * total_sulfur_dioxide +
    np.random.normal(0, 0.5, n_samples)
)
quality = np.clip(quality, 3, 9).round()

df = pd.DataFrame({
    'fixed_acidity': fixed_acidity,
    'volatile_acidity': volatile_acidity,
    'citric_acid': citric_acid,
    'residual_sugar': residual_sugar,
    'chlorides': chlorides,
    'free_sulfur_dioxide': free_sulfur_dioxide,
    'total_sulfur_dioxide': total_sulfur_dioxide,
    'density': density,
    'pH': pH,
    'sulphates': sulphates,
    'alcohol': alcohol,
    'quality': quality
})

print("=" * 60)
print("LASSO REGRESSION FOR WINE QUALITY")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare data
X = df.drop('quality', axis=1)
y = df['quality']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Fit Lasso Regression
lasso = Lasso(alpha=0.01, random_state=42)
lasso.fit(X_train_scaled, y_train)

# Predictions
y_pred = lasso.predict(X_test_scaled)
residuals = y_test - y_pred

# Evaluation
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"Mean Squared Error: {mse:.4f}")
print(f"R² Score: {r2:.4f}")

print(f"\nLasso Coefficients:")
print(f"  Intercept: {lasso.intercept_:.4f}")
for feature, coef in zip(X.columns, lasso.coef_):
    status = "(Selected)" if abs(coef) > 0.001 else "(Shrunk to ~0)"
    print(f"  {feature}: {coef:.4f} {status}")

# Count non-zero coefficients
n_selected = sum(abs(coef) > 0.001 for coef in lasso.coef_)
print(f"\nFeatures selected by Lasso: {n_selected}/{len(X.columns)}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: Coefficient Plot
coef_df = pd.DataFrame({
    'Feature': X.columns,
    'Coefficient': lasso.coef_
}).sort_values('Coefficient')

colors = ['red' if c < 0 else 'green' for c in coef_df['Coefficient']]
axes[0].barh(coef_df['Feature'], coef_df['Coefficient'], color=colors, edgecolor='black')
axes[0].axvline(x=0, color='black', linestyle='-', lw=1)
axes[0].set_xlabel('Coefficient Value')
axes[0].set_title('Lasso Regression Coefficients (Wine Quality)')
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Residual Plot
axes[1].scatter(y_pred, residuals, alpha=0.6, edgecolors='black', color='purple')
axes[1].axhline(y=0, color='red', linestyle='--', lw=2)
axes[1].set_xlabel('Predicted Quality')
axes[1].set_ylabel('Residuals')
axes[1].set_title('Residual Plot')
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('lasso_regression_wine.png', dpi=300)
plt.show()

Question 61: Lasso Regression for Health Data

Python

import numpy as np
import pandas as pd
from sklearn.linear_model import Lasso
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Generate HealthData dataset
np.random.seed(42)
n_samples = 400

age = np.random.randint(18, 80, n_samples)
bmi = np.random.uniform(15, 45, n_samples)
blood_pressure = np.random.uniform(80, 180, n_samples)

# Health score (target)
health_score = (
    100 - 0.3 * age - 1.5 * (bmi - 22)**2 / 100 - 0.2 * (blood_pressure - 120)
    + np.random.normal(0, 10, n_samples)
)
health_score = np.clip(health_score, 0, 100)

df = pd.DataFrame({
    'age': age,
    'bmi': bmi,
    'blood_pressure': blood_pressure,
    'health_score': health_score
})

print("=" * 60)
print("LASSO REGRESSION FOR HEALTH SCORE PREDICTION")
print("=" * 60)
print(f"\nDataset Shape: {df.shape}")
print(f"\nFirst 10 rows:")
print(df.head(10))

# Prepare data
X = df[['age', 'bmi', 'blood_pressure']]
y = df['health_score']

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Fit Lasso Regression
lasso = Lasso(alpha=0.5, random_state=42)
lasso.fit(X_train_scaled, y_train)

# Predictions
y_pred = lasso.predict(X_test_scaled)
residuals = y_test - y_pred

# Evaluation
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL RESULTS")
print("-" * 40)
print(f"Mean Squared Error: {mse:.4f}")
print(f"R² Score: {r2:.4f}")

print(f"\nLasso Coefficients:")
print(f"  Intercept: {lasso.intercept_:.4f}")
for feature, coef in zip(X.columns, lasso.coef_):
    print(f"  {feature}: {coef:.4f}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: Coefficient Plot
coef_df = pd.DataFrame({
    'Feature': X.columns,
    'Coefficient': lasso.coef_
})

colors = ['red' if c < 0 else 'green' for c in coef_df['Coefficient']]
axes[0].bar(coef_df['Feature'], coef_df['Coefficient'], color=colors, edgecolor='black')
axes[0].axhline(y=0, color='black', linestyle='-', lw=1)
axes[0].set_ylabel('Coefficient Value')
axes[0].set_title('Lasso Regression Coefficients')
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Residual Plot
axes[1].scatter(y_pred, residuals, alpha=0.6, edgecolors='black', color='teal')
axes[1].axhline(y=0, color='red', linestyle='--', lw=2)
axes[1].set_xlabel('Predicted Health Score')
axes[1].set_ylabel('Residuals')
axes[1].set_title('Residual Plot')
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('lasso_regression_health.png', dpi=300)
plt.show()

Question 62: PCA Demonstration (100 samples, 3 features)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from mpl_toolkits.mplot3d import Axes3D

# Generate random dataset
np.random.seed(42)
n_samples = 100
n_features = 3

# Generate correlated data
X = np.random.randn(n_samples, n_features)
X[:, 1] = X[:, 0] * 0.8 + np.random.randn(n_samples) * 0.5
X[:, 2] = X[:, 0] * 0.5 + X[:, 1] * 0.3 + np.random.randn(n_samples) * 0.5

print("=" * 60)
print("PCA DEMONSTRATION")
print("=" * 60)
print(f"\nOriginal Data Shape: {X.shape}")
print(f"\nFirst 5 samples:")
print(X[:5])

# Apply PCA
pca = PCA()
X_pca = pca.fit_transform(X)

print(f"\nPCA Results:")
print(f"Explained Variance Ratio: {pca.explained_variance_ratio_}")
print(f"Cumulative Variance: {np.cumsum(pca.explained_variance_ratio_)}")

# Visualization
fig = plt.figure(figsize=(15, 5))

# Plot 1: Original 3D Data
ax1 = fig.add_subplot(131, projection='3d')
ax1.scatter(X[:, 0], X[:, 1], X[:, 2], c='blue', alpha=0.6, edgecolors='black')
ax1.set_xlabel('Feature 1')
ax1.set_ylabel('Feature 2')
ax1.set_zlabel('Feature 3')
ax1.set_title('Original 3D Data')

# Plot 2: PCA Reduced 2D
ax2 = fig.add_subplot(132)
ax2.scatter(X_pca[:, 0], X_pca[:, 1], c='green', alpha=0.6, edgecolors='black')
ax2.set_xlabel(f'PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)')
ax2.set_ylabel(f'PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)')
ax2.set_title('PCA Reduced (2D)')
ax2.grid(True, linestyle='--', alpha=0.5)

# Plot 3: Explained Variance
ax3 = fig.add_subplot(133)
ax3.bar(range(1, 4), pca.explained_variance_ratio_, color='steelblue', edgecolor='black')
ax3.plot(range(1, 4), np.cumsum(pca.explained_variance_ratio_), 'ro-', label='Cumulative')
ax3.set_xlabel('Principal Component')
ax3.set_ylabel('Explained Variance Ratio')
ax3.set_title('Variance Explained')
ax3.legend()
ax3.set_xticks([1, 2, 3])

plt.tight_layout()
plt.savefig('pca_demonstration.png', dpi=300)
plt.show()

Question 63: PCA on Custom Dataset with Explained Variance

Python

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_iris

# Load Iris dataset as example
iris = load_iris()
X = iris.data
y = iris.target

print("=" * 60)
print("PCA WITH EXPLAINED VARIANCE")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Features: {iris.feature_names}")

# Standardize data
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Apply PCA (keep all components)
pca = PCA()
X_pca = pca.fit_transform(X_scaled)

# Print explained variance for all components
print("\n" + "-" * 40)
print("EXPLAINED VARIANCE OF PRINCIPAL COMPONENTS")
print("-" * 40)
for i, (var, cum_var) in enumerate(zip(pca.explained_variance_ratio_, 
                                        np.cumsum(pca.explained_variance_ratio_)), 1):
    print(f"PC{i}: {var*100:.2f}% (Cumulative: {cum_var*100:.2f}%)")

print(f"\nTotal Variance Explained: {sum(pca.explained_variance_ratio_)*100:.2f}%")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Individual variance
axes[0].bar(range(1, len(pca.explained_variance_ratio_) + 1), 
            pca.explained_variance_ratio_ * 100, 
            color='steelblue', edgecolor='black')
axes[0].set_xlabel('Principal Component')
axes[0].set_ylabel('Variance Explained (%)')
axes[0].set_title('Individual Variance Explained')
axes[0].set_xticks(range(1, len(pca.explained_variance_ratio_) + 1))

# Cumulative variance
cumulative = np.cumsum(pca.explained_variance_ratio_) * 100
axes[1].plot(range(1, len(cumulative) + 1), cumulative, 'go-', linewidth=2, markersize=10)
axes[1].axhline(y=95, color='red', linestyle='--', label='95% threshold')
axes[1].set_xlabel('Number of Components')
axes[1].set_ylabel('Cumulative Variance (%)')
axes[1].set_title('Cumulative Variance Explained')
axes[1].set_xticks(range(1, len(cumulative) + 1))
axes[1].legend()
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('pca_explained_variance.png', dpi=300)
plt.show()

Question 64: PCA on make_blobs (3D to 2D)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs
from sklearn.decomposition import PCA
from mpl_toolkits.mplot3d import Axes3D

# Generate synthetic dataset with 3 features and 3 centers
X, y = make_blobs(n_samples=300, n_features=3, centers=3, cluster_std=1.5, random_state=42)

print("=" * 60)
print("PCA ON MAKE_BLOBS (3D TO 2D)")
print("=" * 60)
print(f"\nOriginal Data Shape: {X.shape}")
print(f"Number of Clusters: {len(np.unique(y))}")

# Apply PCA to reduce to 2 dimensions
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)

print(f"\nPCA Results:")
print(f"Reduced Shape: {X_pca.shape}")
print(f"Explained Variance Ratio: {pca.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(pca.explained_variance_ratio_)*100:.2f}%")

# Visualization - Side by side
fig = plt.figure(figsize=(14, 6))

# Plot 1: Original 3D Dataset
ax1 = fig.add_subplot(121, projection='3d')
colors = ['red', 'green', 'blue']
for i in range(3):
    mask = (y == i)
    ax1.scatter(X[mask, 0], X[mask, 1], X[mask, 2], 
                c=colors[i], s=50, alpha=0.6, label=f'Cluster {i}')
ax1.set_xlabel('Feature 1')
ax1.set_ylabel('Feature 2')
ax1.set_zlabel('Feature 3')
ax1.set_title('Original 3D Dataset')
ax1.legend()

# Plot 2: PCA Reduced 2D Dataset
ax2 = fig.add_subplot(122)
for i in range(3):
    mask = (y == i)
    ax2.scatter(X_pca[mask, 0], X_pca[mask, 1], 
                c=colors[i], s=50, alpha=0.6, edgecolors='black', label=f'Cluster {i}')
ax2.set_xlabel(f'PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)')
ax2.set_ylabel(f'PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)')
ax2.set_title('PCA Reduced 2D Dataset')
ax2.legend()
ax2.grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('pca_make_blobs_3d_to_2d.png', dpi=300)
plt.show()

Question 65: LDA on Iris Dataset (2D)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

# Load Iris dataset
iris = load_iris()
X = iris.data
y = iris.target

print("=" * 60)
print("LDA ON IRIS DATASET")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Features: {iris.feature_names}")
print(f"Classes: {iris.target_names}")

# Apply LDA to reduce to 2 dimensions
lda = LinearDiscriminantAnalysis(n_components=2)
X_lda = lda.fit_transform(X, y)

print(f"\nLDA Results:")
print(f"Reduced Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")

# Visualization
plt.figure(figsize=(10, 8))

colors = ['red', 'green', 'blue']
markers = ['o', 's', '^']

for i, (species, color, marker) in enumerate(zip(iris.target_names, colors, markers)):
    mask = (y == i)
    plt.scatter(X_lda[mask, 0], X_lda[mask, 1], 
                c=color, marker=marker, s=80, 
                alpha=0.7, edgecolors='black', label=species)

plt.xlabel(f'LD1 ({lda.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
plt.ylabel(f'LD2 ({lda.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
plt.title('LDA of Iris Dataset (2D)', fontsize=14)
plt.legend(title='Species')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('lda_iris_2d.png', dpi=300)
plt.show()

Question 66: LDA on Wine Dataset (2D)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_wine
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

# Load Wine dataset
wine = load_wine()
X = wine.data
y = wine.target

print("=" * 60)
print("LDA ON WINE DATASET")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Number of Features: {len(wine.feature_names)}")
print(f"Classes: {wine.target_names}")

# Apply LDA to reduce to 2 dimensions
lda = LinearDiscriminantAnalysis(n_components=2)
X_lda = lda.fit_transform(X, y)

print(f"\nLDA Results:")
print(f"Reduced Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(lda.explained_variance_ratio_)*100:.2f}%")

# Visualization
plt.figure(figsize=(10, 8))

colors = ['red', 'green', 'blue']
for i, color in enumerate(colors):
    mask = (y == i)
    plt.scatter(X_lda[mask, 0], X_lda[mask, 1], 
                c=color, s=80, alpha=0.7, 
                edgecolors='black', label=f'Wine Class {i}')

plt.xlabel(f'LD1 ({lda.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
plt.ylabel(f'LD2 ({lda.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
plt.title('LDA of Wine Dataset (2D)', fontsize=14)
plt.legend(title='Wine Type')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('lda_wine_2d.png', dpi=300)
plt.show()

Question 67: LDA on Breast Cancer Dataset (1D)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

# Load Breast Cancer dataset
data = load_breast_cancer()
X = data.data
y = data.target

print("=" * 60)
print("LDA ON BREAST CANCER DATASET (1D)")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Classes: {data.target_names}")
print(f"Class Distribution: Malignant={sum(y==0)}, Benign={sum(y==1)}")

# Apply LDA to reduce to 1 dimension
lda = LinearDiscriminantAnalysis(n_components=1)
X_lda = lda.fit_transform(X, y)

print(f"\nLDA Results:")
print(f"Reduced Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Plot 1: 1D Scatter
colors = ['red', 'green']
for i, (target_name, color) in enumerate(zip(data.target_names, colors)):
    mask = (y == i)
    axes[0].scatter(X_lda[mask], np.zeros_like(X_lda[mask]) + i * 0.1, 
                    c=color, s=50, alpha=0.5, label=target_name)

axes[0].set_xlabel('LD1', fontsize=12)
axes[0].set_title('LDA of Breast Cancer (1D Scatter)', fontsize=14)
axes[0].legend()
axes[0].set_yticks([])
axes[0].grid(True, linestyle='--', alpha=0.5)

# Plot 2: Histogram
for i, (target_name, color) in enumerate(zip(data.target_names, colors)):
    mask = (y == i)
    axes[1].hist(X_lda[mask], bins=30, alpha=0.6, color=color, label=target_name, edgecolor='black')

axes[1].set_xlabel('LD1', fontsize=12)
axes[1].set_ylabel('Frequency', fontsize=12)
axes[1].set_title('Distribution of Classes along LD1', fontsize=14)
axes[1].legend()

plt.tight_layout()
plt.savefig('lda_breast_cancer_1d.png', dpi=300)
plt.show()

Question 68: LDA on Digits Dataset (2D)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_digits
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

# Load Digits dataset
digits = load_digits()
X = digits.data
y = digits.target

print("=" * 60)
print("LDA ON HANDWRITTEN DIGITS DATASET")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Number of Classes: {len(np.unique(y))}")
print(f"Classes: {np.unique(y)}")

# Apply LDA to reduce to 2 dimensions
lda = LinearDiscriminantAnalysis(n_components=2)
X_lda = lda.fit_transform(X, y)

print(f"\nLDA Results:")
print(f"Reduced Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(lda.explained_variance_ratio_)*100:.2f}%")

# Visualization
plt.figure(figsize=(12, 10))

colors = plt.cm.tab10(np.linspace(0, 1, 10))
for i in range(10):
    mask = (y == i)
    plt.scatter(X_lda[mask, 0], X_lda[mask, 1], 
                c=[colors[i]], s=30, alpha=0.6, label=str(i))

plt.xlabel(f'LD1 ({lda.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
plt.ylabel(f'LD2 ({lda.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
plt.title('LDA of Handwritten Digits (2D)', fontsize=14)
plt.legend(title='Digit', loc='best')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('lda_digits_2d.png', dpi=300)
plt.show()

# Show sample digits
fig, axes = plt.subplots(2, 5, figsize=(12, 5))
for i, ax in enumerate(axes.flat):
    ax.imshow(digits.images[i], cmap='gray')
    ax.set_title(f'Digit: {digits.target[i]}')
    ax.axis('off')
plt.suptitle('Sample Handwritten Digits', fontsize=14)
plt.tight_layout()
plt.savefig('sample_digits.png', dpi=300)
plt.show()

Question 69: LDA on Iris Dataset (Alternative Implementation)

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

# Load Iris dataset
iris = load_iris()
X = iris.data
y = iris.target

print("=" * 60)
print("LDA ON IRIS DATASET (COMPLETE IMPLEMENTATION)")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Features: {iris.feature_names}")
print(f"Classes: {iris.target_names}")

# Apply LDA to reduce to 2 dimensions
lda = LinearDiscriminantAnalysis(n_components=2)
X_lda = lda.fit_transform(X, y)

print(f"\nLDA Results:")
print(f"Reduced Shape: {X_lda.shape}")
print(f"Explained Variance Ratio: {lda.explained_variance_ratio_}")
print(f"Total Variance Explained: {sum(lda.explained_variance_ratio_)*100:.2f}%")

# Classification using LDA
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

lda_classifier = LinearDiscriminantAnalysis()
lda_classifier.fit(X_train, y_train)
y_pred = lda_classifier.predict(X_test)
accuracy = accuracy_score(y_test, y_pred)

print(f"\nClassification Accuracy: {accuracy:.4f}")

# Visualization
plt.figure(figsize=(10, 8))

colors = ['red', 'green', 'blue']
markers = ['o', 's', '^']

for i, (species, color, marker) in enumerate(zip(iris.target_names, colors, markers)):
    mask = (y == i)
    plt.scatter(X_lda[mask, 0], X_lda[mask, 1], 
                c=color, marker=marker, s=80, 
                alpha=0.7, edgecolors='black', label=species)

plt.xlabel(f'LD1 ({lda.explained_variance_ratio_[0]*100:.2f}%)', fontsize=12)
plt.ylabel(f'LD2 ({lda.explained_variance_ratio_[1]*100:.2f}%)', fontsize=12)
plt.title('LDA of Iris Dataset', fontsize=14)
plt.legend(title='Species')
plt.grid(True, linestyle='--', alpha=0.5)
plt.tight_layout()
plt.savefig('lda_iris_complete.png', dpi=300)
plt.show()

Question 70: SVM Classification for Iris Dataset

Python

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import seaborn as sns

# Load Iris dataset
iris = load_iris()
X = iris.data
y = iris.target

print("=" * 60)
print("SVM CLASSIFICATION FOR IRIS DATASET")
print("=" * 60)
print(f"\nDataset Shape: {X.shape}")
print(f"Features:")
for i, name in enumerate(iris.feature_names):
    print(f"  {i+1}. {name}")
print(f"\nClasses: {iris.target_names}")

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Train SVC Model
svm = SVC(kernel='rbf', C=1.0, gamma='scale', random_state=42)
svm.fit(X_train_scaled, y_train)

# Predictions
y_pred = svm.predict(X_test_scaled)

# Evaluation
accuracy = accuracy_score(y_test, y_pred)

print("\n" + "-" * 40)
print("MODEL PERFORMANCE")
print("-" * 40)
print(f"Accuracy: {accuracy:.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=iris.target_names))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0],
            xticklabels=iris.target_names, yticklabels=iris.target_names)
axes[0].set_title('Confusion Matrix', fontsize=14)
axes[0].set_xlabel('Predicted')
axes[0].set_ylabel('Actual')

# PCA visualization of predictions
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
X_test_pca = pca.fit_transform(X_test_scaled)

colors = ['red', 'green', 'blue']
# Plot actual classes
for i, (species, color) in enumerate(zip(iris.target_names, colors)):
    mask = (y_test == i)
    axes[1].scatter(X_test_pca[mask, 0], X_test_pca[mask, 1], 
                    c=color, marker='o', s=100, alpha=0.7, 
                    edgecolors='black', label=f'{species} (Actual)')

# Mark misclassified points
misclassified = (y_test != y_pred)
if sum(misclassified) > 0:
    axes[1].scatter(X_test_pca[misclassified, 0], X_test_pca[misclassified, 1], 
                    c='none', marker='o', s=200, edgecolors='red', linewidths=3, 
                    label='Misclassified')

axes[1].set_xlabel('PC1')
axes[1].set_ylabel('PC2')
axes[1].set_title('SVM Predictions (PCA Visualization)')
axes[1].legend(loc='best')
axes[1].grid(True, linestyle='--', alpha=0.5)

plt.tight_layout()
plt.savefig('svm_iris_classification.png', dpi=300)
plt.show()

# Compare different kernels
print("\n" + "-" * 40)
print("KERNEL COMPARISON")
print("-" * 40)

kernels = ['linear', 'rbf', 'poly', 'sigmoid']
kernel_accuracies = []

for kernel in kernels:
    svm_temp = SVC(kernel=kernel, random_state=42)
    svm_temp.fit(X_train_scaled, y_train)
    acc = svm_temp.score(X_test_scaled, y_test)
    kernel_accuracies.append(acc)
    print(f"{kernel.capitalize()} Kernel: {acc:.4f}")

# Kernel comparison plot
plt.figure(figsize=(10, 6))
bars = plt.bar(kernels, kernel_accuracies, color=['blue', 'green', 'orange', 'red'], edgecolor='black')
plt.xlabel('Kernel Type', fontsize=12)
plt.ylabel('Accuracy', fontsize=12)
plt.title('SVM Performance with Different Kernels', fontsize=14)
plt.ylim(0, 1.1)

for bar, acc in zip(bars, kernel_accuracies):
    plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, 
             f'{acc:.4f}', ha='center', fontsize=11)

plt.tight_layout()
plt.savefig('svm_kernel_comparison_iris.png', dpi=300)
plt.show()
"""


def get_all_questions():
    matches = list(re.finditer(r"Question\s+(\d+):", ALL_CONTENT))
    result = []

    for match in matches:
        q_number = match.group(1)
        index = match.start()
        result.append((q_number, index))

    return result


def get_question(q_number):
    pattern = rf"Question\s+{q_number}:"
    match = re.search(pattern, ALL_CONTENT)

    if not match:
        return "Question not found."

    start = match.start()

    # Find next question
    next_match = re.search(r"Question\s+\d+:", ALL_CONTENT[start + 1:])
    if next_match:
        end = start + 1 + next_match.start()
        return ALL_CONTENT[start:end]
    else:
        return ALL_CONTENT[start:]


def main():
    if len(sys.argv) < 2:
        print("Usage: ippdf_lj <question_number | all | list>")
        return

    arg = sys.argv[1]

    if arg == "all":
        print(ALL_CONTENT)

    elif arg == "list":
        for q_num, idx in get_all_questions():
            print(f"Question {q_num} starts at index {idx}")

    else:
        print(get_question(arg))


if __name__ == "__main__":
    main()