# -*- coding: utf-8 -*-
""" 


NN (Neural Networks)
============================
This module hosts functions for training and evaluating artificial neural networks for labeled classification tasks.

Currently, 3 types of models can be trained:

    PLAN (Potentiation Learning Artificial Neural Network)
        * Training Time for Small Projects: fast
        * Training Time for Big Projects: fast
        * Explainability: high
        * Learning Capacity: medium (compared to single perceptrons)

    MLP (Multi-Layer Perceptron → Deep Learning) -- With non-bias
        * Training Time for Small Projects: fast
        * Training Time for Big Projects: slow
        * Explainability: low
        * Learning Capacity: high

    PTNN (Potentiation Transfer Neural Network) -- With non-bias
        * Training Time for Small Projects: fast
        * Training Time for Big Projects: fast
        * Explainability: medium
        * Learning Capacity: high

Read learn function docstring for know how to use of these model architectures.

For more information about PLAN: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PLAN/PLAN.pdf

Module functions:
-----------------
- plan_fit()
- learn()
- evaluate()

Examples: https://github.com/HCB06/PyerualJetwork/tree/main/Welcome_to_PyerualJetwork/ExampleCodes

PyerualJetwork document: https://github.com/HCB06/PyerualJetwork/blob/main/Welcome_to_PyerualJetwork/PYERUALJETWORK_USER_MANUEL_AND_LEGAL_INFORMATION(EN).pdf

- Creator: Hasan Can Beydili
- YouTube: https://www.youtube.com/@HasanCanBeydili
- Linkedin: https://www.linkedin.com/in/hasan-can-beydili-77a1b9270/
- Instagram: https://www.instagram.com/canbeydili
- Contact: tchasancan@gmail.com
"""

import numpy as np
import copy
import random
from multiprocessing import Pool, cpu_count

### LIBRARY IMPORTS ###
from .ui import loading_bars, initialize_loading_bar, format_number, format_fraction
from .cpu.activation_functions import all_activations
from .model_ops import get_acc, get_preds_softmax, get_preds
from .memory_ops import optimize_labels, transfer_to_gpu
from .fitness_functions import wals

### GLOBAL VARIABLES ###
bar_format_normal = loading_bars()[0]
bar_format_learner = loading_bars()[1]

# BUILD -----

def plan_fit(
    x_train,
    y_train,
    activations=['linear'],
    W=None,
    auto_normalization=False,
    cuda=False,
    dtype=np.float32
):
    """
    Creates a PLAN model to fitting data.
    
    Args:
        x_train (aray-like[num or cup]): List or numarray of input data.

        y_train (aray-like[num or cup]): List or numarray of target labels. (one hot encoded)

        activations (list): For deeper PLAN networks, activation function parameters. For more information please run this code: neu.activations_list() default: [None] (optional)

        W (numpy.ndarray or cupy.ndarray): If you want to re-continue or update model

        auto_normalization (bool, optional): Normalization may solves overflow problem. Default: False

        cuda (bool, optional): CUDA GPU acceleration ? Default = False.
        
        dtype (numpy.dtype): Data type for the arrays. np.float32 by default. Example: np.float64 or np.float16.

    Returns:
        numpyarray: (Weight matrix).
    """
    
    from .cpu.data_ops import normalization
    if not cuda:
        from .cpu.activation_functions import apply_activation
        array_type = np

    else:
        from .cuda.activation_functions import apply_activation
        import cupy as cp
        array_type = cp
        
        
    # Pre-check

    if len(x_train) != len(y_train): raise ValueError("x_train and y_train must have the same length.")

    weight = array_type.zeros((len(y_train[0]), len(x_train[0].ravel()))).astype(dtype, copy=False) if W is None else W

    if auto_normalization is True: x_train = normalization(apply_activation(x_train, activations))
    elif auto_normalization is False: x_train = apply_activation(x_train, activations)
    else: raise ValueError('normalization parameter only be True or False')

    weight += y_train.T @ x_train if not cuda else array_type.array(y_train).T @ array_type.array(x_train)

    return normalization(weight.get() if cuda else weight,dtype=dtype)


def learn(x_train, y_train, optimizer, gen, pop_size, fit_start=True, batch_size=1,
           weight_evolve=True, neural_web_history=False, show_current_activations=False, auto_normalization=False,
           neurons_history=False, early_stop=False, show_history=False, target_loss=None,
           interval=33.33, target_acc=None, loss='categorical_crossentropy', acc_impact=0.9, loss_impact=0.1,
           start_this_act=None, start_this_W=None, neurons=[], activation_functions=[], parallel_training=False,
           thread_count=cpu_count(), decision_boundary_history=False, cuda=False, dtype=np.float32,
           gradient_train=False,
           gradient_optimizer='adam',      # 'sgd'|'momentum'|'rmsprop'|'adam'|'adamw'|'nadam'|'adagrad'
           learning_rate=0.001,
           beta1=0.9, beta2=0.999, epsilon=1e-8,
           momentum=0.9,                   # sgd-momentum / nesterov için
           rho=0.9,                        # rmsprop için
           l2=0.0, l1=0.0,                # regularization
           clip_value=0.0):               # gradient clipping (0=kapalı)

    from .ene import define_genomes
    from .cpu.visualizations import display_decision_boundary_history, create_decision_boundary_hist, plot_decision_boundary
    from .model_ops import get_model_template

    if cuda is False:
        from .cpu.data_ops import batcher
        from .cpu.loss_functions import categorical_crossentropy, binary_crossentropy
        from .cpu.visualizations import (
            draw_neural_web, display_visualizations_for_learner,
            update_history_plots_for_learner, initialize_visualization_for_learner,
            update_neuron_history_for_learner)
    else:
        from .cuda.data_ops import batcher
        from .cuda.loss_functions import categorical_crossentropy, binary_crossentropy
        from .cuda.visualizations import (
            draw_neural_web, display_visualizations_for_learner,
            update_history_plots_for_learner, initialize_visualization_for_learner,
            update_neuron_history_for_learner)

    data = 'Train'
    template_model = get_model_template()

    except_this = ['spiral', 'circular']
    activations = [item for item in all_activations() if item not in except_this]
    activations_len = len(activations)
    """
    def format_number(val):
        if abs(val) >= 1e4 or (abs(val) < 1e-2 and val != 0):
            return f"{val:.4e}"
        else:
            return f"{val:.4f}"
    """
    # ── gradient train için ortak yardımcı ───────────────────────────────────
    def _run_gradient_train(init_weights, grad_epochs, final_activations, activation_potentiation):
        """
        Mini-batch gradient descent döngüsü.

        Loading bar: total=grad_epochs — bar her EPOCH'ta bir adım ilerler.
        Batch adımlarında sadece postfix güncellenir (bar kıpırdamaz).
        Epoch bitince Train Accuracy/Loss yazılır, bar bir adım ilerler.
        """
        grad_weights = [np.copy(w) for w in init_weights]
        grad_state   = None
        global_step  = 0

        # ── Batch boyutunu belirle ────────────────────────────────────────────
        N_total = len(x_train)

        if isinstance(batch_size, float) and 0.0 < batch_size < 1.0:
            bs = max(1, int(N_total * batch_size))
        elif isinstance(batch_size, int) and 0 < batch_size < N_total:
            bs = batch_size
        else:
            bs = N_total   # full-batch

        steps_per_epoch = max(1, N_total // bs)
        is_full_batch   = (bs == N_total)

        # ── Bar: total=grad_epochs, epoch başına 1 adım ──────────────────────
        progress = initialize_loading_bar(
            total=grad_epochs,
            desc="",
            ncols=85,
            bar_format=bar_format_learner
        )
        postfix_dict = {}

        train_model = train_loss = train_acc = best_weight = None

        for epoch in range(1, grad_epochs + 1):

            postfix_dict["Epoch"] = format_fraction(epoch, grad_epochs)
            progress.set_postfix(postfix_dict)

            # ── Epoch başında karıştır ────────────────────────────────────────
            if not is_full_batch:
                perm   = np.random.permutation(N_total)
                x_shuf = x_train[perm]
                y_shuf = y_train[perm]
            else:
                x_shuf = x_train
                y_shuf = y_train

            for step in range(steps_per_epoch):

                global_step += 1

                # ── Mini-batch ────────────────────────────────────────────────
                start   = step * bs
                end     = min(start + bs, N_total)
                x_batch = x_shuf[start:end]
                y_batch = y_shuf[start:end]

                # ── define_genomes formatına çevir ────────────────────────────
                best_weight = np.empty(len(grad_weights), dtype=object)
                for idx, w in enumerate(grad_weights):
                    best_weight[idx] = w

                # ── İleri geçiş + cache ───────────────────────────────────────
                batch_model = evaluate(
                    x_batch, y_batch,
                    W=best_weight,
                    activations=final_activations,
                    activation_potentiations=activation_potentiation,
                    auto_normalization=auto_normalization,
                    cuda=cuda,
                    model_type='MLP',
                    return_cache=True,
                )
                softmax_preds = batch_model[get_preds_softmax()]
                cache         = batch_model[6]

                if loss == 'categorical_crossentropy':
                    batch_loss = categorical_crossentropy(
                        y_true_batch=y_batch, y_pred_batch=softmax_preds)
                else:
                    batch_loss = binary_crossentropy(
                        y_true_batch=y_batch, y_pred_batch=softmax_preds)

                # Bar: batch adımlarında postfix güncelle, bar kıpırdamaz
                postfix_dict["Batch Accuracy"] = format_number(float(batch_model[get_acc()]))
                postfix_dict["Batch Loss"]     = format_number(float(batch_loss))
                postfix_dict.pop("Train Accuracy", None)
                postfix_dict.pop("Train Loss", None)
                progress.set_postfix(postfix_dict)

                # ── Backprop + optimizer ──────────────────────────────────────
                grad_weights, grad_state = gradient_fit(
                    y_train=np.array(y_batch, dtype=np.float32),
                    softmax_preds=np.array(softmax_preds, dtype=np.float32),
                    cache=cache,
                    weights=grad_weights,
                    activations=final_activations,
                    optimizer=gradient_optimizer,
                    state=grad_state,
                    t=global_step,
                    learning_rate=learning_rate,
                    beta1=beta1, beta2=beta2, epsilon=epsilon,
                    momentum=momentum, rho=rho,
                    l2=l2, l1=l1,
                    clip_value=clip_value,
                    cuda=cuda,
                )

            # ── EPOCH SONU ────────────────────────────────────────────────────
            # full-batch: son batch == tam set, ekstra evaluate yok
            # mini-batch: tam set üzerinde evaluate
            if not is_full_batch:
                best_weight = np.empty(len(grad_weights), dtype=object)
                for idx, w in enumerate(grad_weights):
                    best_weight[idx] = w

                train_model = evaluate(
                    x_train, y_train,
                    W=best_weight,
                    activations=final_activations,
                    activation_potentiations=activation_potentiation,
                    auto_normalization=auto_normalization,
                    cuda=cuda,
                    model_type='MLP',
                )
                if loss == 'categorical_crossentropy':
                    train_loss = categorical_crossentropy(
                        y_true_batch=y_train,
                        y_pred_batch=train_model[get_preds_softmax()])
                else:
                    train_loss = binary_crossentropy(
                        y_true_batch=y_train,
                        y_pred_batch=train_model[get_preds_softmax()])
                train_acc = float(train_model[get_acc()])
            else:
                train_model = batch_model
                train_loss  = batch_loss
                train_acc   = float(batch_model[get_acc()])

            # Batch etiketlerini kaldır, Train etiketlerini yaz, bar bir ileri
            postfix_dict.pop("Batch Accuracy", None)
            postfix_dict.pop("Batch Loss", None)
            postfix_dict["Train Accuracy"] = format_number(train_acc)
            postfix_dict["Train Loss"]     = format_number(float(train_loss))
            progress.set_postfix(postfix_dict)
            progress.update(1)   # ← bar sadece burada ilerler (epoch başına 1 adım)

            # ── Erken durdurma ────────────────────────────────────────────────
            if target_acc is not None and train_acc >= target_acc:
                progress.close()
                return template_model._replace(
                    weights=best_weight,
                    predictions=train_model[get_preds()],
                    accuracy=train_acc,
                    activations=final_activations,
                    softmax_predictions=train_model[get_preds_softmax()],
                    model_type='MLP',
                    activation_potentiation=activation_potentiation)

            if target_loss is not None and train_loss <= target_loss:
                progress.close()
                return template_model._replace(
                    weights=best_weight,
                    predictions=train_model[get_preds()],
                    accuracy=train_acc,
                    activations=final_activations,
                    softmax_predictions=train_model[get_preds_softmax()],
                    model_type='MLP',
                    activation_potentiation=activation_potentiation)

        progress.close()
        return template_model._replace(
            weights=best_weight,
            predictions=train_model[get_preds()],
            accuracy=train_acc,
            activations=final_activations,
            softmax_predictions=train_model[get_preds_softmax()],
            model_type='MLP',
            activation_potentiation=activation_potentiation)
    # ─────────────────────────────────────────────────────────────────────────

    # Pre-checks
    if cuda:
        x_train = transfer_to_gpu(x_train, dtype=dtype)
        y_train = transfer_to_gpu(y_train, dtype=dtype)

    if pop_size > activations_len and fit_start is True:
        for _ in range(pop_size - len(activations)):
            activations.append(activations[random.randint(0, len(activations) - 1)])

    y_train = optimize_labels(y_train, cuda=cuda)

    if not gradient_train:
        if pop_size < activations_len:
            raise ValueError(f"pop_size must be higher or equal to {activations_len}")

    if target_acc is not None and (target_acc < 0 or target_acc > 1):
        raise ValueError('target_acc must be in range 0 and 1')
    if fit_start is not True and fit_start is not False:
        raise ValueError('fit_start parameter only be True or False. Please read doc-string')

    if neurons != []:
        weight_evolve = True
        if activation_functions == []:
            activation_functions = ['linear'] * len(neurons)

        if fit_start is False:
            activations = activation_functions
            model_type = 'MLP'
            activation_potentiations = [0] * pop_size
            activation_potentiation = None
            is_mlp = True
            transfer_learning = False
        else:
            model_type = 'PLAN'
            transfer_learning = True
            neurons_copy = neurons.copy()
            neurons = []
            gen_copy = gen.copy()
            gen = gen[0] + gen[1]
            activation_potentiations = [0] * pop_size
            activation_potentiation = None
            is_mlp = False
    else:
        model_type = 'PLAN'
        transfer_learning = False
        activation_potentiations = [0] * pop_size
        activation_potentiation = None
        is_mlp = False

    # ── Saf MLP + gradient_train ──────────────────────────────────────────────
    if gradient_train and model_type == 'MLP':
        if start_this_W is not None:
            init_weights = [np.copy(w) for w in start_this_W]
        else:
            _w_pop_tmp, _ = define_genomes(
                input_shape=len(x_train[0]), output_shape=len(y_train[0]),
                neurons=neurons, activation_functions=activation_functions,
                population_size=1, dtype=dtype)
            init_weights = list(_w_pop_tmp[0])

        return _run_gradient_train(init_weights, gen, activation_functions, None)
    # ─────────────────────────────────────────────────────────────────────────

    viz_objects = initialize_visualization_for_learner(show_history, neurons_history, neural_web_history, x_train, y_train)
    best_acc = 0
    best_loss = float('inf')
    best_fitness = float('-inf')
    best_acc_per_gen_list = []
    postfix_dict = {}
    loss_list = []
    target_pop = []

    progress = initialize_loading_bar(
        total=gen if isinstance(gen, int) else gen[0] + gen[1],
        desc="", ncols=85, bar_format=bar_format_learner)

    if fit_start is False:
        weight_pop, act_pop = define_genomes(
            input_shape=len(x_train[0]), output_shape=len(y_train[0]),
            neurons=neurons, activation_functions=activations,
            population_size=pop_size, dtype=dtype)
    else:
        weight_pop = [0] * len(activations)
        act_pop    = [0] * len(activations)

    if start_this_act is not None and start_this_W is not None:
        weight_pop[0] = start_this_W
        act_pop[0]    = start_this_act

    for i in range(gen):

        # PLAN → PTNN dönüşümü
        if model_type == 'PLAN' and transfer_learning:
            if i == gen_copy[0]:
                model_type = 'PTNN'
                neurons = neurons_copy

                for individual in range(len(weight_pop)):
                    weight_pop[individual] = np.copy(best_weight)
                    activation_potentiations[individual] = (
                        final_activations.copy()
                        if isinstance(final_activations, list) else final_activations)

                activation_potentiation = activation_potentiations[0]
                neurons_copy = [len(y_train[0])] + neurons_copy
                activation_functions = ['linear'] + activation_functions

                weight_pop, act_pop = define_genomes(
                    input_shape=len(x_train[0]), output_shape=len(y_train[0]),
                    neurons=neurons_copy, activation_functions=activation_functions,
                    population_size=pop_size, dtype=dtype)

                for l in range(1, len(weight_pop[0])):
                    original_shape = weight_pop[0][l].shape
                    weight_pop[0][l] = np.eye(original_shape[0], original_shape[1],
                                              dtype=weight_pop[0][l].dtype)
                for l in range(len(weight_pop)):
                    weight_pop[l][0] = np.copy(best_weight)

                best_weight     = np.array(weight_pop[0], dtype=object)
                final_activations = act_pop[0]
                is_mlp     = True
                fit_start  = False

                # ── PTNN + gradient_train ─────────────────────────────────────
                if gradient_train:
                    progress.close()
                    return _run_gradient_train(
                        init_weights=list(best_weight),
                        grad_epochs=gen - i,
                        final_activations=final_activations,
                        activation_potentiation=activation_potentiation)
                # ─────────────────────────────────────────────────────────────

        postfix_dict["Gen"] = str(i + 1) + '/' + str(gen)
        progress.set_postfix(postfix_dict)

        if parallel_training:
            eval_params = []
            fit_params  = []

            with Pool(processes=thread_count) as pool:
                x_train_batch, y_train_batch = batcher(x_train, y_train, batch_size)

                if model_type == 'PLAN' and i == 0:
                    act_pop = copy.deepcopy(activations)

                eval_params = [
                    (x_train_batch, y_train_batch, None, model_type,
                     weight_pop[j], act_pop[j], activation_potentiations[j],
                     auto_normalization, None, cuda)
                    for j in range(pop_size)
                ]

                if model_type == 'PLAN':
                    if weight_evolve is True and i > 0:
                        pass
                    else:
                        fit_params = [
                            (x_train_batch, y_train_batch, act_pop[j],
                             weight_pop[j], auto_normalization, cuda, dtype)
                            for j in range(pop_size)
                        ]
                        W = pool.starmap(plan_fit, fit_params)
                        weight_pop  = W
                        eval_params = [(*ep[:4], w, *ep[5:])
                                       for ep, w in zip(eval_params, W)]

                start_W   = start_this_W   if (start_this_W   is not None and i == 0) else eval_params[0][4]
                start_act = start_this_act if (start_this_act is not None and i == 0) else eval_params[0][5]

                eval_params[0] = (
                    eval_params[0][0], eval_params[0][1], eval_params[0][2],
                    eval_params[0][3], start_W, start_act,
                    eval_params[0][6], eval_params[0][7],
                    eval_params[0][8], eval_params[0][9])

                models = pool.starmap(evaluate, eval_params)
                y_preds = [m[get_preds_softmax()] for m in models]
                loss_func = categorical_crossentropy if loss == 'categorical_crossentropy' else binary_crossentropy
                losses = pool.starmap(loss_func, [(eval_params[f][1], y_preds[f]) for f in range(pop_size)])
                target_pop = pool.starmap(wals, [(models[f][get_acc()], losses[f], acc_impact, loss_impact) for f in range(pop_size)])
                if cuda:
                    target_pop = [fit.get() for fit in target_pop]

            best_idx  = np.argmax(target_pop)
            best_fitness = target_pop[best_idx]
            best_weight  = models[best_idx][0]
            best_loss    = losses[best_idx]
            best_acc     = models[best_idx][get_acc()]
            best_model   = models[best_idx]
            x_train_batch = eval_params[best_idx][0]
            y_train_batch = eval_params[best_idx][1]

            if isinstance(eval_params[best_idx][5], list) and model_type == 'PLAN':
                final_activations = eval_params[best_idx][5].copy()
            elif isinstance(eval_params[best_idx][5], str):
                final_activations = eval_params[best_idx][5]
            else:
                final_activations = copy.deepcopy(eval_params[best_idx][5])

            if model_type == 'PLAN':
                final_activations = ([final_activations[0]]
                                     if len(set(final_activations)) == 1
                                     else final_activations)
            if batch_size == 1:
                postfix_dict[f"{data} Accuracy"] = format_number(best_acc)
                postfix_dict[f"{data} Loss"]     = format_number(best_loss)
                progress.set_postfix(postfix_dict)
            if show_current_activations:
                print(f", Current Activations={final_activations}", end='')

        else:
            for j in range(pop_size):
                x_train_batch, y_train_batch = batcher(x_train, y_train, batch_size=batch_size)

                if fit_start is True and i == 0:
                    if not (start_this_act is not None and j == 0):
                        act_pop[j] = activations[j]
                        W = plan_fit(x_train_batch, y_train_batch, activations=act_pop[j],
                                     cuda=cuda, auto_normalization=auto_normalization, dtype=dtype)
                        weight_pop[j] = W

                if weight_evolve is False:
                    weight_pop[j] = plan_fit(x_train_batch, y_train_batch,
                                             activations=act_pop[j], cuda=cuda,
                                             auto_normalization=auto_normalization, dtype=dtype)

                model = evaluate(x_train_batch, y_train_batch, W=weight_pop[j],
                                 activations=act_pop[j],
                                 activation_potentiations=activation_potentiations[j],
                                 auto_normalization=auto_normalization, cuda=cuda,
                                 model_type=model_type)
                acc = model[get_acc()]

                if loss == 'categorical_crossentropy':
                    train_loss = categorical_crossentropy(y_true_batch=y_train_batch,
                                                         y_pred_batch=model[get_preds_softmax()])
                else:
                    train_loss = binary_crossentropy(y_true_batch=y_train_batch,
                                                     y_pred_batch=model[get_preds_softmax()])

                fitness = wals(acc, train_loss, acc_impact, loss_impact)
                target_pop.append(fitness if not cuda else fitness.get())

                if fitness >= best_fitness:
                    best_fitness = fitness
                    best_acc     = acc
                    best_loss    = train_loss
                    best_weight  = (np.copy(weight_pop[j]) if model_type == 'PLAN'
                                    else copy.deepcopy(weight_pop[j]))
                    best_model   = model

                    if isinstance(act_pop[j], list) and model_type == 'PLAN':
                        final_activations = act_pop[j].copy()
                    elif isinstance(act_pop[j], str):
                        final_activations = act_pop[j]
                    else:
                        final_activations = copy.deepcopy(act_pop[j])

                    if model_type == 'PLAN':
                        final_activations = ([final_activations[0]]
                                             if len(set(final_activations)) == 1
                                             else final_activations)
                    if batch_size == 1:
                        postfix_dict[f"{data} Accuracy"] = format_number(best_acc)
                        postfix_dict[f"{data} Loss"]     = format_number(best_loss)
                        progress.set_postfix(postfix_dict)
                    if show_current_activations:
                        print(f", Current Activations={final_activations}", end='')

                    if target_acc is not None and best_acc >= target_acc:
                        progress.close()
                        train_model = evaluate(x_train, y_train, W=best_weight,
                                               activations=final_activations,
                                               activation_potentiations=activation_potentiation,
                                               auto_normalization=auto_normalization,
                                               cuda=cuda, model_type=model_type)
                        if loss == 'categorical_crossentropy':
                            train_loss = categorical_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
                        else:
                            train_loss = binary_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
                        print('\nActivations: ', final_activations)
                        print('Activation Potentiation: ', activation_potentiation)
                        print('Train Accuracy:', train_model[get_acc()])
                        print('Train Loss: ', train_loss, '\n')
                        print('Model Type:', model_type)
                        if decision_boundary_history: display_decision_boundary_history(fig, artist, interval=interval)
                        display_visualizations_for_learner(viz_objects, best_weight, data, best_acc, best_loss, y_train, interval)
                        model = template_model._replace(weights=best_weight, predictions=best_model[get_preds()],
                                                       accuracy=best_acc, activations=final_activations,
                                                       softmax_predictions=best_model[get_preds_softmax()],
                                                       model_type=model_type, activation_potentiation=activation_potentiation)
                        return model

                    if target_loss is not None and best_loss <= target_loss:
                        progress.close()
                        train_model = evaluate(x_train, y_train, W=best_weight,
                                               activations=final_activations,
                                               activation_potentiations=activation_potentiation,
                                               auto_normalization=auto_normalization,
                                               cuda=cuda, model_type=model_type)
                        if loss == 'categorical_crossentropy':
                            train_loss = categorical_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
                        else:
                            train_loss = binary_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
                        print('\nActivations: ', final_activations)
                        print('Activation Potentiation: ', activation_potentiation)
                        print('Train Accuracy:', train_model[get_acc()])
                        print('Train Loss: ', train_loss, '\n')
                        print('Model Type:', model_type)
                        if decision_boundary_history: display_decision_boundary_history(fig, artist, interval=interval)
                        display_visualizations_for_learner(viz_objects, best_weight, data, best_acc, train_loss, y_train, interval)
                        model = template_model._replace(weights=best_weight, predictions=best_model[get_preds()],
                                                       accuracy=best_acc, activations=final_activations,
                                                       softmax_predictions=best_model[get_preds_softmax()],
                                                       model_type=model_type, activation_potentiation=activation_potentiation)

                        return model
                    
        progress.update(1)

        if batch_size != 1:
            train_model = evaluate(x_train, y_train, W=best_weight, activations=final_activations,
                                   activation_potentiations=activation_potentiation,
                                   auto_normalization=auto_normalization, cuda=cuda, model_type=model_type)
            if loss == 'categorical_crossentropy':
                train_loss = categorical_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
            else:
                train_loss = binary_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
            best_acc_per_gen_list.append(train_model[get_acc()])
            loss_list.append(train_loss)
            postfix_dict[f"{data} Accuracy"] = format_number(train_model[get_acc()])
            postfix_dict[f"{data} Loss"]     = format_number(train_loss)
            progress.set_postfix(postfix_dict)
        else:
            best_acc_per_gen_list.append(best_acc)
            loss_list.append(best_loss)

        if show_history:
            if batch_size == 1:
                gen_list = range(1, len(best_acc_per_gen_list) + 2)
                update_history_plots_for_learner(viz_objects, gen_list,
                    loss_list + [best_loss], best_acc_per_gen_list + [best_acc], x_train, final_activations)
            else:
                gen_list = range(1, len(best_acc_per_gen_list) + 2)
                update_history_plots_for_learner(viz_objects, gen_list,
                    loss_list + [loss_list[-1]], best_acc_per_gen_list + [best_acc_per_gen_list[-1]], x_train, final_activations)

        if neurons_history:
            viz_objects['neurons']['artists'] = update_neuron_history_for_learner(
                np.copy(best_weight), viz_objects['neurons']['ax'],
                viz_objects['neurons']['row'], viz_objects['neurons']['col'],
                y_train[0], viz_objects['neurons']['artists'],
                data=data, fig1=viz_objects['neurons']['fig'], acc=best_acc, loss=train_loss)

        if neural_web_history:
            art5_1, art5_2, art5_3 = draw_neural_web(W=best_weight, ax=viz_objects['web']['ax'],
                                                      G=viz_objects['web']['G'], return_objs=True)
            viz_objects['web']['artists'].append([art5_1] + [art5_2] + list(art5_3.values()))

        if decision_boundary_history:
            if i == 0:
                fig, ax = create_decision_boundary_hist()
                artist  = []
            artist = plot_decision_boundary(x_train_batch, y_train_batch,
                                            activations=act_pop[np.argmax(target_pop)],
                                            W=weight_pop[np.argmax(target_pop)],
                                            cuda=cuda, model_type=model_type, ax=ax, artist=artist)

        weight_pop = (np.array(weight_pop, copy=False, dtype=dtype)
                      if model_type == 'PLAN'
                      else np.array(weight_pop, copy=False, dtype=object))

        weight_pop, act_pop = optimizer(weight_pop, act_pop, i,
                                        np.array(target_pop, dtype=dtype, copy=False),
                                        weight_evolve=weight_evolve, is_mlp=is_mlp, bar_status=False)
        target_pop = []

        if early_stop and i > 0:
            if best_acc_per_gen_list[i] == best_acc_per_gen_list[i - 1]:
                progress.close()
                train_model = evaluate(x_train, y_train, W=best_weight,
                                       activations=final_activations,
                                       activation_potentiations=activation_potentiation,
                                       auto_normalization=auto_normalization,
                                       cuda=cuda, model_type=model_type)
                if loss == 'categorical_crossentropy':
                    train_loss = categorical_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
                else:
                    train_loss = binary_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
                print('\nActivations: ', final_activations)
                print('Activation Potentiation: ', activation_potentiation)
                print('Train Accuracy:', train_model[get_acc()])
                print('Train Loss: ', train_loss, '\n')
                print('Model Type:', model_type)
                if decision_boundary_history: display_decision_boundary_history(fig, artist, interval=interval)
                display_visualizations_for_learner(viz_objects, best_weight, data, best_acc, train_loss, y_train, interval)
                model = template_model._replace(weights=best_weight, predictions=best_model[get_preds()],
                                               accuracy=best_acc, activations=final_activations,
                                               softmax_predictions=best_model[get_preds_softmax()],
                                               model_type=model_type, activation_potentiation=activation_potentiation)
                return model

    # Final evaluation
    progress.close()
    train_model = evaluate(x_train, y_train, W=best_weight,
                           activations=final_activations,
                           activation_potentiations=activation_potentiation,
                           auto_normalization=auto_normalization, cuda=cuda, model_type=model_type)
    if loss == 'categorical_crossentropy':
        train_loss = categorical_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])
    else:
        train_loss = binary_crossentropy(y_true_batch=y_train, y_pred_batch=train_model[get_preds_softmax()])

    print('\nActivations: ', final_activations)
    print('Activation Potentiation: ', activation_potentiation)
    print('Train Accuracy:', train_model[get_acc()])
    print('Train Loss: ', train_loss, '\n')
    print('Model Type:', model_type)
    if decision_boundary_history: display_decision_boundary_history(fig, artist, interval=interval)
    display_visualizations_for_learner(viz_objects, best_weight, data, best_acc, train_loss, y_train, interval)
    model = template_model._replace(weights=best_weight, predictions=best_model[get_preds()],
                                   accuracy=best_acc, activations=final_activations,
                                   softmax_predictions=best_model[get_preds_softmax()],
                                   model_type=model_type, activation_potentiation=activation_potentiation)
    
    return model


def gradient_fit(
    y_train:       np.ndarray,      # (N, C) one-hot
    softmax_preds: np.ndarray,      # (N, C) evaluate'den gelen softmax çıkışı
    cache:         list,            # predict_from_memory(return_activations=True) çıktısı
                                    # MLP: [Input, Z0, Z1, ..., Z_last]
    weights:       list,            # [W0, W1, …]  Wi.shape = (out_i, in_i)
    activations:   list,            # gizli katman aktivasyon isimleri ['tanh', 'relu', …]
                                    # len == num_layers - 1
    optimizer:     str  = 'adam',   # 'sgd'|'momentum'|'rmsprop'|'adam'|'adamw'|'nadam'|'adagrad'
    state:         dict = None,     # optimizer state — ilk çağrıda None ver
    t:             int  = 1,        # epoch / zaman adımı
    learning_rate: float = 0.001,
    beta1:         float = 0.9,
    beta2:         float = 0.999,
    epsilon:       float = 1e-8,
    momentum:      float = 0.9,     # sgd-momentum / nesterov
    rho:           float = 0.9,     # rmsprop
    l2:            float = 0.0,     # L2 weight decay
    l1:            float = 0.0,     # L1 regularization
    clip_value:    float = 0.0,     # gradient clipping (0 = kapalı)
    cuda:          bool  = False,
) -> tuple:
    """
    Tek backprop + optimizer adımı. İleri geçiş YOKTUR.
    evaluate(return_cache=True) → predict_from_memory(return_activations=True)
    tarafından yapılır; cache ve softmax_preds buraya gelir.

    Performans optimizasyonları:
      - acts list comprehension (döngüsüz)
      - delta in-place işlemler (np.multiply, np.add, +=, -=)
      - Adam: bias correction dışarıda tek hesap, lr_adj ile birleşik
      - Gereksiz array kopyası yok

    Cache formatı (MLP, return_activations=True):
        cache[0]   = Input           (N, in0)
        cache[1]   = Z0 pre-act      (N, out0)
        cache[k+1] = Zk pre-act      (N, out_k)

    Döndürür: (weights, state)
    """

    if not cuda:
        from .cpu.activation_functions import apply_activation, apply_activation_derivative
    else:
        from .cuda.activation_functions import apply_activation, apply_activation_derivative

    num_layers = len(weights)
    N          = y_train.shape[0]
    opt        = optimizer.lower()

    # ── Acts: cache'den aktivasyon SONRASI değerler ───────────────────────────
    # cache[0]        = Input  (aktivasyon yok)
    # cache[i+1]      = Z_i    (pre-activation)  → apply_activation → A_i
    # acts[num_layers] = softmax_preds
    #
    # List comprehension: döngü Python overhead'ını minimuma indirir,
    # apply_activation zaten BLAS-vektörize numpy işlemi yapar.

    acts = (
        [cache[0]]
        + [apply_activation(cache[i + 1], [activations[i]]) for i in range(num_layers - 1)]
        + [softmax_preds]
    )

    # ── State başlatma ────────────────────────────────────────────────────────
    if state is None:
        state = {}
        if opt in ('adam', 'adamw', 'nadam'):
            state['m'] = [np.zeros_like(W) for W in weights]
            state['v'] = [np.zeros_like(W) for W in weights]
        elif opt in ('rmsprop', 'momentum', 'sgd'):
            state['v'] = [np.zeros_like(W) for W in weights]
        elif opt == 'adagrad':
            state['G'] = [np.zeros_like(W) for W in weights]

    # ── GERİ YAYILIM ─────────────────────────────────────────────────────────
    # Softmax + CCE birleşik türevi: delta = (ŷ − y) / N  (in-place)
    delta = softmax_preds - y_train          # yeni array (softmax_preds korunur)
    delta *= 1.0 / N                         # in-place bölme

    grads = [None] * num_layers

    for i in range(num_layers - 1, -1, -1):

        # dL/dW_i = delta.T @ acts[i]   (out_i, in_i)
        g = delta.T @ acts[i]

        # Regularization
        if l2 > 0.0 and opt != 'adamw':
            g += l2 * weights[i]             # in-place
        if l1 > 0.0:
            g += l1 * np.sign(weights[i])    # in-place

        # Gradient clipping (element-wise, in-place)
        if clip_value > 0.0:
            np.clip(g, -clip_value, clip_value, out=g)

        grads[i] = g

        if i > 0:
            # delta → bir alt katmana ilet, in-place ReLU türevi
            # cache[i] = Z_{i-1}  (pre-activation of layer i-1)
            new_delta = delta @ weights[i]                               # (N, out_{i-1})
            dA_dZ = apply_activation_derivative(cache[i], activations[i - 1])
            np.multiply(new_delta, dA_dZ, out=new_delta)                 # in-place türev
            delta = new_delta

    # ── OPTİMİZER GÜNCELLEMESİ ───────────────────────────────────────────────

    if opt == 'sgd':
        for i in range(num_layers):
            # v = momentum*v + g  (in-place)
            np.multiply(momentum, state['v'][i], out=state['v'][i])
            np.add(state['v'][i], grads[i], out=state['v'][i])
            weights[i] -= learning_rate * state['v'][i]

    elif opt == 'momentum':
        # Nesterov: w ← w − momentum*v_prev + (1+momentum)*v_new
        for i in range(num_layers):
            v_prev = state['v'][i].copy()
            np.multiply(momentum, state['v'][i], out=state['v'][i])
            state['v'][i] -= learning_rate * grads[i]                   # in-place
            weights[i] += -momentum * v_prev + (1.0 + momentum) * state['v'][i]

    elif opt == 'rmsprop':
        for i in range(num_layers):
            # E[g²] = rho*E[g²] + (1-rho)*g²  (in-place)
            np.multiply(rho, state['v'][i], out=state['v'][i])
            state['v'][i] += (1.0 - rho) * (grads[i] * grads[i])
            denom = np.sqrt(state['v'][i])
            denom += epsilon                                             # in-place +ε
            weights[i] -= learning_rate * grads[i] / denom

    elif opt == 'adam':
        # Bias correction dışarıda tek hesap → her katmanda tekrar hesaplanmaz
        b1t    = 1.0 - beta1 ** t
        b2t    = 1.0 - beta2 ** t
        lr_adj = learning_rate * (b2t ** 0.5) / b1t   # combined lr correction
        for i in range(num_layers):
            # m = beta1*m + (1-beta1)*g  (in-place)
            np.multiply(beta1, state['m'][i], out=state['m'][i])
            state['m'][i] += (1.0 - beta1) * grads[i]
            # v = beta2*v + (1-beta2)*g²  (in-place)
            np.multiply(beta2, state['v'][i], out=state['v'][i])
            state['v'][i] += (1.0 - beta2) * (grads[i] * grads[i])
            # w -= lr_adj * m / (√v + ε)
            denom = np.sqrt(state['v'][i])
            denom += epsilon
            weights[i] -= lr_adj * state['m'][i] / denom

    elif opt == 'adamw':
        # Adam + ayrık weight decay
        b1t    = 1.0 - beta1 ** t
        b2t    = 1.0 - beta2 ** t
        lr_adj = learning_rate * (b2t ** 0.5) / b1t
        for i in range(num_layers):
            np.multiply(beta1, state['m'][i], out=state['m'][i])
            state['m'][i] += (1.0 - beta1) * grads[i]
            np.multiply(beta2, state['v'][i], out=state['v'][i])
            state['v'][i] += (1.0 - beta2) * (grads[i] * grads[i])
            denom = np.sqrt(state['v'][i])
            denom += epsilon
            weights[i] -= lr_adj * state['m'][i] / denom
            weights[i] -= learning_rate * l2 * weights[i]               # ayrık weight decay

    elif opt == 'nadam':
        # Nesterov Adam
        b1t  = 1.0 - beta1 ** t
        b1t1 = 1.0 - beta1 ** (t + 1)
        b2t  = 1.0 - beta2 ** t
        for i in range(num_layers):
            np.multiply(beta1, state['m'][i], out=state['m'][i])
            state['m'][i] += (1.0 - beta1) * grads[i]
            np.multiply(beta2, state['v'][i], out=state['v'][i])
            state['v'][i] += (1.0 - beta2) * (grads[i] * grads[i])
            # Nesterov m_hat: bir adım ileriden
            m_hat = beta1 * state['m'][i] / b1t1 + (1.0 - beta1) * grads[i] / b1t
            denom = np.sqrt(state['v'][i] / b2t)
            denom += epsilon
            weights[i] -= learning_rate * m_hat / denom

    elif opt == 'adagrad':
        for i in range(num_layers):
            state['G'][i] += grads[i] * grads[i]                        # in-place birikimli
            denom = np.sqrt(state['G'][i])
            denom += epsilon
            weights[i] -= learning_rate * grads[i] / denom

    else:
        raise ValueError(
            f"Bilinmeyen optimizer: '{optimizer}'. "
            "Seçenekler: 'sgd', 'momentum', 'rmsprop', 'adam', 'adamw', 'nadam', 'adagrad'"
        )

    return weights, state

def evaluate(
    x_test,
    y_test,
    model=None,
    model_type=None,
    W=None,
    activations=['linear'],
    activation_potentiations=[],
    auto_normalization=False,
    show_report=False,
    cuda=False,
    return_cache=False,         # True ise backprop için layer cache döner
) -> tuple:
    """
    Evaluates the neural network model using the given test data.
    Args:
        x_test (np.ndarray): Test data.
        y_test (np.ndarray): Test labels (one-hot encoded).
        model (tuple, optional): Trained model.
        model_type: (str, optional): Type of the model. Options: 'PLAN', 'MLP', 'PTNN'.
        W (array-like, optional): Neural net weight matrix.
        activations (list, optional): Activation list for PLAN or MLP models. Default = ['linear'].
        activation_potentiations (list, optional): Extra activation potentiation list for PTNN. Default = [].
        auto_normalization (bool, optional): Normalization for x_test? Default = False.
        show_report (bool, optional): Show test report. Default = False.
        cuda (bool, optional): CUDA GPU acceleration? Default = False.
        return_cache (bool, optional): If True, returns layer-wise pre-activation cache
                                       from predict_from_memory for use in backprop.
                                       Cache = [Input, Z0, Z1, ..., Z_last]
                                       Default = False.
    Returns:
        return_cache=False: W, preds, accuracy, None, None, softmax_preds
        return_cache=True : W, preds, accuracy, None, None, softmax_preds, cache
    """
    from .cpu.visualizations import plot_evaluate
    from .model_ops import predict_from_memory

    if not cuda:
        from .cpu.data_ops import normalization
        array_type = np

        if model:
            sample_acc = model.accuracy
            if hasattr(sample_acc, "get"): model = model._replace(accuracy=sample_acc.get())
    else:
        from .cuda.data_ops import normalization
        import cupy as cp
        array_type = cp
        x_test = cp.array(x_test)
        y_test = cp.array(y_test)
        if model:
            sample_acc = model.accuracy
            if isinstance(sample_acc, np.number): model = model._replace(accuracy=cp.array(sample_acc))

    if model is None:
        from .model_ops import get_model_template
        template_model = get_model_template()

        model = template_model._replace(
            weights=W,
            activations=activations,
            model_type=model_type,
            activation_potentiation=activation_potentiations
        )

    if return_cache:
        # predict_from_memory'den layer cache'i al
        # activations_list = [Input, Z0, Z1, ..., Z_last]  (pre-activation değerler)
        result, cache = predict_from_memory(x_test, model, cuda=cuda, return_activations=True)
    else:
        result = predict_from_memory(x_test, model, cuda=cuda)
        cache = None

    if auto_normalization:
        x_test = normalization(x_test, dtype=x_test.dtype)

    max_vals     = array_type.max(result, axis=1, keepdims=True)
    exp_shifted  = array_type.exp(result - max_vals)
    softmax_preds = exp_shifted / array_type.sum(exp_shifted, axis=1, keepdims=True)
    accuracy     = (array_type.argmax(softmax_preds, axis=1) == array_type.argmax(y_test, axis=1)).mean()

    if show_report:
        plot_evaluate(x_test, y_test, result, acc=accuracy, model=model, cuda=cuda)

    if return_cache:
        return W, result, accuracy, None, None, softmax_preds, cache

    return W, result, accuracy, None, None, softmax_preds