# This file is adapted from the LLama project:
# https://github.com/facebookresearch/llama/blob/main/llama/model.py

# Original LLama code by Facebook AI Research 
# Adapted by Aisuko

import math
from dataclasses import dataclass
from typing import Optional, Tuple

import torch
import torch.nn.functional as F
import torch.nn as nn
from torch import nn
from torch.nn import Embedding, Linear


@dataclass
class ModelArgs:
    """
    A class to store the arguments for the model.

    Attributes:
    -----------
    dim : int
        The dimension of the model.
    n_layers : int
        The number of layers in the model.
    n_heads : int
        The number of heads in the model.
    n_kv_heads : Optional[int]
        The number of key-value heads in the model.
    vocab_size : int
        The size of the vocabulary.
    multiple_of : int
        The multiple of the SwiGLU hidden layer size.
    ffn_dim_multiplier : Optional[float]
        The multiplier for the feedforward network dimension.
    norm_eps : float
        The epsilon value for normalization.

    max_batch_size : int
        The maximum batch size for the model.
    max_seq_len : int
        The maximum sequence length for the model.
    """
    dim: int = 4096
    n_layers: int = 32
    n_heads: int = 32
    n_kv_heads: Optional[int] = None
    vocab_size: int = -1  # defined later by tokenizer
    multiple_of: int = 256  # make SwiGLU hidden layer size multiple of large power of 2
    ffn_dim_multiplier: Optional[float] = None
    norm_eps: float = 1e-5

    max_batch_size: int = 32
    max_seq_len: int = 2048


class RMSNorm(torch.nn.Module):
    """
    Root Mean Square Normalization (RMSNorm) layer.

    Args:
        dim (int): The dimension of the input tensor.
        eps (float): A small value added to the denominator for numerical stability. Default: 1e-6.
    """
    def __init__(self, dim: int, eps: float = 1e-6):
        super().__init__()
        self.eps = eps
        self.weight = nn.Parameter(torch.ones(dim))

    def _norm(self, x):
        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)

    def forward(self, x):
        output = self._norm(x.float()).type_as(x)
        return output * self.weight


def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0):
    """
    Precomputes the frequency of cosine and sine functions for a given dimension and end value.

    Args:
        dim (int): The dimension of the frequency tensor.
        end (int): The end value of the frequency tensor.
        theta (float, optional): The scaling factor for the frequency tensor. Defaults to 10000.0.

    Returns:
        freqs_cis (torch.Tensor): A complex tensor of shape (end, dim/2) containing the precomputed frequency values.
    """
    freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))
    t = torch.arange(end, device=freqs.device)  # type: ignore
    freqs = torch.outer(t, freqs).float()  # type: ignore
    freqs_cis = torch.polar(torch.ones_like(freqs), freqs)  # complex64
    return freqs_cis


def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor):
    """
    Reshapes the input tensor `freqs_cis` to match the shape of `x` for broadcasting.

    Args:
        freqs_cis (torch.Tensor): A tensor of shape (x.shape[1], x.shape[-1]).
        x (torch.Tensor): The input tensor.

    Returns:
        torch.Tensor: The reshaped tensor of the same shape as `x` except for the second and second-to-last dimensions.
    """
    ndim = x.ndim
    assert 0 <= 1 < ndim
    assert freqs_cis.shape == (x.shape[1], x.shape[-1])
    shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)]
    return freqs_cis.view(*shape)


def apply_rotary_emb(
    xq: torch.Tensor,
    xk: torch.Tensor,
    freqs_cis: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
    """
    Applies rotary embeddings to the input tensors xq and xk using the given frequency tensor freqs_cis.

    Args:
        xq (torch.Tensor): The query tensor.
        xk (torch.Tensor): The key tensor.
        freqs_cis (torch.Tensor): The frequency tensor.

    Returns:
        Tuple[torch.Tensor, torch.Tensor]: A tuple of the output tensors xq_out and xk_out.
    """
    xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
    xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
    freqs_cis = reshape_for_broadcast(freqs_cis, xq_)
    xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3)
    xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3)
    return xq_out.type_as(xq), xk_out.type_as(xk)


def repeat_kv(x: torch.Tensor, n_rep: int) -> torch.Tensor:
    """torch.repeat_interleave(x, dim=2, repeats=n_rep)"""
    bs, slen, n_kv_heads, head_dim = x.shape
    if n_rep == 1:
        return x
    return (
        x[:, :, :, None, :]
        .expand(bs, slen, n_kv_heads, n_rep, head_dim)
        .reshape(bs, slen, n_kv_heads * n_rep, head_dim)
    )


class Attention(nn.Module):
    def __init__(self, args: ModelArgs):
        """
        Initializes the Attention module.

        Args:
        - args: ModelArgs object containing the following attributes:
            - n_heads: int, number of attention heads
            - n_kv_heads: int, number of key-value attention heads
            - dim: int, dimension of the model
            - max_batch_size: int, maximum batch size
            - max_seq_len: int, maximum sequence length
        """
        super().__init__()
        self.n_kv_heads = args.n_heads if args.n_kv_heads is None else args.n_kv_heads
        model_parallel_size = 1#fs_init.get_model_parallel_world_size()
        self.n_local_heads = args.n_heads // model_parallel_size
        self.n_local_kv_heads = self.n_kv_heads // model_parallel_size
        self.n_rep = self.n_local_heads // self.n_local_kv_heads
        self.head_dim = args.dim // args.n_heads

        self.wq = nn.Linear(
            args.dim,
            args.n_heads * self.head_dim,
            bias=False,
        )
        self.wk = nn.Linear(
            args.dim,
            self.n_kv_heads * self.head_dim,
            bias=False,
        )
        self.wv = nn.Linear(
            args.dim,
            self.n_kv_heads * self.head_dim,
            bias=False,
        )
        self.wo = nn.Linear(
            args.n_heads * self.head_dim,
            args.dim,
            bias=False,
        )

        self.cache_k = torch.zeros(
            (
                args.max_batch_size,
                args.max_seq_len,
                self.n_local_kv_heads,
                self.head_dim,
            )
        )
        self.cache_v = torch.zeros(
            (
                args.max_batch_size,
                args.max_seq_len,
                self.n_local_kv_heads,
                self.head_dim,
            )
        )

    def forward(
        self,
        x: torch.Tensor,
        start_pos: int,
        freqs_cis: torch.Tensor,
        mask: Optional[torch.Tensor],
    ) -> torch.Tensor:
        """
        Performs a forward pass of the Attention module.

        Args:
        - x: torch.Tensor of shape (batch_size, sequence_length, model_dimension), input tensor
        - start_pos: int, starting position of the sequence
        - freqs_cis: torch.Tensor of shape (sequence_length, model_dimension), cosine and sine frequencies for rotary embeddings
        - mask: Optional[torch.Tensor] of shape (batch_size, sequence_length, sequence_length), mask tensor

        Returns:
        - output: torch.Tensor of shape (batch_size, sequence_length, model_dimension), output tensor
        """
        bsz, seqlen, _ = x.shape
        xq, xk, xv = self.wq(x), self.wk(x), self.wv(x)

        xq = xq.view(bsz, seqlen, self.n_local_heads, self.head_dim)
        xk = xk.view(bsz, seqlen, self.n_local_kv_heads, self.head_dim)
        xv = xv.view(bsz, seqlen, self.n_local_kv_heads, self.head_dim)

        xq, xk = apply_rotary_emb(xq, xk, freqs_cis=freqs_cis)

        self.cache_k = self.cache_k.to(xq)
        self.cache_v = self.cache_v.to(xq)

        self.cache_k[:bsz, start_pos : start_pos + seqlen] = xk
        self.cache_v[:bsz, start_pos : start_pos + seqlen] = xv

        keys = self.cache_k[:bsz, : start_pos + seqlen]
        values = self.cache_v[:bsz, : start_pos + seqlen]

        # repeat k/v heads if n_kv_heads < n_heads
        keys = repeat_kv(keys, self.n_rep)  # (bs, seqlen, n_local_heads, head_dim)
        values = repeat_kv(values, self.n_rep)  # (bs, seqlen, n_local_heads, head_dim)

        xq = xq.transpose(1, 2)  # (bs, n_local_heads, seqlen, head_dim)
        keys = keys.transpose(1, 2)
        values = values.transpose(1, 2)
        scores = torch.matmul(xq, keys.transpose(2, 3)) / math.sqrt(self.head_dim)
        if mask is not None:
            scores = scores + mask  # (bs, n_local_heads, seqlen, cache_len + seqlen)
        scores = F.softmax(scores.float(), dim=-1).type_as(xq)
        output = torch.matmul(scores, values)  # (bs, n_local_heads, seqlen, head_dim)
        output = output.transpose(1, 2).contiguous().view(bsz, seqlen, -1)
        return self.wo(output)


class FeedForward(nn.Module):
    """
    A feedforward neural network module.

    Args:
        dim (int): The input dimension.
        hidden_dim (int): The hidden dimension.
        multiple_of (int): The output dimension is a multiple of this value.
        ffn_dim_multiplier (Optional[float]): A multiplier for the hidden dimension.

    Attributes:
        w1 (nn.Linear): The first linear layer.
        w2 (nn.Linear): The second linear layer.
        w3 (nn.Linear): The third linear layer.
    """

    def __init__(
        self,
        dim: int,
        hidden_dim: int,
        multiple_of: int,
        ffn_dim_multiplier: Optional[float],
    ):
        super().__init__()
        hidden_dim = int(2 * hidden_dim / 3)
        # custom dim factor multiplier
        if ffn_dim_multiplier is not None:
            hidden_dim = int(ffn_dim_multiplier * hidden_dim)
        hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)

        self.w1 = Linear(
            dim, hidden_dim, bias=False
        )
        self.w2 = Linear(
            hidden_dim, dim, bias=False
        )
        self.w3 = Linear(
            dim, hidden_dim, bias=False
        )

    def forward(self, x):
        """
        Forward pass of the feedforward neural network.

        Args:
            x (torch.Tensor): The input tensor.

        Returns:
            torch.Tensor: The output tensor.
        """
        return self.w2(F.silu(self.w1(x)) * self.w3(x))


class TransformerBlock(nn.Module):
    """
    A transformer block that consists of a self-attention layer and a feedforward layer.

    Args:
        layer_id (int): The ID of the layer.
        args (ModelArgs): The arguments for the model.

    Attributes:
        n_heads (int): The number of attention heads.
        dim (int): The dimension of the model.
        head_dim (int): The dimension of each attention head.
        attention (Attention): The self-attention layer.
        feed_forward (FeedForward): The feedforward layer.
        layer_id (int): The ID of the layer.
        attention_norm (RMSNorm): The normalization layer for the attention output.
        ffn_norm (RMSNorm): The normalization layer for the feedforward output.
    """

    def __init__(self, layer_id: int, args: ModelArgs):
        super().__init__()
        self.n_heads = args.n_heads
        self.dim = args.dim
        self.head_dim = args.dim // args.n_heads
        self.attention = Attention(args)
        self.feed_forward = FeedForward(
            dim=args.dim,
            hidden_dim=4 * args.dim,
            multiple_of=args.multiple_of,
            ffn_dim_multiplier=args.ffn_dim_multiplier,
        )
        self.layer_id = layer_id
        self.attention_norm = RMSNorm(args.dim, eps=args.norm_eps)
        self.ffn_norm = RMSNorm(args.dim, eps=args.norm_eps)

    def forward(
        self,
        x: torch.Tensor,
        start_pos: int,
        freqs_cis: torch.Tensor,
        mask: Optional[torch.Tensor],
    ) -> torch.Tensor:
        """
        Forward pass of the transformer block.

        Args:
            x (torch.Tensor): The input tensor.
            start_pos (int): The starting position for the input sequence.
            freqs_cis (torch.Tensor): The frequency tensor for the input sequence.
            mask (Optional[torch.Tensor]): The mask tensor for the input sequence.

        Returns:
            torch.Tensor: The output tensor.
        """
        h = x + self.attention.forward(
            self.attention_norm(x), start_pos, freqs_cis, mask
        )
        out = h + self.feed_forward.forward(self.ffn_norm(h))
        return out



class Transformer(nn.Module):
    """
    A Transformer model for sequence-to-sequence tasks.

    Args:
        params (ModelArgs): A dataclass containing the model hyperparameters.

    Attributes:
        params (ModelArgs): The model hyperparameters.
        vocab_size (int): The size of the vocabulary.
        n_layers (int): The number of layers in the Transformer.
        tok_embeddings (Embedding): The token embeddings layer.
        layers (ModuleList): The list of TransformerBlock layers.
        norm (RMSNorm): The normalization layer.
        output (Linear): The output layer.
        freqs_cis (torch.Tensor): The precomputed cosine frequencies.

    Methods:
        forward(tokens, start_pos): The forward pass of the model.
    """

    def __init__(self, params: ModelArgs):
        super().__init__()
        self.params = params
        self.vocab_size = params.vocab_size
        self.n_layers = params.n_layers

        self.tok_embeddings = Embedding(
            params.vocab_size, params.dim
        )

        self.layers = torch.nn.ModuleList()
        for layer_id in range(params.n_layers):
            self.layers.append(TransformerBlock(layer_id, params))

        self.norm = RMSNorm(params.dim, eps=params.norm_eps)
        self.output = Linear(
            params.dim, params.vocab_size, bias=False
        )

        self.freqs_cis = precompute_freqs_cis(
            self.params.dim // self.params.n_heads, self.params.max_seq_len * 2
        )

    @torch.inference_mode()
    def forward(self, tokens: torch.Tensor, start_pos: int):
        """
        The forward pass of the model.

        Args:
            tokens (torch.Tensor): The input sequence of tokens.
            start_pos (int): The starting position of the sequence.

        Returns:
            output (torch.Tensor): The output sequence of logits.
        """
        _bsz, seqlen = tokens.shape
        h = self.tok_embeddings(tokens)
        self.freqs_cis = self.freqs_cis.to(h.device)
        freqs_cis = self.freqs_cis[start_pos : start_pos + seqlen]

        mask = None
        if seqlen > 1:
            mask = torch.full(
                (1, 1, seqlen, seqlen), float("-inf"), device=tokens.device
            )
            mask = torch.triu(mask, diagonal=start_pos + 1).type_as(h)

        for layer in self.layers:
            h = layer(h, start_pos, freqs_cis, mask)
        h = self.norm(h)
        output = self.output(h).float()
        return output
    