# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0  (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch


def npu_rms_norm(x, gamma, epsilon=1e-5):
    rstd = torch.rsqrt(torch.mean(torch.pow(x, 2), axis=-1, keepdim=True) + epsilon)
    res = x * rstd * gamma
    return res, rstd.float()


def npu_rms_norm_backward(grad, x, gamma, rstd):
    mean_gy = (grad * x * gamma * rstd).mean(dim=-1, keepdim=True)
    grad_x = (grad * gamma - x * rstd * mean_gy) * rstd
    grad_gamma = x * grad * rstd
    return grad_x.cpu(), grad_gamma.cpu()

