// Copyright (c) Meta Platforms, Inc. and affiliates.

#include "cinderx/Jit/codegen/autogen.h"

#include "cinderx/Common/util.h"
#include "cinderx/Jit/code_patcher.h"
#include "cinderx/Jit/codegen/arch.h"
#include "cinderx/Jit/codegen/gen_asm_utils.h"
#include "cinderx/Jit/frame.h"
#include "cinderx/Jit/generators_rt.h"
#include "cinderx/Jit/jit_rt.h"
#include "cinderx/Jit/lir/instruction.h"
#include "cinderx/Jit/lir/printer.h"

#include <type_traits>
#include <vector>

using namespace asmjit;
using namespace jit::lir;
using namespace jit::codegen;

namespace jit::codegen::autogen {

#define ANY "*"

namespace {
// Add a pattern to an existing trie tree. If the trie tree is nullptr, create a
// new one.
std::unique_ptr<PatternNode> addPattern(
    std::unique_ptr<PatternNode> patterns,
    const std::string& s,
    PatternNode::func_t func) {
  JIT_DCHECK(!s.empty(), "pattern string should not be empty.");

  if (patterns == nullptr) {
    patterns = std::make_unique<PatternNode>();
  }

  PatternNode* cur = patterns.get();
  for (auto& c : s) {
    auto iter = cur->next.find(c);
    if (iter == cur->next.end()) {
      cur = cur->next.emplace(c, std::make_unique<PatternNode>())
                .first->second.get();
      continue;
    }
    cur = iter->second.get();
  }

  JIT_DCHECK(cur->func == nullptr, "Found duplicated pattern.");
  cur->func = func;

  return patterns;
}

// Find the function associated to the pattern given in s.
PatternNode::func_t findByPattern(
    const PatternNode* patterns,
    const std::string& s) {
  auto cur = patterns;
  if (s.empty()) {
    // handle the special case of matching '*' with an empty string
    auto iter = cur->next.find('*');
    if (iter != cur->next.end()) {
      cur = iter->second.get();
      return cur->func;
    }
  }
  for (auto& c : s) {
    auto iter = cur->next.find(c);
    if (iter != cur->next.end()) {
      cur = iter->second.get();
      continue;
    }

    iter = cur->next.find('?');
    if (iter != cur->next.end()) {
      cur = iter->second.get();
      continue;
    }

    iter = cur->next.find('*');
    if (iter != cur->next.end()) {
      cur = iter->second.get();
      break;
    }

    return nullptr;
  }

  return cur->func;
}

} // namespace

// this function generates operand patterns from the inputs and outputs
// of a given instruction instr and calls the correspoinding code generation
// functions.
void AutoTranslator::translateInstr(Environ* env, const Instruction* instr)
    const {
  auto opcode = instr->opcode();
  if (opcode == Instruction::kBind) {
    return;
  }
  auto& instr_map = map_get(instr_rule_map_, opcode);

  std::string pattern;
  pattern.reserve(instr->getNumInputs() + instr->getNumOutputs());

  if (instr->getNumOutputs()) {
    auto operand = instr->output();

    switch (operand->type()) {
      case OperandBase::kReg:
        pattern += (operand->isVecD() ? "X" : "R");
        break;
      case OperandBase::kStack:
      case OperandBase::kMem:
      case OperandBase::kInd:
        pattern += "M";
        break;
      default:
        JIT_ABORT("Output operand has to be of type register or memory");
    }
  }

  instr->foreachInputOperand([&](const OperandBase* operand) {
    switch (operand->type()) {
      case OperandBase::kReg:
        pattern += (operand->isVecD() ? "x" : "r");
        break;
      case OperandBase::kStack:
      case OperandBase::kMem:
      case OperandBase::kInd:
        pattern += "m";
        break;
      case OperandBase::kImm:
        pattern += "i";
        break;
      case OperandBase::kLabel:
        pattern += "b";
        break;
      default:
        JIT_ABORT(
            "Illegal input type {} for instruction {}",
            operand->type(),
            *instr);
    }
  });

  auto func = findByPattern(instr_map.get(), pattern);
  JIT_CHECK(
      func != nullptr,
      "No pattern found for opcode {}: {}",
      InstrProperty::getProperties(instr).name,
      pattern);
  func(env, instr);
}

namespace {

void fillLiveValueLocations(
    CodeRuntime* code_runtime,
    std::size_t deopt_idx,
    const Instruction* instr,
    size_t begin_input,
    size_t end_input) {
  ThreadedCompileSerialize guard;

  DeoptMetadata& deopt_meta = code_runtime->getDeoptMetadata(deopt_idx);
  for (size_t i = begin_input; i < end_input; i++) {
    auto loc = instr->getInput(i)->getPhyRegOrStackSlot();
    deopt_meta.live_values[i - begin_input].location = loc;
  }
}

// Translate GUARD instruction
void TranslateGuard(Environ* env, const Instruction* instr) {
#if defined(CINDER_X86_64)
  auto as = env->as;

  // the first four operands of the guard instruction are:
  //   * kind
  //   * deopt meta id
  //   * guard var (physical register) (0 for AlwaysFail)
  //   * target (for GuardIs and GuardType, and 0 for all others)

  auto deopt_label = as->newLabel();
  auto kind = instr->getInput(0)->getConstant();

  arch::Gp reg = x86::rax;
  bool is_double = false;
  if (kind != kAlwaysFail) {
    if (instr->getInput(2)->dataType() == jit::lir::OperandBase::kDouble) {
      assert(kind == kNotZero);
      auto vecd_reg = AutoTranslator::getVecD(instr->getInput(2));
      as->ptest(vecd_reg, vecd_reg);
      as->jz(deopt_label);
      is_double = true;
    } else {
      reg = AutoTranslator::getGp(instr->getInput(2));
    }
  }

  auto emit_cmp = [&](auto reg_arg) {
    constexpr size_t kTargetIndex = 3;
    auto target_opnd = instr->getInput(kTargetIndex);
    if (target_opnd->isImm() || target_opnd->isMem()) {
      auto target = target_opnd->getConstantOrAddress();
      JIT_DCHECK(
          fitsSignedInt<32>(target),
          "Constant operand should fit in a 32-bit register, got {:x}.",
          target);
      as->cmp(reg_arg, target);
    } else {
      auto target_reg = AutoTranslator::getGp(target_opnd);
      as->cmp(reg_arg, target_reg);
    }
  };

  if (!is_double) {
    switch (kind) {
      case kNotZero: {
        as->test(reg, reg);
        as->jz(deopt_label);
        break;
      }
      case kNotNegative: {
        as->test(reg, reg);
        as->js(deopt_label);
        break;
      }
      case kZero: {
        as->test(reg, reg);
        as->jnz(deopt_label);
        break;
      }
      case kAlwaysFail:
        as->jmp(deopt_label);
        break;
      case kIs:
        emit_cmp(reg);
        as->jne(deopt_label);
        break;
      case kHasType: {
        emit_cmp(x86::qword_ptr(reg, offsetof(PyObject, ob_type)));
        as->jne(deopt_label);
        break;
      }
    }
  }

  auto index = instr->getInput(1)->getConstant();
  // skip the first four inputs in Guard, which are
  // kind, deopt_meta id, guard var, and target.
  fillLiveValueLocations(env->code_rt, index, instr, 4, instr->getNumInputs());
  env->deopt_exits.emplace_back(index, deopt_label, instr);
#else
  CINDER_UNSUPPORTED
#endif
}

void TranslateDeoptPatchpoint(Environ* env, const Instruction* instr) {
  auto as = env->as;

  auto patcher =
      reinterpret_cast<JumpPatcher*>(instr->getInput(0)->getMemoryAddress());

  // Generate patchpoint by writing in an appropriately sized nop.  As a future
  // optimization, we may be able to avoid reserving space for the patchpoint if
  // we can prove that the following bytes are not the target of a jump.
  auto patchpoint_label = as->newLabel();
  as->bind(patchpoint_label);

  auto stored_bytes = patcher->storedBytes();
  as->embed(stored_bytes.data(), stored_bytes.size());

  // Fill in deopt metadata
  auto index = instr->getInput(1)->getConstant();
  // skip the first two inputs which are the patcher and deopt metadata id
  fillLiveValueLocations(env->code_rt, index, instr, 2, instr->getNumInputs());
  auto deopt_label = as->newLabel();
  env->deopt_exits.emplace_back(index, deopt_label, instr);

  // The runtime will link the patcher to the appropriate point in the code
  // once code generation has completed.
  env->pending_deopt_patchers.emplace_back(
      patcher, patchpoint_label, deopt_label);
}

void TranslateCompare(Environ* env, const Instruction* instr) {
#if defined(CINDER_X86_64)
  auto as = env->as;
  const OperandBase* inp0 = instr->getInput(0);
  const OperandBase* inp1 = instr->getInput(1);

  if (inp1->isImm() || inp1->isMem()) {
    as->cmp(AutoTranslator::getGp(inp0), inp1->getConstantOrAddress());
  } else if (!inp1->isVecD()) {
    as->cmp(AutoTranslator::getGp(inp0), AutoTranslator::getGp(inp1));
  } else {
    as->comisd(AutoTranslator::getVecD(inp0), AutoTranslator::getVecD(inp1));
  }
  auto output = AutoTranslator::getGp(instr->output());
  switch (instr->opcode()) {
    case Instruction::kEqual:
      as->sete(output);
      break;
    case Instruction::kNotEqual:
      as->setne(output);
      break;
    case Instruction::kGreaterThanSigned:
      as->setg(output);
      break;
    case Instruction::kGreaterThanEqualSigned:
      as->setge(output);
      break;
    case Instruction::kLessThanSigned:
      as->setl(output);
      break;
    case Instruction::kLessThanEqualSigned:
      as->setle(output);
      break;
    case Instruction::kGreaterThanUnsigned:
      as->seta(output);
      break;
    case Instruction::kGreaterThanEqualUnsigned:
      as->setae(output);
      break;
    case Instruction::kLessThanUnsigned:
      as->setb(output);
      break;
    case Instruction::kLessThanEqualUnsigned:
      as->setbe(output);
      break;
    default:
      JIT_ABORT("bad instruction for TranslateCompare");
  }
  if (instr->output()->dataType() != OperandBase::k8bit) {
    as->movzx(
        AutoTranslator::getGp(instr->output()),
        asmjit::x86::gpb(instr->output()->getPhyRegister().loc));
  }
#else
  CINDER_UNSUPPORTED
#endif
}

void translateIntToBool(Environ* env, const Instruction* instr) {
#if defined(CINDER_X86_64)
  x86::Builder* as = env->as;
  const OperandBase* input = instr->getInput(0);
  x86::Gp output = AutoTranslator::getGp(instr->output());
  JIT_CHECK(
      instr->output()->dataType() == OperandBase::k8bit,
      "Output should be 8bits, not {}",
      instr->output()->dataType());
  if (input->isImm()) {
    as->mov(output, input->getConstant() ? 1 : 0);
  } else {
    as->test(AutoTranslator::getGp(input), AutoTranslator::getGp(input));
    as->setne(output);
  }
#else
  CINDER_UNSUPPORTED
#endif
}

// Store meta-data about this yield in a generator suspend data pointed to by
// suspend_data_r. Data includes things like the address to resume execution at,
// and owned entries in the suspended spill data needed for GC operations etc.
void emitStoreGenYieldPoint(
    arch::Builder* as,
    Environ* env,
    const Instruction* yield,
    asmjit::Label resume_label,
    arch::Gp suspend_data_r,
    arch::Gp scratch_r) {
  bool is_yield_from = yield->isYieldFrom() ||
      yield->isYieldFromSkipInitialSend() ||
      yield->isYieldFromHandleStopAsyncIteration();

  auto calc_spill_offset = [&](size_t live_input_n) {
    PhyLocation mem = yield->getInput(live_input_n)->getStackSlot();
    return mem.loc / kPointerSize;
  };

  size_t input_n = yield->getNumInputs() - 1;
  size_t deopt_idx = yield->getInput(input_n)->getConstant();

  size_t live_regs_input = input_n - 1;
  int num_live_regs = yield->getInput(live_regs_input)->getConstant();
  fillLiveValueLocations(
      env->code_rt,
      deopt_idx,
      yield,
      live_regs_input - num_live_regs,
      live_regs_input);

  auto yield_from_offset =
      is_yield_from ? calc_spill_offset(2) : kInvalidYieldFromOffset;
  GenYieldPoint* gen_yield_point = env->code_rt->addGenYieldPoint(
      GenYieldPoint{deopt_idx, yield_from_offset});

  env->unresolved_gen_entry_labels.emplace(gen_yield_point, resume_label);
  if (yield->origin()) {
    env->pending_debug_locs.emplace_back(resume_label, yield->origin());
  }

#if defined(CINDER_X86_64)
  as->mov(scratch_r, reinterpret_cast<uint64_t>(gen_yield_point));
  auto yieldPointOffset = offsetof(GenDataFooter, yieldPoint);
  as->mov(x86::qword_ptr(suspend_data_r, yieldPointOffset), scratch_r);
#else
  CINDER_UNSUPPORTED
#endif
}

void emitLoadResumedYieldInputs(
    arch::Builder* as,
    const Instruction* instr,
    PhyLocation sent_in_source_loc,
    arch::Gp tstate_reg) {
#if defined(CINDER_X86_64)
  PhyLocation tstate = instr->getInput(0)->getStackSlot();
  as->mov(x86::ptr(x86::rbp, tstate.loc), tstate_reg);

  const lir::Operand* target = instr->output();

  if (target->isStack()) {
    as->mov(
        x86::ptr(x86::rbp, target->getStackSlot().loc),
        x86::gpq(sent_in_source_loc.loc));
    return;
  }

  if (target->isReg()) {
    PhyLocation target_loc = target->getPhyRegister();
    if (target_loc != sent_in_source_loc) {
      as->mov(x86::gpq(target_loc.loc), x86::gpq(sent_in_source_loc.loc));
    }
    return;
  }

  JIT_CHECK(
      target->isNone(),
      "Have an output that isn't a register or a stack slot, {}",
      target->type());
#else
  CINDER_UNSUPPORTED
#endif
}

void translateYieldInitial(Environ* env, const Instruction* instr) {
#if defined(CINDER_X86_64)
#if PY_VERSION_HEX < 0x030C0000
  arch::Builder* as = env->as;

  // Load tstate into RDI for call to JITRT_MakeGenObject*.

  // Consider avoiding reloading the tstate in from memory if it was already in
  // a register before spilling. Still needs to be in memory though so it can be
  // recovered after calling JITRT_MakeGenObject* which will trash it.
  PhyLocation tstate = instr->getInput(0)->getStackSlot();
  as->mov(x86::rdi, x86::ptr(x86::rbp, tstate.loc));

  // Make a generator object to be returned by the epilogue.
  as->lea(x86::rsi, x86::ptr(env->gen_resume_entry_label));
  JIT_CHECK(
      env->shadow_frames_and_spill_size % kPointerSize == 0,
      "Bad spill alignment");
  as->mov(x86::rdx, env->shadow_frames_and_spill_size / kPointerSize);
  as->mov(x86::rcx, reinterpret_cast<uint64_t>(env->code_rt));
  JIT_CHECK(instr->origin()->IsInitialYield(), "expected InitialYield");
  PyCodeObject* code = static_cast<const hir::InitialYield*>(instr->origin())
                           ->frameState()
                           ->code;
  as->mov(x86::r8, reinterpret_cast<uint64_t>(code));
  if (code->co_flags & CO_COROUTINE) {
    emitCall(*env, reinterpret_cast<uint64_t>(JITRT_MakeGenObjectCoro), instr);
  } else if (code->co_flags & CO_ASYNC_GENERATOR) {
    emitCall(
        *env, reinterpret_cast<uint64_t>(JITRT_MakeGenObjectAsyncGen), instr);
  } else {
    emitCall(*env, reinterpret_cast<uint64_t>(JITRT_MakeGenObject), instr);
  }
  // Resulting generator is now in RAX for filling in below and epilogue return.
  const auto gen_reg = x86::rax;

  // Exit early if return from JITRT_MakeGenObject was nullptr.
  as->test(gen_reg, gen_reg);
  as->jz(env->hard_exit_label);

  // Set RDI to gen->gi_jit_data for use in emitStoreGenYieldPoint() and data
  // copy using 'movsq' below.
  auto gi_jit_data_offset = offsetof(PyGenObject, gi_jit_data);
  as->mov(x86::rdi, x86::ptr(gen_reg, gi_jit_data_offset));

  // Arbitrary scratch register for use in emitStoreGenYieldPoint().
  auto scratch_r = x86::r9;
  asmjit::Label resume_label = as->newLabel();
  emitStoreGenYieldPoint(as, env, instr, resume_label, x86::rdi, scratch_r);

  // Store variables spilled by this point to generator.
  int spill_bytes = env->initial_yield_spill_size_;
  JIT_CHECK(spill_bytes % kPointerSize == 0, "Bad spill alignment");

  // Point rsi at the bottom word of the current spill space.
  as->lea(x86::rsi, x86::ptr(x86::rbp, -spill_bytes));
  // Point rdi at the bottom word of the generator's spill space.
  as->sub(x86::rdi, spill_bytes);
  as->mov(x86::rcx, spill_bytes / kPointerSize);
  as->rep().movsq();

  // Jump to bottom half of epilogue
  as->jmp(env->hard_exit_label);

  // Resumed execution in this generator begins here
  as->bind(resume_label);

  // Sent in value is in RSI, and tstate is in RCX from resume entry-point args
  emitLoadResumedYieldInputs(as, instr, RSI, x86::rcx);
#else
  arch::Builder* as = env->as;

  // Load tstate into RDI for call to
  // JITRT_UnlinkGenFrameAndReturnGenDataFooter.

  // Consider avoiding reloading the tstate in from memory if it was already in
  // a register before spilling. Still needs to be in memory though so it can be
  // recovered after calling JITRT_MakeGenObject* which will trash it.
  PhyLocation tstate = instr->getInput(0)->getStackSlot();
  as->mov(x86::rdi, x86::ptr(x86::rbp, tstate.loc));

  emitCall(
      *env,
      reinterpret_cast<uint64_t>(JITRT_UnlinkGenFrameAndReturnGenDataFooter),
      instr);
  // This will return pointers to a generator in RAX and JIT data in RDX.

  // Arbitrary scratch register for use in emitStoreGenYieldPoint(). Any
  // caller-saved register not used in this scope will do because we're on the
  // exit path now.
  auto scratch_r = x86::r9;
  asmjit::Label resume_label = as->newLabel();
  emitStoreGenYieldPoint(as, env, instr, resume_label, x86::rdx, scratch_r);

  // Jump to epilogue
  as->jmp(env->exit_for_yield_label);

  // Resumed execution in this generator begins here
  as->bind(resume_label);

  // Sent in value is in RSI, and tstate is in RCX from resume entry-point args
  emitLoadResumedYieldInputs(as, instr, RSI, x86::rcx);
#endif
#else
  CINDER_UNSUPPORTED
#endif
}

void translateYieldValue(Environ* env, const Instruction* instr) {
#if defined(CINDER_X86_64)
  arch::Builder* as = env->as;

  // Make sure tstate is in RDI for use in epilogue.
  PhyLocation tstate = instr->getInput(0)->getStackSlot();
  as->mov(x86::rdi, x86::ptr(x86::rbp, tstate.loc));

  // Value to send goes to RAX so it can be yielded (returned) by epilogue.
  if (instr->getInput(1)->isImm()) {
    as->mov(x86::rax, instr->getInput(1)->getConstant());
  } else {
    PhyLocation value_out = instr->getInput(1)->getStackSlot();
    as->mov(x86::rax, x86::ptr(x86::rbp, value_out.loc));
  }

  // Arbitrary scratch register for use in emitStoreGenYieldPoint()
  auto scratch_r = x86::r9;
  auto resume_label = as->newLabel();
  emitStoreGenYieldPoint(as, env, instr, resume_label, x86::rbp, scratch_r);

  // Jump to epilogue
  as->jmp(env->exit_for_yield_label);

  // Resumed execution in this generator begins here
  as->bind(resume_label);

  // Sent in value is in RSI, and tstate is in RCX from resume entry-point args
  emitLoadResumedYieldInputs(as, instr, RSI, x86::rcx);
#else
  CINDER_UNSUPPORTED
#endif
}

void translateYieldFrom(Environ* env, const Instruction* instr) {
#if defined(CINDER_X86_64)
  arch::Builder* as = env->as;
  bool skip_initial_send = instr->isYieldFromSkipInitialSend();

  // Make sure tstate is in RDI for use in epilogue and here.
  PhyLocation tstate = instr->getInput(0)->getStackSlot();
  auto tstate_phys_reg = x86::rdi;
  as->mov(tstate_phys_reg, x86::ptr(x86::rbp, tstate.loc));

  // If we're skipping the initial send the send value is actually the first
  // value to yield and so needs to go into RAX to be returned. Otherwise,
  // put initial send value in RSI, the same location future send values will
  // be on resume.
  PhyLocation send_value = instr->getInput(1)->getStackSlot();
  const auto send_value_phys_reg = skip_initial_send ? RAX : RSI;
  as->mov(
      x86::gpq(send_value_phys_reg.loc), x86::ptr(x86::rbp, send_value.loc));

  asmjit::Label yield_label = as->newLabel();
  if (skip_initial_send) {
    as->jmp(yield_label);
  } else {
    // Setup call to JITRT_GenSend

    // Put tstate and the current generator into RCX and RDI respectively, and
    // set finish_yield_from (RDX) to 0. This register setup matches that when
    // `resume_label` is reached from the resume entry.
    auto gen_offs = offsetof(GenDataFooter, gen);
    as->mov(x86::rcx, tstate_phys_reg);
    as->mov(x86::rdi, x86::ptr(x86::rbp, gen_offs));
    as->xor_(x86::rdx, x86::rdx);
  }

  // Resumed execution begins here
  auto resume_label = as->newLabel();
  as->bind(resume_label);

  // Save tstate from resume to callee-saved reigster.
  as->mov(x86::rbx, x86::rcx);

  // 'send_value', and 'finish_yield_from' will already be in RSI and RCX
  // respectively, either from code above on initial start or from resume entry
  // point args.

  // Load sub-iterator into RDI
  PhyLocation iter_slot = instr->getInput(2)->getStackSlot();
  as->mov(x86::rdi, x86::ptr(x86::rbp, iter_slot.loc));

  uint64_t func = reinterpret_cast<uint64_t>(
      instr->isYieldFromHandleStopAsyncIteration()
          ? JITRT_GenSendHandleStopAsyncIteration
          : JITRT_GenSend);
  emitCall(*env, func, instr);
  // Yielded or final result value now in RAX. If the result was nullptr then
  // done will be set so we'll correctly jump to the following CheckExc.
  const auto yf_result_phys_reg = RAX;
  const auto done_r = x86::rdx;

  // Restore tstate from callee-saved register.
  as->mov(tstate_phys_reg, x86::rbx);

  // If not done, jump to epilogue which will yield/return the value from
  // JITRT_GenSend in RAX.
  as->test(done_r, done_r);
  asmjit::Label done_label = as->newLabel();
  as->jnz(done_label);

  as->bind(yield_label);
  // Arbitrary scratch register for use in emitStoreGenYieldPoint()
  auto scratch_r = x86::r9;
  emitStoreGenYieldPoint(as, env, instr, resume_label, x86::rbp, scratch_r);
  as->jmp(env->exit_for_yield_label);

  as->bind(done_label);
  emitLoadResumedYieldInputs(as, instr, yf_result_phys_reg, tstate_phys_reg);
#else
  CINDER_UNSUPPORTED
#endif
}

// ***********************************************************************
// The following templates and macros implement the auto generation table.
// The generator table defines a hash table, whose key is instruction type,
// and value is another hash table mapping instruction operand pattern and
// a function carrying out certain Actions for the instruction with the
// operand pattern.
// The list of Actions are encoded in the template class RuleActions as its
// template arguments. Currently, there are two types of Actions:
//   * AsmAction - generate an asm instruction
//   * CallAction - call a user defined instruction
// The Action classes are also templates, whose argument lists encode the
// parameters for the Action. For example, an AsmAction's argument list has
// the assembly instruction mnemonic and its operands.
// ***********************************************************************
template <int N>
const OperandBase* LIROperandMapper(const Instruction* instr) {
  auto num_outputs = instr->getNumOutputs();
  if (N < num_outputs) {
    return instr->output();
  } else {
    return instr->getInput(N - num_outputs);
  }
}

template <int N>
int LIROperandSizeMapper(const Instruction* instr) {
  auto size_type = InstrProperty::getProperties(instr).opnd_size_type;
  switch (size_type) {
    case kDefault:
      return LIROperandMapper<N>(instr)->sizeInBits();
    case kAlways64:
      return 64;
    case kOut:
      return LIROperandMapper<0>(instr)->sizeInBits();
  }

  JIT_ABORT("Unknown size type");
}

template <int N>
struct ImmOperand {
  using asmjit_type = const asmjit::Imm&;

  static asmjit::Imm GetAsmOperand(Environ*, const Instruction* instr) {
    return asmjit::Imm(LIROperandMapper<N>(instr)->getConstant());
  }
};

template <typename T>
struct ImmOperandNegate {
  using asmjit_type = const asmjit::Imm&;

  static asmjit::Imm GetAsmOperand(Environ* env, const Instruction* instr) {
    return asmjit::Imm(
        -T::GetAsmOperand(env, instr).template valueAs<int64_t>());
  }
};

template <typename T>
struct ImmOperandInvert {
  using asmjit_type = const asmjit::Imm&;

  static asmjit::Imm GetAsmOperand(Environ* env, const Instruction* instr) {
    return asmjit::Imm(
        ~T::GetAsmOperand(env, instr).template valueAs<uint64_t>());
  }
};

template <int N, int Size = -1>
struct RegOperand {
  using asmjit_type = const arch::Gp&;
  static arch::Gp GetAsmOperand(Environ*, const Instruction* instr) {
    static_assert(
        Size == -1 || Size == 8 || Size == 16 || Size == 32 || Size == 64,
        "Invalid Size");

#if defined(CINDER_X86_64)
    int size = Size == -1 ? LIROperandSizeMapper<N>(instr) : Size;

    PhyLocation reg = LIROperandMapper<N>(instr)->getPhyRegister();
    switch (size) {
      case 8:
        return asmjit::x86::gpb(reg.loc);
      case 16:
        return asmjit::x86::gpw(reg.loc);
      case 32:
        return asmjit::x86::gpd(reg.loc);
      case 64:
        return asmjit::x86::gpq(reg.loc);
    }
#else
    CINDER_UNSUPPORTED
#endif

    JIT_ABORT("Incorrect operand size.");
  }
};

template <int N>
struct VecDOperand {
  using asmjit_type = const arch::VecD&;
  static arch::VecD GetAsmOperand(Environ*, const Instruction* instr) {
#if defined(CINDER_X86_64)
    return asmjit::x86::xmm(
        LIROperandMapper<N>(instr)->getPhyRegister().loc - VECD_REG_BASE);
#else
    CINDER_UNSUPPORTED
    return arch::VecD();
#endif
  }
};

#define OP(v)                                       \
  typename std::conditional_t<                      \
      pattern[v] == 'i',                            \
      ImmOperand<v>,                                \
      std::conditional_t<                           \
          (pattern[v] == 'x' || pattern[v] == 'X'), \
          VecDOperand<v>,                           \
          RegOperand<v>>>

#define REG_OP(v, size) RegOperand<v, size>

arch::Mem AsmIndirectOperandBuilder(const OperandBase* operand) {
  JIT_DCHECK(operand->isInd(), "operand should be an indirect reference");

#if defined(CINDER_X86_64)
  auto indirect = operand->getMemoryIndirect();

  OperandBase* base = indirect->getBaseRegOperand();
  OperandBase* index = indirect->getIndexRegOperand();

  if (index == nullptr) {
    return asmjit::x86::ptr(
        x86::gpq(base->getPhyRegister().loc), indirect->getOffset());
  } else {
    return asmjit::x86::ptr(
        x86::gpq(base->getPhyRegister().loc),
        x86::gpq(index->getPhyRegister().loc),
        indirect->getMultipiler(),
        indirect->getOffset());
  }
#else
  CINDER_UNSUPPORTED
  return arch::Mem();
#endif
}

template <int N>
struct MemOperand {
  using asmjit_type = const arch::Mem&;
  static arch::Mem GetAsmOperand(Environ*, const Instruction* instr) {
#if defined(CINDER_X86_64)
    const OperandBase* operand = LIROperandMapper<N>(instr);
    auto size = LIROperandSizeMapper<N>(instr) / 8;

    asmjit::x86::Mem memptr;
    if (operand->isStack()) {
      memptr = asmjit::x86::ptr(asmjit::x86::rbp, operand->getStackSlot().loc);
    } else if (operand->isMem()) {
      memptr = asmjit::x86::ptr(
          reinterpret_cast<uint64_t>(operand->getMemoryAddress()));
    } else if (operand->isInd()) {
      memptr = AsmIndirectOperandBuilder(operand);
    } else {
      JIT_ABORT("Unsupported operand type.");
    }

    memptr.setSize(size);
    return memptr;
#else
    CINDER_UNSUPPORTED
    return arch::Mem();
#endif
  }
};

#define MEM(m) MemOperand<m>
#define STK(v) MemOperand<v>

template <int N>
struct LabelOperand {
  using asmjit_type = const asmjit::Label&;
  static asmjit::Label GetAsmOperand(Environ* env, const Instruction* instr) {
    auto block = LIROperandMapper<N>(instr)->getBasicBlock();
    return map_get(env->block_label_map, block);
  }
};

#define LBL(v) LabelOperand<v>

template <typename... Args>
struct OperandList;

template <typename FuncType, FuncType func, typename OpndList>
struct AsmAction;

template <typename FuncType, FuncType func, typename... OpndTypes>
struct AsmAction<FuncType, func, OperandList<OpndTypes...>> {
  static void eval(Environ* env, const Instruction* instr) {
    static_cast<void>(instr);
    (env->as->*func)(OpndTypes::GetAsmOperand(env, instr)...);
  }
};

template <typename... Args>
struct AsminstructionType {
  using type = asmjit::Error (arch::EmitterExplicitT<arch::Builder>::*)(
      typename Args::asmjit_type...);
};

template <void (*func)(Environ*, const Instruction*)>
struct CallAction {
  static void eval(Environ* env, const Instruction* instr) {
    func(env, instr);
  }
};

template <typename... Actions>
struct RuleActions;

template <typename AAction, typename... Actions>
struct RuleActions<AAction, Actions...> {
  static void eval(Environ* env, const Instruction* instr) {
    AAction::eval(env, instr);
    RuleActions<Actions...>::eval(env, instr);
  }
};

template <>
struct RuleActions<> {
  static void eval(Environ*, const Instruction*) {}
};

struct AddDebugEntryAction {
  static void eval(Environ* env, const Instruction* instr) {
    asmjit::Label label = env->as->newLabel();
    env->as->bind(label);
    if (instr->origin()) {
      env->pending_debug_locs.emplace_back(label, instr->origin());
    }
  }
};

} // namespace

#define ASM(instr, args...)                    \
  AsmAction<                                   \
      typename AsminstructionType<args>::type, \
      &arch::Builder::instr,                   \
      OperandList<args>>

// Can't be named CALL as that conflicts with the opcode.
#define CALL_C(func) CallAction<func>

#define ADDDEBUGENTRY() AddDebugEntryAction

#define BEGIN_RULE_TABLE void AutoTranslator::initTable() {
#define END_RULE_TABLE }

#define BEGIN_RULES(__t)                                \
  {                                                     \
    auto& __rules = instr_rule_map_                     \
                        .emplace(                       \
                            std::piecewise_construct,   \
                            std::forward_as_tuple(__t), \
                            std::forward_as_tuple())    \
                        .first->second;

#define END_RULES }
#define GEN(s, actions...)                                  \
  {                                                         \
    UNUSED constexpr char pattern[] = s;                    \
    using rule_actions = RuleActions<actions>;              \
    auto gen = [](Environ* env, const Instruction* instr) { \
      rule_actions::eval(env, instr);                       \
    };                                                      \
    __rules = addPattern(std::move(__rules), s, gen);       \
  }

// ***********************************************************************
// Definition of Auto Generation Table
// The table consisting of multiple rules, and the rules for the same LIR
// instruction are grouped by BEGIN_RULES(LIR instruction type) and
// END_RULES.
// GEN defines a rule for a certain operand pattern of the LIR instruction,
// and maps it to a list of actions:
//   GEN(<operand pattern>, action1, action2, ...)
//
// The operand pattern is defined by a string, and each character in the string
// correpsonds to an operand of the instruction. The character can be one
// of the following:
//   * 'R' - general purpose register operand output
//   * 'r' - general purpose register operand input
//   * 'X' - floating-point register operand output
//   * 'x' - floating-point register operand input
//   * 'i' - immediate operand input
//   * 'M' - memory stack operand output
//   * 'm' - memory stack operand input
// Wildcards "?" and "*" can also be used in patterns, where "?" represents any
// one of the types listed above and "*" represents one or more above types.
// Please note that while "?" can appear anywhere in a pattern, "*" can only be
// used at the end of a pattern.
// The actions can be ASM and CALL_C, meaning generating an assembly instruction
// and call a user-defined function, respectively. The first argument of ASM
// action is the mnemonic of the instruction to be generated, and the following
// arguments are the operands to the instruction. Currently, we have four types
// of assembly instruction operands:
//   * OP  - either an immediate operand or register oeprand
//   * STK - a memory stack location [RBP - ?]
//   * LBL - a label to a basic block
//   * MEM - a memory operand. The size of the memory operand will be set to the
//           size of the LIR instruction operand specified by the first argument
//           of MEM.
// The assembly instruction operands are constructed from one or more LIR
// instruction operands. To specify the LIR operands, we use indices
// of the pattern string. For example:
//   GEN("Rri", ASM(mov, OP(0), MEM(0, 1, 2)))
// means generating a mov instruction, whose first operand is a
// register/immediate operand, constructed from the only output of the LIR
// instruction, and the second operand is memory operand, constructed from the
// register input and the immediate input of the LIR instruction. The size of
// the memory operand is set to the size of the output of the LIR instruction.
// ***********************************************************************

#if defined(CINDER_X86_64)
// clang-format off
BEGIN_RULE_TABLE

BEGIN_RULES(Instruction::kLea)
  GEN("Rm", ASM(lea, OP(0), MEM(1)))
END_RULES

BEGIN_RULES(Instruction::kCall)
  GEN("Ri", ASM(call, OP(1)), ADDDEBUGENTRY())
  GEN("Rr", ASM(call, OP(1)), ADDDEBUGENTRY())
  GEN("i", ASM(call, OP(0)), ADDDEBUGENTRY())
  GEN("r", ASM(call, OP(0)), ADDDEBUGENTRY())
  GEN("m", ASM(call, STK(0)), ADDDEBUGENTRY())
END_RULES

BEGIN_RULES(Instruction::kMove)
  GEN("Rr", ASM(mov, OP(0), OP(1)))
  GEN("Ri", ASM(mov, OP(0), OP(1)))
  GEN("Rm", ASM(mov, OP(0), MEM(1)))
  GEN("Mr", ASM(mov, MEM(0), OP(1)))
  GEN("Mi", ASM(mov, MEM(0), OP(1)))
  GEN("Xx", ASM(movsd, OP(0), OP(1)))
  GEN("Xm", ASM(movsd, OP(0), MEM(1)))
  GEN("Mx", ASM(movsd, MEM(0), OP(1)))
  GEN("Xr", ASM(movq, OP(0), OP(1)))
  GEN("Rx", ASM(movq, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kGuard)
  GEN(ANY, CALL_C(TranslateGuard));
END_RULES

BEGIN_RULES(Instruction::kDeoptPatchpoint)
  GEN(ANY, CALL_C(TranslateDeoptPatchpoint));
END_RULES

BEGIN_RULES(Instruction::kNegate)
  GEN("r", ASM(neg, OP(0)))
  GEN("Ri", ASM(mov, OP(0), ImmOperandNegate<OP(1)>))
  GEN("Rr", ASM(mov, OP(0), OP(1)), ASM(neg, OP(0)))
  GEN("Rm", ASM(mov, OP(0), STK(1)), ASM(neg, OP(0)))
END_RULES

BEGIN_RULES(Instruction::kInvert)
  GEN("Ri", ASM(mov, OP(0), ImmOperandInvert<OP(1)>))
  GEN("Rr", ASM(mov, OP(0), OP(1)), ASM(not_, OP(0)))
  GEN("Rm", ASM(mov, OP(0), STK(1)), ASM(not_, OP(0)))
END_RULES

BEGIN_RULES(Instruction::kMovZX)
  GEN("Rr", ASM(movzx, OP(0), OP(1)))
  GEN("Rm", ASM(movzx, OP(0), STK(1)))
END_RULES

BEGIN_RULES(Instruction::kMovSX)
  GEN("Rr", ASM(movsx, OP(0), OP(1)))
  GEN("Rm", ASM(movsx, OP(0), STK(1)))
END_RULES

BEGIN_RULES(Instruction::kMovSXD)
  GEN("Rr", ASM(movsxd, OP(0), OP(1)))
  GEN("Rm", ASM(movsxd, OP(0), STK(1)))
END_RULES

BEGIN_RULES(Instruction::kUnreachable)
  GEN(ANY, ASM(ud2))
END_RULES

#define DEF_BINARY_OP_RULES(name, instr) \
  BEGIN_RULES(Instruction::name) \
    GEN("ri", ASM(instr, OP(0), OP(1))) \
    GEN("rr", ASM(instr, OP(0), OP(1))) \
    GEN("rm", ASM(instr, OP(0), STK(1))) \
    /* rewriteBinaryOpInstrs() makes it safe to write the output before reading
     * all inputs without inputs_live_across being set for most binary ops; see
     * postalloc.cpp for details. */ \
    GEN("Rri", ASM(mov, OP(0), OP(1)), ASM(instr, OP(0), OP(2))) \
    GEN("Rrr", ASM(mov, OP(0), OP(1)), ASM(instr, OP(0), OP(2))) \
    GEN("Rrm", ASM(mov, OP(0), OP(1)), ASM(instr, OP(0), STK(2))) \
  END_RULES

DEF_BINARY_OP_RULES(kAdd, add)
DEF_BINARY_OP_RULES(kSub, sub)
DEF_BINARY_OP_RULES(kAnd, and_)
DEF_BINARY_OP_RULES(kOr, or_)
DEF_BINARY_OP_RULES(kXor, xor_)
DEF_BINARY_OP_RULES(kMul, imul)

BEGIN_RULES(Instruction::kDiv)
  GEN("rrr", ASM(idiv, OP(0), OP(1), OP(2)) )
  GEN("rrm", ASM(idiv, OP(0), OP(1), STK(2)) )
  GEN("rr", ASM(idiv, OP(0), OP(1)) )
  GEN("rm", ASM(idiv, OP(0), STK(1)) )
END_RULES

BEGIN_RULES(Instruction::kDivUn)
  GEN("rrr", ASM(div, OP(0), OP(1), OP(2)) )
  GEN("rrm", ASM(div, OP(0), OP(1), STK(2)) )
  GEN("rr", ASM(div, OP(0), OP(1)) )
  GEN("rm", ASM(div, OP(0), STK(1)) )
END_RULES

#undef DEF_BINARY_OP_RULES

BEGIN_RULES(Instruction::kFadd)
  /* rewriteBinaryOpInstrs() makes it safe to write the output before reading
   * all inputs without inputs_live_across being set for Fadd; see
   * postalloc.cpp for details. */
  GEN("Xxx", ASM(movsd, OP(0), OP(1)), ASM(addsd, OP(0), OP(2)))
  GEN("xx", ASM(addsd, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kFsub)
  GEN("Xxx", ASM(movsd, OP(0), OP(1)), ASM(subsd, OP(0), OP(2)))
  GEN("xx", ASM(subsd, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kFmul)
  /* rewriteBinaryOpInstrs() makes it safe to write the output before reading
   * all inputs without inputs_live_across being set for Fmul; see
   * postalloc.cpp for details. */
  GEN("Xxx", ASM(movsd, OP(0), OP(1)), ASM(mulsd, OP(0), OP(2)))
  GEN("xx", ASM(mulsd, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kFdiv)
  GEN("Xxx", ASM(movsd, OP(0), OP(1)), ASM(divsd, OP(0), OP(2)))
  GEN("xx", ASM(divsd, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kPush)
  GEN("r", ASM(push, OP(0)))
  GEN("m", ASM(push, STK(0)))
  GEN("i", ASM(push, OP(0)))
END_RULES

BEGIN_RULES(Instruction::kPop)
  GEN("R", ASM(pop, OP(0)))
  GEN("M", ASM(pop, STK(0)))
END_RULES

BEGIN_RULES(Instruction::kCdq)
  GEN("Rr", ASM(cdq, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kCwd)
  GEN("Rr", ASM(cwd, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kCqo)
  GEN("Rr", ASM(cqo, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kExchange)
  GEN("Rr", ASM(xchg, OP(0), OP(1)))
  GEN("Xx", ASM(pxor, OP(0), OP(1)),
            ASM(pxor, OP(1), OP(0)),
            ASM(pxor, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kCmp)
  GEN("rr", ASM(cmp, OP(0), OP(1)))
  GEN("ri", ASM(cmp, OP(0), OP(1)))
  GEN("xx", ASM(comisd, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kTest)
  GEN("rr", ASM(test, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kTest32)
  GEN("rr", ASM(test, REG_OP(0, 32), REG_OP(1, 32)))
END_RULES

BEGIN_RULES(Instruction::kBranch)
  GEN("b", ASM(jmp, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchZ)
  GEN("b", ASM(jz, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchNZ)
  GEN("b", ASM(jnz, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchA)
  GEN("b", ASM(ja, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchB)
  GEN("b", ASM(jb, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchAE)
  GEN("b", ASM(jae, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchBE)
  GEN("b", ASM(jbe, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchG)
  GEN("b", ASM(jg, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchL)
  GEN("b", ASM(jl, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchGE)
  GEN("b", ASM(jge, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchLE)
  GEN("b", ASM(jle, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchC)
  GEN("b", ASM(jc, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchNC)
  GEN("b", ASM(jnc, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchO)
  GEN("b", ASM(jo, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchNO)
  GEN("b", ASM(jno, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchS)
  GEN("b", ASM(js, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchNS)
  GEN("b", ASM(jns, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchE)
  GEN("b", ASM(je, LBL(0)))
END_RULES

BEGIN_RULES(Instruction::kBranchNE)
  GEN("b", ASM(jne, LBL(0)))
END_RULES

#define DEF_COMPARE_OP_RULES(name, fpcomp) \
BEGIN_RULES(Instruction::name) \
  GEN("Rrr", CALL_C(TranslateCompare)) \
  GEN("Rri", CALL_C(TranslateCompare)) \
  GEN("Rrm", CALL_C(TranslateCompare)) \
  if (fpcomp) { \
    GEN("Rxx", CALL_C(TranslateCompare)) \
  } \
END_RULES

DEF_COMPARE_OP_RULES(kEqual, true)
DEF_COMPARE_OP_RULES(kNotEqual, true)
DEF_COMPARE_OP_RULES(kGreaterThanUnsigned, true)
DEF_COMPARE_OP_RULES(kGreaterThanEqualUnsigned, true)
DEF_COMPARE_OP_RULES(kLessThanUnsigned, true)
DEF_COMPARE_OP_RULES(kLessThanEqualUnsigned, true)
DEF_COMPARE_OP_RULES(kGreaterThanSigned, false)
DEF_COMPARE_OP_RULES(kGreaterThanEqualSigned, false)
DEF_COMPARE_OP_RULES(kLessThanSigned, false)
DEF_COMPARE_OP_RULES(kLessThanEqualSigned, false)

#undef DEF_COMPARE_OP_RULES

BEGIN_RULES(Instruction::kInc)
  GEN("r", ASM(inc, OP(0)))
  GEN("m", ASM(inc, STK(0)))
END_RULES

BEGIN_RULES(Instruction::kDec)
  GEN("r", ASM(dec, OP(0)))
  GEN("m", ASM(dec, STK(0)))
END_RULES

BEGIN_RULES(Instruction::kBitTest)
  GEN("ri", ASM(bt, OP(0), OP(1)))
END_RULES

BEGIN_RULES(Instruction::kYieldInitial)
  GEN(ANY, CALL_C(translateYieldInitial))
END_RULES

#if PY_VERSION_HEX < 0x030C0000
BEGIN_RULES(Instruction::kYieldFrom)
  GEN(ANY, CALL_C(translateYieldFrom))
END_RULES
#else
// In 3.12+ YieldFrom is a pseudo-op which is YieldValue plus enough
// information to know which live value contains the target iterator. See
// emitStoreGenYieldPoint() for where this is captured. The target iterator is
// used for things like the result of reading gi_yieldfrom.
BEGIN_RULES(Instruction::kYieldFrom)
  GEN(ANY, CALL_C(translateYieldValue))
END_RULES
#endif

BEGIN_RULES(Instruction::kYieldFromSkipInitialSend)
  GEN(ANY, CALL_C(translateYieldFrom))
END_RULES

BEGIN_RULES(Instruction::kYieldFromHandleStopAsyncIteration)
  GEN(ANY, CALL_C(translateYieldFrom))
END_RULES

BEGIN_RULES(Instruction::kYieldValue)
  GEN(ANY, CALL_C(translateYieldValue))
END_RULES

BEGIN_RULES(Instruction::kSelect)
  GEN("Rrri", ASM(mov, OP(0), OP(3)),
              ASM(test, OP(1), OP(1)),
              ASM(cmovnz, OP(0), OP(2)))
END_RULES

BEGIN_RULES(Instruction::kIntToBool)
  GEN("Rr", CALL_C(translateIntToBool))
  GEN("Ri", CALL_C(translateIntToBool))
END_RULES

END_RULE_TABLE
// clang-format on
#else

BEGIN_RULE_TABLE
END_RULE_TABLE

#endif

} // namespace jit::codegen::autogen
