/* +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Copyright (c) 2008-2017, Petr Kobalicek

This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.

Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:

1. The origin of this software must not be misrepresented; you must not
   claim that you wrote the original software. If you use this software
   in a product, an acknowledgment in the product documentation would be
   appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
   misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
#ifdef __PLUMED_HAS_ASMJIT
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
// [AsmJit]
// Complete x86/x64 JIT and Remote Assembler for C++.
//
// [License]
// Zlib - See LICENSE.md file in the package.

// [Export]
#define ASMJIT_EXPORTS

// [Guard]
#include "./asmjit_build.h"
#if defined(ASMJIT_BUILD_X86)

// [Dependencies]
#include "./cpuinfo.h"
#include "./logging.h"
#include "./misc_p.h"
#include "./utils.h"
#include "./x86assembler.h"
#include "./x86logging_p.h"

// [Api-Begin]
#include "./asmjit_apibegin.h"

namespace PLMD {
namespace asmjit {

// ============================================================================
// [FastUInt8]
// ============================================================================

#if ASMJIT_ARCH_X86 || ASMJIT_ARCH_X64
typedef unsigned char FastUInt8;
#else
typedef unsigned int FastUInt8;
#endif

// ============================================================================
// [Constants]
// ============================================================================

//! \internal
//!
//! X86/X64 bytes used to encode important prefixes.
enum X86Byte {
  //! 1-byte REX prefix mask.
  kX86ByteRex = 0x40,

  //! 1-byte REX.W component.
  kX86ByteRexW = 0x08,

  //! 2-byte VEX prefix:
  //!   - `[0]` - `0xC5`.
  //!   - `[1]` - `RvvvvLpp`.
  kX86ByteVex2 = 0xC5,

  //! 3-byte VEX prefix.
  //!   - `[0]` - `0xC4`.
  //!   - `[1]` - `RXBmmmmm`.
  //!   - `[2]` - `WvvvvLpp`.
  kX86ByteVex3 = 0xC4,

  //! 3-byte XOP prefix.
  //!   - `[0]` - `0x8F`.
  //!   - `[1]` - `RXBmmmmm`.
  //!   - `[2]` - `WvvvvLpp`.
  kX86ByteXop3 = 0x8F,

  //! 4-byte EVEX prefix.
  //!   - `[0]` - `0x62`.
  //!   - `[1]` - Payload0 or `P[ 7: 0]` - `[R  X  B  R' 0  0  m  m]`.
  //!   - `[2]` - Payload1 or `P[15: 8]` - `[W  v  v  v  v  1  p  p]`.
  //!   - `[3]` - Payload2 or `P[23:16]` - `[z  L' L  b  V' a  a  a]`.
  //!
  //! Groups:
  //!   - `P[ 1: 0]` - OPCODE: EVEX.mmmmm, only lowest 2 bits [1:0] used.
  //!   - `P[ 3: 2]` - ______: Must be 0.
  //!   - `P[    4]` - REG-ID: EVEX.R' - 5th bit of 'RRRRR'.
  //!   - `P[    5]` - REG-ID: EVEX.B  - 4th bit of 'BBBBB'.
  //!   - `P[    6]` - REG-ID: EVEX.X  - 5th bit of 'BBBBB' or 4th bit of 'XXXX' (with SIB).
  //!   - `P[    7]` - REG-ID: EVEX.R  - 4th bit of 'RRRRR'.
  //!   - `P[ 9: 8]` - OPCODE: EVEX.pp.
  //!   - `P[   10]` - ______: Must be 1.
  //!   - `P[14:11]` - REG-ID: 4 bits of 'VVVV'.
  //!   - `P[   15]` - OPCODE: EVEX.W.
  //!   - `P[18:16]` - REG-ID: K register k0...k7 (Merging/Zeroing Vector Ops).
  //!   - `P[   19]` - REG-ID: 5th bit of 'VVVVV'.
  //!   - `P[   20]` - OPCODE: Broadcast/Rounding Control/SAE bit.
  //!   - `P[22.21]` - OPCODE: Vector Length (L' and  L) / Rounding Control.
  //!   - `P[   23]` - OPCODE: Zeroing/Merging.
  kX86ByteEvex = 0x62
};

// AsmJit specific (used to encode VVVVV field in XOP/VEX/EVEX).
enum VexVVVVV {
  kVexVVVVVShift = 7,
  kVexVVVVVMask = 0x1F << kVexVVVVVShift
};

//! \internal
//!
//! Instruction 2-byte/3-byte opcode prefix definition.
struct X86OpCodeMM {
  uint8_t len;
  uint8_t data[3];
};

//! \internal
//!
//! Mandatory prefixes used to encode legacy [66, F3, F2] or [9B] byte.
static const uint8_t x86OpCodePP[8] = { 0x00, 0x66, 0xF3, 0xF2, 0x00, 0x00, 0x00, 0x9B };

//! \internal
//!
//! Instruction 2-byte/3-byte opcode prefix data.
static const X86OpCodeMM x86OpCodeMM[] = {
  { 0, { 0x00, 0x00, 0 } }, // #00 (0b0000).
  { 1, { 0x0F, 0x00, 0 } }, // #01 (0b0001).
  { 2, { 0x0F, 0x38, 0 } }, // #02 (0b0010).
  { 2, { 0x0F, 0x3A, 0 } }, // #03 (0b0011).
  { 2, { 0x0F, 0x01, 0 } }, // #04 (0b0100).
  { 0, { 0x00, 0x00, 0 } }, // #05 (0b0101).
  { 0, { 0x00, 0x00, 0 } }, // #06 (0b0110).
  { 0, { 0x00, 0x00, 0 } }, // #07 (0b0111).
  { 0, { 0x00, 0x00, 0 } }, // #08 (0b1000).
  { 0, { 0x00, 0x00, 0 } }, // #09 (0b1001).
  { 0, { 0x00, 0x00, 0 } }, // #0A (0b1010).
  { 0, { 0x00, 0x00, 0 } }, // #0B (0b1011).
  { 0, { 0x00, 0x00, 0 } }, // #0C (0b1100).
  { 0, { 0x00, 0x00, 0 } }, // #0D (0b1101).
  { 0, { 0x00, 0x00, 0 } }, // #0E (0b1110).
  { 0, { 0x00, 0x00, 0 } }  // #0F (0b1111).
};

static const uint8_t x86SegmentPrefix[8] = { 0x00, 0x26, 0x2E, 0x36, 0x3E, 0x64, 0x65, 0x00 };
static const uint8_t x86OpCodePushSeg[8] = { 0x00, 0x06, 0x0E, 0x16, 0x1E, 0xA0, 0xA8, 0x00 };
static const uint8_t x86OpCodePopSeg[8]  = { 0x00, 0x07, 0x00, 0x17, 0x1F, 0xA1, 0xA9, 0x00 };

// ============================================================================
// [asmjit::X86MemInfo | X86VEXPrefix | X86LLByRegType | X86CDisp8Table]
// ============================================================================

//! \internal
//!
//! Memory operand's info bits.
//!
//! A lookup table that contains various information based on the BASE and INDEX
//! information of a memory operand. This is much better and safer than playing
//! with IFs in the code and can check for errors must faster and better.
enum X86MemInfo_Enum {
  kX86MemInfo_0         = 0x00,

  kX86MemInfo_BaseGp    = 0x01, //!< Has BASE reg, REX.B can be 1, compatible with REX.B byte.
  kX86MemInfo_Index     = 0x02, //!< Has INDEX reg, REX.X can be 1, compatible with REX.X byte.

  kX86MemInfo_BaseLabel = 0x10, //!< Base is Label.
  kX86MemInfo_BaseRip   = 0x20, //!< Base is RIP.

  kX86MemInfo_67H_X86   = 0x40, //!< Address-size override in 32-bit mode.
  kX86MemInfo_67H_X64   = 0x80, //!< Address-size override in 64-bit mode.
  kX86MemInfo_67H_Mask  = 0xC0  //!< Contains all address-size override bits.
};

template<uint32_t X>
struct X86MemInfo_T {
  enum {
    B = (X     ) & 0x1F,
    I = (X >> 5) & 0x1F,

    kBase  = ((B >= X86Reg::kRegGpw  && B <= X86Reg::kRegGpq ) ? kX86MemInfo_BaseGp    :
              (B == X86Reg::kRegRip                          ) ? kX86MemInfo_BaseRip   :
              (B == Label::kLabelTag                         ) ? kX86MemInfo_BaseLabel : 0),

    kIndex = ((I >= X86Reg::kRegGpw  && I <= X86Reg::kRegGpq ) ? kX86MemInfo_Index     :
              (I >= X86Reg::kRegXmm  && I <= X86Reg::kRegZmm ) ? kX86MemInfo_Index     : 0),

    k67H   = ((B == X86Reg::kRegGpw  && I == X86Reg::kRegNone) ? kX86MemInfo_67H_X86   :
              (B == X86Reg::kRegGpd  && I == X86Reg::kRegNone) ? kX86MemInfo_67H_X64   :
              (B == X86Reg::kRegNone && I == X86Reg::kRegGpw ) ? kX86MemInfo_67H_X86   :
              (B == X86Reg::kRegNone && I == X86Reg::kRegGpd ) ? kX86MemInfo_67H_X64   :
              (B == X86Reg::kRegGpw  && I == X86Reg::kRegGpw ) ? kX86MemInfo_67H_X86   :
              (B == X86Reg::kRegGpd  && I == X86Reg::kRegGpd ) ? kX86MemInfo_67H_X64   :
              (B == X86Reg::kRegGpw  && I == X86Reg::kRegXmm ) ? kX86MemInfo_67H_X86   :
              (B == X86Reg::kRegGpd  && I == X86Reg::kRegXmm ) ? kX86MemInfo_67H_X64   :
              (B == X86Reg::kRegGpw  && I == X86Reg::kRegYmm ) ? kX86MemInfo_67H_X86   :
              (B == X86Reg::kRegGpd  && I == X86Reg::kRegYmm ) ? kX86MemInfo_67H_X64   :
              (B == X86Reg::kRegGpw  && I == X86Reg::kRegZmm ) ? kX86MemInfo_67H_X86   :
              (B == X86Reg::kRegGpd  && I == X86Reg::kRegZmm ) ? kX86MemInfo_67H_X64   :
              (B == Label::kLabelTag && I == X86Reg::kRegGpw ) ? kX86MemInfo_67H_X86   :
              (B == Label::kLabelTag && I == X86Reg::kRegGpd ) ? kX86MemInfo_67H_X64   : 0),

    kValue = kBase | kIndex | k67H | 0x04 | 0x08
  };
};

// The result stored in the LUT is a combination of
//   - 67H - Address override prefix - depends on BASE+INDEX register types and
//           the target architecture.
//   - REX - A possible combination of REX.[B|X|R|W] bits in REX prefix where
//           REX.B and REX.X are possibly masked out, but REX.R and REX.W are
//           kept as is.
static const uint8_t x86MemInfo[] = { ASMJIT_TABLE_T_1024(X86MemInfo_T, kValue, 0) };

// VEX3 or XOP xor bits applied to the opcode before emitted. The index to this
// table is 'mmmmm' value, which contains all we need. This is only used by a
// 3 BYTE VEX and XOP prefixes, 2 BYTE VEX prefix is handled differently. The
// idea is to minimize the difference between VEX3 vs XOP when encoding VEX
// or XOP instruction. This should minimize the code required to emit such
// instructions and should also make it faster as we don't need any branch to
// decide between VEX3 vs XOP.
//            ____    ___
// [_OPCODE_|WvvvvLpp|RXBmmmmm|VEX3_XOP]
template<uint32_t X>
struct X86VEXPrefix_T {
  enum { kValue = ((X & 0x08) ? kX86ByteXop3 : kX86ByteVex3) | (0xF << 19) | (0x7 << 13) };
};
static const uint32_t x86VEXPrefix[] = { ASMJIT_TABLE_T_16(X86VEXPrefix_T, kValue, 0) };

// Table that contains LL opcode field addressed by a register size / 16. It's
// used to propagate L.256 or L.512 when YMM or ZMM registers are used,
// respectively.
template<uint32_t X>
struct X86LLBySizeDiv16_T {
  enum {
    kValue = (X & (64 >> 4)) ? X86Inst::kOpCode_LL_512 :
             (X & (32 >> 4)) ? X86Inst::kOpCode_LL_256 : 0
  };
};
static const uint32_t x86LLBySizeDiv16[] = { ASMJIT_TABLE_T_16(X86LLBySizeDiv16_T, kValue, 0) };

// Table that contains LL opcode field addressed by a register size / 16. It's
// used to propagate L.256 or L.512 when YMM or ZMM registers are used,
// respectively.
template<uint32_t X>
struct X86LLByRegType_T {
  enum {
    kValue = X == X86Reg::kRegZmm ? X86Inst::kOpCode_LL_512 :
             X == X86Reg::kRegYmm ? X86Inst::kOpCode_LL_256 : 0
  };
};
static const uint32_t x86LLByRegType[] = { ASMJIT_TABLE_T_16(X86LLByRegType_T, kValue, 0) };

// Table that contains a scale (shift left) based on 'TTWLL' field and
// the instruction's tuple-type (TT) field. The scale is then applied to
// the BASE-N stored in each opcode to calculate the final compressed
// displacement used by all EVEX encoded instructions.
template<uint32_t X>
struct X86CDisp8SHL_T {
  enum {
    TT = (((X) >> 3) << X86Inst::kOpCode_CDTT_Shift),
    LL = (((X) >> 0) & 0x3),
    W  = (((X) >> 2) & 0x1),

    kValue = (TT == X86Inst::kOpCode_CDTT_None ? ((LL==0) ? 0 : (LL==1) ? 0   : 0  ) :
              TT == X86Inst::kOpCode_CDTT_ByLL ? ((LL==0) ? 0 : (LL==1) ? 1   : 2  ) :
              TT == X86Inst::kOpCode_CDTT_T1W  ? ((LL==0) ? W : (LL==1) ? 1+W : 2+W) :
              TT == X86Inst::kOpCode_CDTT_DUP  ? ((LL==0) ? 0 : (LL==1) ? 2   : 3  ) : 0 ) << X86Inst::kOpCode_CDSHL_Shift
  };
};
static const uint32_t x86CDisp8SHL[] = { ASMJIT_TABLE_T_32(X86CDisp8SHL_T, kValue, 0) };

// Table that contains MOD byte of a 16-bit [BASE + disp] address.
//   0xFF == Invalid.
static const uint8_t x86Mod16BaseTable[8] = {
  0xFF, // AX -> N/A.
  0xFF, // CX -> N/A.
  0xFF, // DX -> N/A.
  0x07, // BX -> 111.
  0xFF, // SP -> N/A.
  0x06, // BP -> 110.
  0x04, // SI -> 100.
  0x05  // DI -> 101.
};

// Table that contains MOD byte of a 16-bit [BASE + INDEX + disp] combination.
//   0xFF == Invalid.
template<uint32_t X>
struct X86Mod16BaseIndexTable_T {
  enum {
    B = X >> 3,
    I = X & 0x7,

    kValue = ((B == X86Gp::kIdBx && I == X86Gp::kIdSi) || (B == X86Gp::kIdSi && I == X86Gp::kIdBx)) ? 0x00 :
             ((B == X86Gp::kIdBx && I == X86Gp::kIdDi) || (B == X86Gp::kIdDi && I == X86Gp::kIdBx)) ? 0x01 :
             ((B == X86Gp::kIdBp && I == X86Gp::kIdSi) || (B == X86Gp::kIdSi && I == X86Gp::kIdBp)) ? 0x02 :
             ((B == X86Gp::kIdBp && I == X86Gp::kIdDi) || (B == X86Gp::kIdDi && I == X86Gp::kIdBp)) ? 0x03 : 0xFF
  };
};
static const uint8_t x86Mod16BaseIndexTable[] = { ASMJIT_TABLE_T_64(X86Mod16BaseIndexTable_T, kValue, 0) };

// ============================================================================
// [asmjit::X86Assembler - Helpers]
// ============================================================================

static ASMJIT_INLINE bool x86IsJmpOrCall(uint32_t instId) noexcept {
  return instId == X86Inst::kIdJmp ||
         instId == X86Inst::kIdCall;
}

static ASMJIT_INLINE bool x86IsImplicitMem(const Operand_& op, uint32_t base) noexcept {
  return op.isMem() && op.as<X86Mem>().getBaseId() == base;
}

static ASMJIT_INLINE int64_t x86SignExtend32To64(int64_t imm) noexcept {
  return static_cast<int64_t>(static_cast<int32_t>(imm & 0xFFFFFFFF));
}

//! Get `O` field of `opCode`.
static ASMJIT_INLINE uint32_t x86ExtractO(uint32_t opCode) noexcept {
  return (opCode >> X86Inst::kOpCode_O_Shift) & 0x07;
}

static ASMJIT_INLINE uint32_t x86ExtractREX(uint32_t opCode, uint32_t options) noexcept {
  // kOpCode_REX was designed in a way that when shifted there will be no bytes
  // set except REX.[B|X|R|W]. The returned value forms a real REX prefix byte.
  // This case is tested by `X86Inst.cpp`.
  return (opCode | options) >> X86Inst::kOpCode_REX_Shift;
}

//! Combine `regId` and `vvvvvId` into a single value (used by AVX and AVX-512).
static ASMJIT_INLINE uint32_t x86PackRegAndVvvvv(uint32_t regId, uint32_t vvvvvId) noexcept {
  return regId + (vvvvvId << kVexVVVVVShift);
}

static ASMJIT_INLINE uint32_t x86OpCodeLByVMem(const Operand_& op) noexcept {
  return x86LLByRegType[op.as<X86Mem>().getIndexType()];
}

static ASMJIT_INLINE uint32_t x86OpCodeLBySize(uint32_t size) noexcept {
  return x86LLBySizeDiv16[size / 16];
}

static ASMJIT_INLINE uint32_t x86ExtractLLMM(uint32_t opCode, uint32_t options) noexcept {
  uint32_t x = opCode & (X86Inst::kOpCode_LL_Mask | X86Inst::kOpCode_MM_Mask);
  uint32_t y = options & X86Inst::kOptionVex3;
  return (x | y) >> X86Inst::kOpCode_MM_Shift;
}

//! Encode MOD byte.
static ASMJIT_INLINE uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm) noexcept {
  ASMJIT_ASSERT(m <= 3);
  ASMJIT_ASSERT(o <= 7);
  ASMJIT_ASSERT(rm <= 7);
  return (m << 6) + (o << 3) + rm;
}

//! Encode SIB byte.
static ASMJIT_INLINE uint32_t x86EncodeSib(uint32_t s, uint32_t i, uint32_t b) noexcept {
  ASMJIT_ASSERT(s <= 3);
  ASMJIT_ASSERT(i <= 7);
  ASMJIT_ASSERT(b <= 7);
  return (s << 6) + (i << 3) + b;
}

// ============================================================================
// [asmjit::X86Assembler - Construction / Destruction]
// ============================================================================

X86Assembler::X86Assembler(CodeHolder* code) noexcept : Assembler() {
  if (code)
    code->attach(this);
}
X86Assembler::~X86Assembler() noexcept {}

// ============================================================================
// [asmjit::X86Assembler - Events]
// ============================================================================

Error X86Assembler::onAttach(CodeHolder* code) noexcept {
  uint32_t archType = code->getArchType();
  if (!ArchInfo::isX86Family(archType))
    return DebugUtils::errored(kErrorInvalidArch);

  ASMJIT_PROPAGATE(Base::onAttach(code));

  if (archType == ArchInfo::kTypeX86) {
    // 32 bit architecture - X86.
    _setAddressOverrideMask(kX86MemInfo_67H_X86);
    _globalOptions |= X86Inst::_kOptionInvalidRex;
    _nativeGpArray = x86OpData.gpd;
  }
  else {
    // 64 bit architecture - X64 or X32.
    _setAddressOverrideMask(kX86MemInfo_67H_X64);
    _nativeGpArray = x86OpData.gpq;
  }

  _nativeGpReg = _nativeGpArray[0];
  return kErrorOk;
}

Error X86Assembler::onDetach(CodeHolder* code) noexcept {
  return Base::onDetach(code);
}

// ============================================================================
// [asmjit::X86Assembler - Helpers]
// ============================================================================

#define EMIT_BYTE(VAL)                               \
  do {                                               \
    cursor[0] = static_cast<uint8_t>((VAL) & 0xFFU); \
    cursor += 1;                                     \
  } while (0)

#define EMIT_16(VAL)                                 \
  do {                                               \
    Utils::writeU16uLE(cursor,                       \
      static_cast<uint32_t>((VAL) & 0xFFFFU));       \
    cursor += 2;                                     \
  } while (0)

#define EMIT_32(VAL)                                 \
  do {                                               \
    Utils::writeU32uLE(cursor,                       \
      static_cast<uint32_t>((VAL) & 0xFFFFFFFFU));   \
    cursor += 4;                                     \
  } while (0)

#define ADD_66H_P(EXP)                                                   \
  do {                                                                   \
    opCode |= (static_cast<uint32_t>(EXP) << X86Inst::kOpCode_PP_Shift); \
  } while (0)

#define ADD_66H_P_BY_SIZE(SIZE)                                          \
  do {                                                                   \
    opCode |= (static_cast<uint32_t>((SIZE) & 0x02))                     \
           << (X86Inst::kOpCode_PP_Shift - 1);                           \
  } while (0)

#define ADD_REX_W(EXP)                                                   \
  do {                                                                   \
    if (EXP)                                                             \
      opCode |= X86Inst::kOpCode_W;                                      \
  } while (0)

#define ADD_REX_W_BY_SIZE(SIZE)                                          \
  do {                                                                   \
    if ((SIZE) == 8)                                                     \
      opCode |= X86Inst::kOpCode_W;                                      \
  } while (0)

#define ADD_PREFIX_BY_SIZE(SIZE)                                         \
  do {                                                                   \
    ADD_66H_P_BY_SIZE(SIZE);                                             \
    ADD_REX_W_BY_SIZE(SIZE);                                             \
  } while (0)

#define ADD_VEX_W(EXP)                                                   \
  do {                                                                   \
    opCode |= static_cast<uint32_t>(EXP) << X86Inst::kOpCode_W_Shift;    \
  } while (0)

#define EMIT_PP(OPCODE)                                                  \
  do {                                                                   \
    uint32_t ppIndex =                                                   \
      ((OPCODE                   ) >> X86Inst::kOpCode_PP_Shift) &       \
      (X86Inst::kOpCode_PP_FPUMask >> X86Inst::kOpCode_PP_Shift) ;       \
    uint8_t ppCode = x86OpCodePP[ppIndex];                               \
                                                                         \
    cursor[0] = ppCode;                                                  \
    cursor   += ppIndex != 0;                                            \
  } while (0)

#define EMIT_MM_OP(OPCODE)                                               \
  do {                                                                   \
    uint32_t op = OPCODE & (0x00FF | X86Inst::kOpCode_MM_Mask);          \
                                                                         \
    uint32_t mmIndex = op >> X86Inst::kOpCode_MM_Shift;                  \
    const X86OpCodeMM& mmCode = x86OpCodeMM[mmIndex];                    \
                                                                         \
    if (mmIndex) {                                                       \
      cursor[0] = mmCode.data[0];                                        \
      cursor[1] = mmCode.data[1];                                        \
      cursor   += mmCode.len;                                            \
    }                                                                    \
                                                                         \
    EMIT_BYTE(op);                                                       \
  } while (0)

// If the operand is BPL|SPL|SIL|DIL|R8B-15B
//   - Force REX prefix
// If the operand is AH|BH|CH|DH
//   - patch its index from 0..3 to 4..7 as encoded by X86.
//   - Disallow REX prefix.
#define FIXUP_GPB(REG_OP, REG_ID)                                   \
  do {                                                                   \
    if (!static_cast<const X86Gp&>(REG_OP).isGpbHi()) {                  \
      options |= (REG_ID >= 4) ? X86Inst::kOptionRex : 0;                \
    }                                                                    \
    else {                                                               \
      options |= X86Inst::_kOptionInvalidRex;                            \
      REG_ID += 4;                                                       \
    }                                                                    \
  } while (0)

#define ENC_OPS1(OP0)                     ((Operand::kOp##OP0))
#define ENC_OPS2(OP0, OP1)                ((Operand::kOp##OP0) + ((Operand::kOp##OP1) << 3))
#define ENC_OPS3(OP0, OP1, OP2)           ((Operand::kOp##OP0) + ((Operand::kOp##OP1) << 3) + ((Operand::kOp##OP2) << 6))
#define ENC_OPS4(OP0, OP1, OP2, OP3)      ((Operand::kOp##OP0) + ((Operand::kOp##OP1) << 3) + ((Operand::kOp##OP2) << 6) + ((Operand::kOp##OP3) << 9))

// ============================================================================
// [asmjit::X86Assembler - Emit]
// ============================================================================

Error X86Assembler::_emit(uint32_t instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) {
  Error err;

  const Operand_* rmRel;         // Memory operand or operand that holds Label|Imm.
  uint32_t rmInfo;               // Memory operand's info based on x86MemInfo.
  uint32_t rbReg;                // Memory base or modRM register.
  uint32_t rxReg;                // Memory index register.
  uint32_t opReg;                // ModR/M opcode or register id.
  uint32_t opCode;               // Instruction opcode.

  LabelEntry* label;             // Label entry.
  RelocEntry* re = nullptr;      // Relocation entry.
  int32_t relOffset;             // Relative offset
  FastUInt8 relSize = 0;         // Relative size.

  int64_t imVal = 0;             // Immediate value (must be 64-bit).
  FastUInt8 imLen = 0;           // Immediate length.

  const uint32_t kSHR_W_PP = X86Inst::kOpCode_PP_Shift - 16;
  const uint32_t kSHR_W_EW = X86Inst::kOpCode_EW_Shift - 23;

  uint8_t* cursor = _bufferPtr;
  uint32_t options = static_cast<uint32_t>(instId >= X86Inst::_kIdCount)       |
                     static_cast<uint32_t>((size_t)(_bufferEnd - cursor) < 16) |
                     getGlobalOptions() | getOptions();

  const X86Inst* instData = X86InstDB::instData + instId;
  const X86Inst::CommonData* commonData;

  // Handle failure and rare cases first.
  const uint32_t kErrorsAndSpecialCases =
    CodeEmitter::kOptionMaybeFailureCase | // Error and buffer check.
    CodeEmitter::kOptionStrictValidation | // Strict validation.
    X86Inst::kOptionRep                  | // REP/REPZ prefix.
    X86Inst::kOptionRepnz                | // REPNZ prefix.
    X86Inst::kOptionLock                 | // LOCK prefix.
    X86Inst::kOptionXAcquire             | // XACQUIRE prefix.
    X86Inst::kOptionXRelease             ; // XRELEASE prefix.

  // Signature of the first 3 operands.
  uint32_t isign3 = o0.getOp() + (o1.getOp() << 3) + (o2.getOp() << 6);

  if (ASMJIT_UNLIKELY(options & kErrorsAndSpecialCases)) {
    // Don't do anything if we are in error state.
    if (_lastError) return _lastError;

    if (options & CodeEmitter::kOptionMaybeFailureCase) {
      // Unknown instruction.
      if (ASMJIT_UNLIKELY(instId >= X86Inst::_kIdCount))
        goto InvalidArgument;

      // Grow request, happens rarely.
      if ((size_t)(_bufferEnd - cursor) < 16) {
        err = _code->growBuffer(&_section->_buffer, 16);
        if (ASMJIT_UNLIKELY(err)) goto Failed;

        cursor = _bufferPtr;
        options &= ~1;
      }
    }

    // Strict validation.
#if !defined(ASMJIT_DISABLE_VALIDATION)
    if (options & CodeEmitter::kOptionStrictValidation) {
      Operand_ opArray[6];

      opArray[0].copyFrom(o0);
      opArray[1].copyFrom(o1);
      opArray[2].copyFrom(o2);
      opArray[3].copyFrom(o3);

      if (options & kOptionOp4Op5Used) {
        opArray[4].copyFrom(_op4);
        opArray[5].copyFrom(_op5);
      }
      else {
        opArray[4].reset();
        opArray[5].reset();
      }

      err = Inst::validate(getArchType(), Inst::Detail(instId, options, _extraReg), opArray, 6);
      if (ASMJIT_UNLIKELY(err)) goto Failed;
    }
#endif // !ASMJIT_DISABLE_VALIDATION

    uint32_t iFlags = instData->getFlags();

    // LOCK, XACQUIRE, and XRELEASE prefixes.
    if (options & X86Inst::kOptionLock) {
      bool xAcqRel = (options & (X86Inst::kOptionXAcquire | X86Inst::kOptionXRelease)) != 0;

      if (ASMJIT_UNLIKELY(!(iFlags & (X86Inst::kFlagLock)) && !xAcqRel))
        goto InvalidLockPrefix;

      if (xAcqRel) {
        if (ASMJIT_UNLIKELY((options & X86Inst::kOptionXAcquire) && !(iFlags & X86Inst::kFlagXAcquire)))
          goto InvalidXAcquirePrefix;

        if (ASMJIT_UNLIKELY((options & X86Inst::kOptionXRelease) && !(iFlags & X86Inst::kFlagXRelease)))
          goto InvalidXReleasePrefix;

        EMIT_BYTE((options & X86Inst::kOptionXAcquire) ? 0xF2 : 0xF3);
      }

      EMIT_BYTE(0xF0);
    }

    // REP and REPNZ prefixes.
    if (options & (X86Inst::kOptionRep | X86Inst::kOptionRepnz)) {
      if (ASMJIT_UNLIKELY(!(iFlags & (X86Inst::kFlagRep | X86Inst::kFlagRepnz))))
        goto InvalidRepPrefix;

      if (_extraReg.isValid() && ASMJIT_UNLIKELY(_extraReg.getKind() != X86Reg::kKindGp || _extraReg.getId() != X86Gp::kIdCx))
        goto InvalidRepPrefix;

      EMIT_BYTE((options & X86Inst::kOptionRepnz) ? 0xF2 : 0xF3);
    }
  }

  // --------------------------------------------------------------------------
  // [Encoding Scope]
  // --------------------------------------------------------------------------

  opCode = instData->getMainOpCode();
  opReg = x86ExtractO(opCode);
  commonData = &instData->getCommonData();

  switch (instData->getEncodingType()) {
    case X86Inst::kEncodingNone:
      goto EmitDone;

    // ------------------------------------------------------------------------
    // [X86]
    // ------------------------------------------------------------------------

    case X86Inst::kEncodingX86Op:
      goto EmitX86Op;

    case X86Inst::kEncodingX86Op_O_I8:
      if (ASMJIT_UNLIKELY(isign3 != ENC_OPS1(Imm)))
        goto InvalidInstruction;

      imVal = o0.as<Imm>().getUInt8();
      imLen = 1;
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingX86Op_O:
      rbReg = 0;
      goto EmitX86R;

    case X86Inst::kEncodingX86Op_xAX:
      if (isign3 == 0)
        goto EmitX86Op;

      if (isign3 == ENC_OPS1(Reg) && o0.getId() == X86Gp::kIdAx)
        goto EmitX86Op;
      break;

    case X86Inst::kEncodingX86Op_xDX_xAX:
      if (isign3 == 0)
        goto EmitX86Op;

      if (isign3 == ENC_OPS2(Reg, Reg) && o0.getId() == X86Gp::kIdDx &&
                                          o1.getId() == X86Gp::kIdAx)
        goto EmitX86Op;
      break;

    case X86Inst::kEncodingX86Op_ZAX:
      if (isign3 == 0)
        goto EmitX86Op;

      rmRel = &o0;
      if (isign3 == ENC_OPS1(Mem) && x86IsImplicitMem(o0, X86Gp::kIdAx))
        goto EmitX86OpImplicitMem;

      break;

    case X86Inst::kEncodingX86I_xAX:
      // Implicit form.
      if (isign3 == ENC_OPS1(Imm)) {
        imVal = o0.as<Imm>().getUInt8();
        imLen = 1;
        goto EmitX86Op;
      }

      // Explicit form.
      if (isign3 == ENC_OPS2(Reg, Imm) && o0.getId() == X86Gp::kIdAx) {
        imVal = o1.as<Imm>().getUInt8();
        imLen = 1;
        goto EmitX86Op;
      }
      break;

    case X86Inst::kEncodingX86M:
      rbReg = o0.getId();
      ADD_PREFIX_BY_SIZE(o0.getSize());

      if (isign3 == ENC_OPS1(Reg))
        goto EmitX86R;

      rmRel = &o0;
      if (isign3 == ENC_OPS1(Mem))
        goto EmitX86M;
      break;

    case X86Inst::kEncodingX86M_GPB_MulDiv:
CaseX86M_GPB_MulDiv:
      // Explicit form?
      if (isign3 > 0x7) {
        // [AX] <- [AX] div|mul r8.
        if (isign3 == ENC_OPS2(Reg, Reg)) {
          if (ASMJIT_UNLIKELY(!X86Reg::isGpw(o0, X86Gp::kIdAx) || !X86Reg::isGpb(o1)))
            goto InvalidInstruction;

          rbReg = o1.getId();
          FIXUP_GPB(o1, rbReg);
          goto EmitX86R;
        }

        // [AX] <- [AX] div|mul m8.
        if (isign3 == ENC_OPS2(Reg, Mem)) {
          if (ASMJIT_UNLIKELY(!X86Reg::isGpw(o0, X86Gp::kIdAx)))
            goto InvalidInstruction;

          rmRel = &o1;
          goto EmitX86M;
        }

        // [?DX:?AX] <- [?DX:?AX] div|mul r16|r32|r64
        if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
          if (ASMJIT_UNLIKELY(o0.getSize() != o1.getSize()))
            goto InvalidInstruction;
          rbReg = o2.getId();

          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());
          goto EmitX86R;
        }

        // [?DX:?AX] <- [?DX:?AX] div|mul m16|m32|m64
        if (isign3 == ENC_OPS3(Reg, Reg, Mem)) {
          if (ASMJIT_UNLIKELY(o0.getSize() != o1.getSize()))
            goto InvalidInstruction;
          rmRel = &o2;

          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());
          goto EmitX86M;
        }

        goto InvalidInstruction;
      }

      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingX86M_GPB:
      if (isign3 == ENC_OPS1(Reg)) {
        rbReg = o0.getId();
        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, rbReg);
          goto EmitX86R;
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());
          goto EmitX86R;
        }
      }

      if (isign3 == ENC_OPS1(Mem)) {
        if (ASMJIT_UNLIKELY(o0.getSize() == 0))
          goto AmbiguousOperandSize;
        rmRel = &o0;

        opCode += o0.getSize() != 1;
        ADD_PREFIX_BY_SIZE(o0.getSize());
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86M_Only:
      if (isign3 == ENC_OPS1(Mem)) {
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Rm:
      ADD_PREFIX_BY_SIZE(o0.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingX86Rm_NoRexW:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Rm_Raw66H:
      // We normally emit either [66|F2|F3], this instruction requires 66+[F2|F3].
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();

        if (o0.getSize() == 2)
          EMIT_BYTE(0x66);
        else
          ADD_REX_W_BY_SIZE(o0.getSize());
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;

        if (o0.getSize() == 2)
          EMIT_BYTE(0x66);
        else
          ADD_REX_W_BY_SIZE(o0.getSize());
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Mr:
      ADD_PREFIX_BY_SIZE(o0.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingX86Mr_NoSize:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        rbReg = o0.getId();
        opReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        rmRel = &o0;
        opReg = o1.getId();
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Arith:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        if (o0.getSize() != o1.getSize())
          goto OperandSizeMismatch;

        opReg = o0.getId();
        rbReg = o1.getId();

        if (o0.getSize() == 1) {
          opCode += 2;
          FIXUP_GPB(o0, opReg);
          FIXUP_GPB(o1, rbReg);

          if (!(options & X86Inst::kOptionModMR))
            goto EmitX86R;

          opCode -= 2;
          Utils::swap(opReg, rbReg);
          goto EmitX86R;
        }
        else {
          opCode += 3;
          ADD_PREFIX_BY_SIZE(o0.getSize());

          if (!(options & X86Inst::kOptionModMR))
            goto EmitX86R;

          opCode -= 2;
          Utils::swap(opReg, rbReg);
          goto EmitX86R;
        }
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;

        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, opReg);
          opCode += 2;
          goto EmitX86M;
        }
        else {
          opCode += 3;
          ADD_PREFIX_BY_SIZE(o0.getSize());
          goto EmitX86M;
        }
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;

        if (o1.getSize() == 1) {
          FIXUP_GPB(o1, opReg);
          goto EmitX86M;
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o1.getSize());
          goto EmitX86M;
        }
      }

      // The remaining instructions use 0x80 opcode.
      opCode = 0x80;

      if (isign3 == ENC_OPS2(Reg, Imm)) {
        uint32_t size = o0.getSize();

        rbReg = o0.getId();
        imVal = static_cast<const Imm&>(o1).getInt64();

        if (size == 1) {
          FIXUP_GPB(o0, rbReg);
          imLen = 1;
        }
        else {
          if (size == 2) {
            ADD_66H_P(1);
          }
          else if (size == 4) {
            // Sign extend so isInt8 returns the right result.
            imVal = x86SignExtend32To64(imVal);
          }
          else if (size == 8) {
            // In 64-bit mode it's not possible to use 64-bit immediate.
            if (Utils::isUInt32(imVal)) {
              // Zero-extend `and` by using a 32-bit GPD destination instead of a 64-bit GPQ.
              if (instId == X86Inst::kIdAnd)
                size = 4;
              else if (!Utils::isInt32(imVal))
                goto InvalidImmediate;
            }
            ADD_REX_W_BY_SIZE(size);
          }

          imLen = std::min<uint32_t>(size, 4);
          if (Utils::isInt8(imVal) && !(options & X86Inst::kOptionLongForm))
            imLen = 1;
        }

        // Alternate Form - AL, AX, EAX, RAX.
        if (rbReg == 0 && (size == 1 || imLen != 1) && !(options & X86Inst::kOptionLongForm)) {
          opCode &= X86Inst::kOpCode_PP_66 | X86Inst::kOpCode_W;
          opCode |= ((opReg << 3) | (0x04 + (size != 1)));
          imLen = std::min<uint32_t>(size, 4);
          goto EmitX86Op;
        }

        opCode += size != 1 ? (imLen != 1 ? 1 : 3) : 0;
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Mem, Imm)) {
        uint32_t memSize = o0.getSize();

        if (ASMJIT_UNLIKELY(memSize == 0))
          goto AmbiguousOperandSize;

        imVal = static_cast<const Imm&>(o1).getInt64();
        imLen = std::min<uint32_t>(memSize, 4);

        // Sign extend so isInt8 returns the right result.
        if (memSize == 4)
          imVal = x86SignExtend32To64(imVal);

        if (Utils::isInt8(imVal) && !(options & X86Inst::kOptionLongForm))
          imLen = 1;

        opCode += memSize != 1 ? (imLen != 1 ? 1 : 3) : 0;
        ADD_PREFIX_BY_SIZE(memSize);

        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Bswap:
      if (isign3 == ENC_OPS1(Reg)) {
        if (ASMJIT_UNLIKELY(o0.getSize() < 4))
          goto InvalidInstruction;

        opReg = o0.getId();
        ADD_REX_W_BY_SIZE(o0.getSize());
        goto EmitX86OpReg;
      }
      break;

    case X86Inst::kEncodingX86Bt:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        ADD_PREFIX_BY_SIZE(o1.getSize());
        opReg = o1.getId();
        rbReg = o0.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        ADD_PREFIX_BY_SIZE(o1.getSize());
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }

      // The remaining instructions use the secondary opcode/r.
      imVal = static_cast<const Imm&>(o1).getInt64();
      imLen = 1;

      opCode = commonData->getAltOpCode();
      opReg = x86ExtractO(opCode);
      ADD_PREFIX_BY_SIZE(o0.getSize());

      if (isign3 == ENC_OPS2(Reg, Imm)) {
        rbReg = o0.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Mem, Imm)) {
        if (ASMJIT_UNLIKELY(o0.getSize() == 0))
          goto AmbiguousOperandSize;

        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Call:
      if (isign3 == ENC_OPS1(Reg)) {
        rbReg = o0.getId();
        goto EmitX86R;
      }

      rmRel = &o0;
      if (isign3 == ENC_OPS1(Mem))
        goto EmitX86M;

      // Call with 32-bit displacement use 0xE8 opcode. Call with 8-bit
      // displacement is not encodable so the alternative opcode field
      // in X86DB must be zero.
      opCode = 0xE8;
      opReg = 0;
      goto EmitJmpCall;

    case X86Inst::kEncodingX86Cmpxchg: {
      // Convert explicit to implicit.
      if (isign3 & (0x7 << 6)) {
        if (!X86Reg::isGp(o2) || o2.getId() != X86Gp::kIdAx)
          goto InvalidInstruction;
        isign3 &= 0x3F;
      }

      if (isign3 == ENC_OPS2(Reg, Reg)) {
        if (o0.getSize() != o1.getSize())
          goto OperandSizeMismatch;

        rbReg = o0.getId();
        opReg = o1.getId();

        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, rbReg);
          FIXUP_GPB(o1, opReg);
          goto EmitX86R;
        }
        else {
          ADD_PREFIX_BY_SIZE(o0.getSize());
          opCode++;
          goto EmitX86R;
        }
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;

        if (o1.getSize() == 1) {
          FIXUP_GPB(o0, opReg);
          goto EmitX86M;
        }
        else {
          ADD_PREFIX_BY_SIZE(o1.getSize());
          opCode++;
          goto EmitX86M;
        }
      }
      break;
    }

    case X86Inst::kEncodingX86Crc:
      opReg = o0.getId();
      ADD_REX_W_BY_SIZE(o0.getSize());

      if (isign3 == ENC_OPS2(Reg, Reg)) {
        rbReg = o1.getId();

        if (o1.getSize() == 1) {
          FIXUP_GPB(o1, rbReg);
          goto EmitX86R;
        }
        else {
          // This seems to be the only exception of encoding 66F2 PP prefix.
          if (o1.getSize() == 2) EMIT_BYTE(0x66);

          opCode++;
          goto EmitX86R;
        }
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        rmRel = &o1;
        if (o1.getSize() == 0)
          goto AmbiguousOperandSize;

        // This seems to be the only exception of encoding 66F2 PP prefix.
        if (o1.getSize() == 2) EMIT_BYTE(0x66);

        opCode += o1.getSize() != 1;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Enter:
      if (isign3 == ENC_OPS2(Imm, Imm)) {
        uint32_t iw = static_cast<const Imm&>(o0).getUInt16();
        uint32_t ib = static_cast<const Imm&>(o1).getUInt8();

        imVal = iw | (ib << 16);
        imLen = 3;
        goto EmitX86Op;
      }
      break;

    case X86Inst::kEncodingX86Imul:
      // First process all forms distinct of `kEncodingX86M_OptB_MulDiv`.
      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        opCode = 0x6B;
        ADD_PREFIX_BY_SIZE(o0.getSize());

        imVal = static_cast<const Imm&>(o2).getInt64();
        imLen = 1;

        if (!Utils::isInt8(imVal) || (options & X86Inst::kOptionLongForm)) {
          opCode -= 2;
          imLen = o0.getSize() == 2 ? 2 : 4;
        }

        opReg = o0.getId();
        rbReg = o1.getId();

        goto EmitX86R;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Imm)) {
        opCode = 0x6B;
        ADD_PREFIX_BY_SIZE(o0.getSize());

        imVal = static_cast<const Imm&>(o2).getInt64();
        imLen = 1;

        // Sign extend so isInt8 returns the right result.
        if (o0.getSize() == 4)
          imVal = x86SignExtend32To64(imVal);

        if (!Utils::isInt8(imVal) || (options & X86Inst::kOptionLongForm)) {
          opCode -= 2;
          imLen = o0.getSize() == 2 ? 2 : 4;
        }

        opReg = o0.getId();
        rmRel = &o1;

        goto EmitX86M;
      }

      if (isign3 == ENC_OPS2(Reg, Reg)) {
        // Must be explicit 'ax, r8' form.
        if (o1.getSize() == 1)
          goto CaseX86M_GPB_MulDiv;

        if (o0.getSize() != o1.getSize())
          goto OperandSizeMismatch;

        opReg = o0.getId();
        rbReg = o1.getId();

        opCode = X86Inst::kOpCode_MM_0F | 0xAF;
        ADD_PREFIX_BY_SIZE(o0.getSize());
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        // Must be explicit 'ax, m8' form.
        if (o1.getSize() == 1)
          goto CaseX86M_GPB_MulDiv;

        opReg = o0.getId();
        rmRel = &o1;

        opCode = X86Inst::kOpCode_MM_0F | 0xAF;
        ADD_PREFIX_BY_SIZE(o0.getSize());
        goto EmitX86M;
      }

      // Shorthand to imul 'reg, reg, imm'.
      if (isign3 == ENC_OPS2(Reg, Imm)) {
        opCode = 0x6B;
        ADD_PREFIX_BY_SIZE(o0.getSize());

        imVal = static_cast<const Imm&>(o1).getInt64();
        imLen = 1;

        // Sign extend so isInt8 returns the right result.
        if (o0.getSize() == 4)
          imVal = x86SignExtend32To64(imVal);

        if (!Utils::isInt8(imVal) || (options & X86Inst::kOptionLongForm)) {
          opCode -= 2;
          imLen = o0.getSize() == 2 ? 2 : 4;
        }

        opReg = rbReg = o0.getId();
        goto EmitX86R;
      }

      // Try implicit form.
      goto CaseX86M_GPB_MulDiv;

    case X86Inst::kEncodingX86In:
      if (isign3 == ENC_OPS2(Reg, Imm)) {
        if (ASMJIT_UNLIKELY(o0.getId() != X86Gp::kIdAx))
          goto InvalidInstruction;

        imVal = o1.as<Imm>().getUInt8();
        imLen = 1;

        opCode = commonData->getAltOpCode() + (o0.getSize() != 1);
        ADD_66H_P_BY_SIZE(o0.getSize());
        goto EmitX86Op;
      }

      if (isign3 == ENC_OPS2(Reg, Reg)) {
        if (ASMJIT_UNLIKELY(o0.getId() != X86Gp::kIdAx || o1.getId() != X86Gp::kIdDx))
          goto InvalidInstruction;

        opCode += o0.getSize() != 1;
        ADD_66H_P_BY_SIZE(o0.getSize());
        goto EmitX86Op;
      }
      break;

    case X86Inst::kEncodingX86Ins:
      if (isign3 == ENC_OPS2(Mem, Reg)) {
        if (ASMJIT_UNLIKELY(!x86IsImplicitMem(o0, X86Gp::kIdDi) || o1.getId() != X86Gp::kIdDx))
          goto InvalidInstruction;

        uint32_t size = o0.getSize();
        if (ASMJIT_UNLIKELY(size == 0))
          goto AmbiguousOperandSize;

        rmRel = &o0;
        opCode += (size != 1);

        ADD_66H_P_BY_SIZE(size);
        goto EmitX86OpImplicitMem;
      }
      break;

    case X86Inst::kEncodingX86IncDec:
      if (isign3 == ENC_OPS1(Reg)) {
        rbReg = o0.getId();

        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, rbReg);
          goto EmitX86R;
        }

        if (is32Bit()) {
          // INC r16|r32 is only encodable in 32-bit mode (collides with REX).
          opCode = commonData->getAltOpCode() + (rbReg & 0x07);
          ADD_66H_P_BY_SIZE(o0.getSize());
          goto EmitX86Op;
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());
          goto EmitX86R;
        }
      }

      if (isign3 == ENC_OPS1(Mem)) {
        rmRel = &o0;
        opCode += o0.getSize() != 1;

        ADD_PREFIX_BY_SIZE(o0.getSize());
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Int:
      if (isign3 == ENC_OPS1(Imm)) {
        imVal = static_cast<const Imm&>(o0).getInt64();
        imLen = 1;
        goto EmitX86Op;
      }
      break;

    case X86Inst::kEncodingX86Jcc:
      if (_globalHints & CodeEmitter::kHintPredictedJumps) {
        if (options & X86Inst::kOptionTaken)
          EMIT_BYTE(0x3E);
        if (options & X86Inst::kOptionNotTaken)
          EMIT_BYTE(0x2E);
      }

      rmRel = &o0;
      opReg = 0;
      goto EmitJmpCall;

    case X86Inst::kEncodingX86JecxzLoop:
      rmRel = &o0;
      // Explicit jecxz|loop [r|e]cx, dst
      if (o0.isReg()) {
        if (ASMJIT_UNLIKELY(!X86Reg::isGp(o0, X86Gp::kIdCx)))
          goto InvalidInstruction;

        if ((is32Bit() && o0.getSize() == 2) || (is64Bit() && o0.getSize() == 4))
          EMIT_BYTE(0x67);

        rmRel = &o1;
      }

      opReg = 0;
      goto EmitJmpCall;

    case X86Inst::kEncodingX86Jmp:
      if (isign3 == ENC_OPS1(Reg)) {
        rbReg = o0.getId();
        goto EmitX86R;
      }

      rmRel = &o0;
      if (isign3 == ENC_OPS1(Mem))
        goto EmitX86M;

      // Jump encoded with 32-bit displacement use 0xE9 opcode. Jump encoded
      // with 8-bit displacement's opcode is stored as an alternative opcode.
      opCode = 0xE9;
      opReg = 0;
      goto EmitJmpCall;

    case X86Inst::kEncodingX86JmpRel:
      rmRel = &o0;
      goto EmitJmpCall;

    case X86Inst::kEncodingX86Lea:
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        ADD_PREFIX_BY_SIZE(o0.getSize());
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Mov:
      // Reg <- Reg
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();

        // Asmjit uses segment registers indexed from 1 to 6, leaving zero as
        // "no segment register used". We have to fix this (decrement the index
        // of the register) when emitting MOV instructions which move to/from
        // a segment register. The segment register is always `opReg`, because
        // the MOV instruction uses either RM or MR encoding.

        // GP <- ??
        if (X86Reg::isGp(o0)) {
          // GP <- GP
          if (X86Reg::isGp(o1)) {
            uint32_t size0 = o0.getSize();
            uint32_t size1 = o1.getSize();

            if (size0 != size1) {
              // We allow 'mov r64, r32' as it's basically zero-extend.
              if (size0 == 8 && size1 == 4)
                size0 = 4; // Zero extend, don't promote to 64-bit.
              else
                goto InvalidInstruction;
            }

            if (size0 == 1) {
              FIXUP_GPB(o0, opReg);
              FIXUP_GPB(o1, rbReg);
              opCode = 0x8A;

              if (!(options & X86Inst::kOptionModMR))
                goto EmitX86R;

              opCode -= 2;
              Utils::swap(opReg, rbReg);
              goto EmitX86R;
            }
            else {
              opCode = 0x8B;
              ADD_PREFIX_BY_SIZE(size0);

              if (!(options & X86Inst::kOptionModMR))
                goto EmitX86R;

              opCode -= 2;
              Utils::swap(opReg, rbReg);
              goto EmitX86R;
            }
          }

          opReg = rbReg;
          rbReg = o0.getId();

          // GP <- SEG
          if (X86Reg::isSeg(o1)) {
            opCode = 0x8C;
            opReg--;
            ADD_PREFIX_BY_SIZE(o0.getSize());
            goto EmitX86R;
          }

          // GP <- CR
          if (X86Reg::isCr(o1)) {
            opCode = 0x20 | X86Inst::kOpCode_MM_0F;

            // Use `LOCK MOV` in 32-bit mode if CR8+ register is accessed (AMD extension).
            if ((opReg & 0x8) && is32Bit()) {
              EMIT_BYTE(0xF0);
              opReg &= 0x7;
            }
            goto EmitX86R;
          }

          // GP <- DR
          if (X86Reg::isDr(o1)) {
            opCode = 0x21 | X86Inst::kOpCode_MM_0F;
            goto EmitX86R;
          }
        }
        else {
          // ?? <- GP
          if (!X86Reg::isGp(o1))
            goto InvalidInstruction;

          // SEG <- GP
          if (X86Reg::isSeg(o0)) {
            opCode = 0x8E;
            opReg--;
            ADD_PREFIX_BY_SIZE(o1.getSize());
            goto EmitX86R;
          }

          // CR <- GP
          if (X86Reg::isCr(o0)) {
            opCode = 0x22 | X86Inst::kOpCode_MM_0F;

            // Use `LOCK MOV` in 32-bit mode if CR8+ register is accessed (AMD extension).
            if ((opReg & 0x8) && is32Bit()) {
              EMIT_BYTE(0xF0);
              opReg &= 0x7;
            }
            goto EmitX86R;
          }

          // DR <- GP
          if (X86Reg::isDr(o0)) {
            opCode = 0x23 | X86Inst::kOpCode_MM_0F;
            goto EmitX86R;
          }
        }

        goto InvalidInstruction;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;

        // SEG <- Mem
        if (X86Reg::isSeg(o0)) {
          opCode = 0x8E;
          opReg--;
          ADD_PREFIX_BY_SIZE(o1.getSize());
          goto EmitX86M;
        }
        // Reg <- Mem
        else {
          if (o0.getSize() == 1) {
            opCode = 0;
            FIXUP_GPB(o0, opReg);
          }
          else {
            opCode = 1;
            ADD_PREFIX_BY_SIZE(o0.getSize());
          }

          // Handle a special form `mov al|ax|eax|rax, [ptr64]` that doesn't use MOD.
          if (o0.getId() == X86Gp::kIdAx && !rmRel->as<X86Mem>().hasBaseOrIndex()) {
            imVal = rmRel->as<X86Mem>().getOffset();
            if (!is64Bit() || (is64Bit() && ((options & X86Inst::kOptionLongForm) || !Utils::isInt32(imVal)))) {
              opCode += 0xA0;
              goto EmitX86OpMovAbs;
            }
          }

          opCode += 0x8A;
          goto EmitX86M;
        }
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;

        // Mem <- SEG
        if (X86Reg::isSeg(o1)) {
          opCode = 0x8C;
          opReg--;
          ADD_PREFIX_BY_SIZE(o0.getSize());
          goto EmitX86M;
        }
        // Mem <- Reg
        else {
          if (o1.getSize() == 1) {
            opCode = 0;
            FIXUP_GPB(o1, opReg);
          }
          else {
            opCode = 1;
            ADD_PREFIX_BY_SIZE(o1.getSize());
          }

          // Handle a special form `mov [ptr64], al|ax|eax|rax` that doesn't use MOD.
          if (o1.getId() == X86Gp::kIdAx && !rmRel->as<X86Mem>().hasBaseOrIndex()) {
            imVal = rmRel->as<X86Mem>().getOffset();
            if (!is64Bit() || (is64Bit() && ((options & X86Inst::kOptionLongForm) || !Utils::isInt32(imVal)))) {
              opCode += 0xA2;
              goto EmitX86OpMovAbs;
            }
          }

          opCode += 0x88;
          goto EmitX86M;
        }
      }

      if (isign3 == ENC_OPS2(Reg, Imm)) {
        opReg = o0.getId();
        imLen = o0.getSize();

        if (imLen == 1) {
          FIXUP_GPB(o0, opReg);

          imVal = static_cast<const Imm&>(o1).getUInt8();
          opCode = 0xB0;
          goto EmitX86OpReg;
        }
        else {
          // 64-bit immediate in 64-bit mode is allowed.
          imVal = static_cast<const Imm&>(o1).getInt64();

          // Optimize the instruction size by using a 32-bit immediate if possible.
          if (imLen == 8 && !(options & X86Inst::kOptionLongForm)) {
            if (Utils::isUInt32(imVal)) {
              // Zero-extend by using a 32-bit GPD destination instead of a 64-bit GPQ.
              imLen = 4;
            }
            else if (Utils::isInt32(imVal)) {
              // Sign-extend, uses 'C7 /0' opcode.
              rbReg = opReg;

              opCode = 0xC7 | X86Inst::kOpCode_W;
              opReg = 0;

              imLen = 4;
              goto EmitX86R;
            }
          }

          opCode = 0xB8;
          ADD_PREFIX_BY_SIZE(imLen);
          goto EmitX86OpReg;
        }
      }

      if (isign3 == ENC_OPS2(Mem, Imm)) {
        uint32_t memSize = o0.getSize();

        if (ASMJIT_UNLIKELY(memSize == 0))
          goto AmbiguousOperandSize;

        imVal = static_cast<const Imm&>(o1).getInt64();
        imLen = std::min<uint32_t>(memSize, 4);

        opCode = 0xC6 + (memSize != 1);
        opReg = 0;
        ADD_PREFIX_BY_SIZE(memSize);

        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86MovsxMovzx:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        ADD_PREFIX_BY_SIZE(o0.getSize());

        if (o1.getSize() == 1) {
          FIXUP_GPB(o1, rbReg);
          goto EmitX86R;
        }
        else {
          opCode++;
          goto EmitX86R;
        }
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opCode += o1.getSize() != 1;
        ADD_PREFIX_BY_SIZE(o0.getSize());

        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Out:
      if (isign3 == ENC_OPS2(Imm, Reg)) {
        if (ASMJIT_UNLIKELY(o1.getId() != X86Gp::kIdAx))
          goto InvalidInstruction;

        imVal = o0.as<Imm>().getUInt8();
        imLen = 1;

        opCode = commonData->getAltOpCode() + (o1.getSize() != 1);
        ADD_66H_P_BY_SIZE(o1.getSize());
        goto EmitX86Op;
      }

      if (isign3 == ENC_OPS2(Reg, Reg)) {
        if (ASMJIT_UNLIKELY(o0.getId() != X86Gp::kIdDx || o1.getId() != X86Gp::kIdAx))
          goto InvalidInstruction;

        opCode += o1.getSize() != 1;
        ADD_66H_P_BY_SIZE(o1.getSize());
        goto EmitX86Op;
      }
      break;

    case X86Inst::kEncodingX86Outs:
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        if (ASMJIT_UNLIKELY(o0.getId() != X86Gp::kIdDx || !x86IsImplicitMem(o1, X86Gp::kIdSi)))
          goto InvalidInstruction;

        uint32_t size = o1.getSize();
        if (ASMJIT_UNLIKELY(size == 0))
          goto AmbiguousOperandSize;

        rmRel = &o1;
        opCode += (size != 1);

        ADD_66H_P_BY_SIZE(size);
        goto EmitX86OpImplicitMem;
      }
      break;

    case X86Inst::kEncodingX86Push:
      if (isign3 == ENC_OPS1(Reg)) {
        if (X86Reg::isSeg(o0)) {
          uint32_t segment = o0.getId();
          if (ASMJIT_UNLIKELY(segment >= X86Seg::kIdCount))
            goto InvalidSegment;

          if (segment >= X86Seg::kIdFs)
            EMIT_BYTE(0x0F);

          EMIT_BYTE(x86OpCodePushSeg[segment]);
          goto EmitDone;
        }
        else {
          goto CaseX86Pop_Gp;
        }
      }

      if (isign3 == ENC_OPS1(Imm)) {
        imVal = static_cast<const Imm&>(o0).getInt64();
        imLen = 4;

        if (Utils::isInt8(imVal) && !(options & X86Inst::kOptionLongForm))
          imLen = 1;

        opCode = imLen == 1 ? 0x6A : 0x68;
        goto EmitX86Op;
      }
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingX86Pop:
      if (isign3 == ENC_OPS1(Reg)) {
        if (X86Reg::isSeg(o0)) {
          uint32_t segment = o0.getId();
          if (ASMJIT_UNLIKELY(segment == X86Seg::kIdCs || segment >= X86Seg::kIdCount))
            goto InvalidSegment;

          if (segment >= X86Seg::kIdFs)
            EMIT_BYTE(0x0F);

          EMIT_BYTE(x86OpCodePopSeg[segment]);
          goto EmitDone;
        }
        else {
CaseX86Pop_Gp:
          // We allow 2 byte, 4 byte, and 8 byte register sizes, although PUSH
          // and POP only allow 2 bytes or native size. On 64-bit we simply
          // PUSH/POP 64-bit register even if 32-bit register was given.
          if (ASMJIT_UNLIKELY(o0.getSize() < 2))
            goto InvalidInstruction;

          opCode = commonData->getAltOpCode();
          opReg = o0.getId();

          ADD_66H_P_BY_SIZE(o0.getSize());
          goto EmitX86OpReg;
        }
      }

      if (isign3 == ENC_OPS1(Mem)) {
        if (ASMJIT_UNLIKELY(o0.getSize() == 0))
          goto AmbiguousOperandSize;

        if (ASMJIT_UNLIKELY(o0.getSize() != 2 && o0.getSize() != getGpSize()))
          goto InvalidInstruction;

        ADD_66H_P_BY_SIZE(o0.getSize());
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Ret:
      if (isign3 == 0) {
        // 'ret' without immediate, change C2 to C3.
        opCode++;
        goto EmitX86Op;
      }

      if (isign3 == ENC_OPS1(Imm)) {
        imVal = static_cast<const Imm&>(o0).getInt64();
        if (imVal == 0 && !(options & X86Inst::kOptionLongForm)) {
          // 'ret' without immediate, change C2 to C3.
          opCode++;
          goto EmitX86Op;
        }
        else {
          imLen = 2;
          goto EmitX86Op;
        }
      }
      break;

    case X86Inst::kEncodingX86Rot:
      if (o0.isReg()) {
        rbReg = o0.getId();

        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, rbReg);
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());
        }

        if (isign3 == ENC_OPS2(Reg, Reg)) {
          if (ASMJIT_UNLIKELY(o1.getId() != X86Gp::kIdCx))
            goto InvalidInstruction;

          opCode += 2;
          goto EmitX86R;
        }

        if (isign3 == ENC_OPS2(Reg, Imm)) {
          imVal = static_cast<const Imm&>(o1).getInt64() & 0xFF;
          imLen = 0;

          if (imVal == 1 && !(options & X86Inst::kOptionLongForm))
            goto EmitX86R;

          imLen = 1;
          opCode -= 0x10;
          goto EmitX86R;
        }
      }
      else {
        opCode += o0.getSize() != 1;
        ADD_PREFIX_BY_SIZE(o0.getSize());

        if (isign3 == ENC_OPS2(Mem, Reg)) {
          if (ASMJIT_UNLIKELY(o1.getId() != X86Gp::kIdCx))
            goto InvalidInstruction;

          opCode += 2;
          rmRel = &o0;
          goto EmitX86M;
        }

        if (isign3 == ENC_OPS2(Mem, Imm)) {
          if (ASMJIT_UNLIKELY(o0.getSize() == 0))
            goto AmbiguousOperandSize;

          imVal = static_cast<const Imm&>(o1).getInt64() & 0xFF;
          imLen = 0;
          rmRel = &o0;

          if (imVal == 1 && !(options & X86Inst::kOptionLongForm))
            goto EmitX86M;

          imLen = 1;
          opCode -= 0x10;
          goto EmitX86M;
        }
      }
      break;

    case X86Inst::kEncodingX86Set:
      if (isign3 == ENC_OPS1(Reg)) {
        rbReg = o0.getId();
        FIXUP_GPB(o0, rbReg);
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS1(Mem)) {
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86ShldShrd:
      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        ADD_PREFIX_BY_SIZE(o0.getSize());
        imVal = static_cast<const Imm&>(o2).getInt64();
        imLen = 1;

        opReg = o1.getId();
        rbReg = o0.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS3(Mem, Reg, Imm)) {
        ADD_PREFIX_BY_SIZE(o1.getSize());
        imVal = static_cast<const Imm&>(o2).getInt64();
        imLen = 1;

        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }

      // The following instructions use opCode + 1.
      opCode++;

      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        if (ASMJIT_UNLIKELY(o2.getId() != X86Gp::kIdCx))
          goto InvalidInstruction;

        ADD_PREFIX_BY_SIZE(o0.getSize());
        opReg = o1.getId();
        rbReg = o0.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS3(Mem, Reg, Reg)) {
        if (ASMJIT_UNLIKELY(o2.getId() != X86Gp::kIdCx))
          goto InvalidInstruction;

        ADD_PREFIX_BY_SIZE(o1.getSize());
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86StrRm:
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        rmRel = &o1;
        if (ASMJIT_UNLIKELY(rmRel->as<X86Mem>().getOffsetLo32() || !X86Reg::isGp(o0.as<X86Reg>(), X86Gp::kIdAx)))
          goto InvalidInstruction;

        uint32_t size = o0.getSize();
        if (o1.hasSize() && ASMJIT_UNLIKELY(o1.getSize() != size))
          goto OperandSizeMismatch;

        ADD_PREFIX_BY_SIZE(size);
        opCode += static_cast<uint32_t>(size != 1);

        goto EmitX86OpImplicitMem;
      }
      break;

    case X86Inst::kEncodingX86StrMr:
      if (isign3 == ENC_OPS2(Mem, Reg)) {
        rmRel = &o0;
        if (ASMJIT_UNLIKELY(rmRel->as<X86Mem>().getOffsetLo32() || !X86Reg::isGp(o1.as<X86Reg>(), X86Gp::kIdAx)))
          goto InvalidInstruction;

        uint32_t size = o1.getSize();
        if (o0.hasSize() && ASMJIT_UNLIKELY(o0.getSize() != size))
          goto OperandSizeMismatch;

        ADD_PREFIX_BY_SIZE(size);
        opCode += static_cast<uint32_t>(size != 1);

        goto EmitX86OpImplicitMem;
      }
      break;

    case X86Inst::kEncodingX86StrMm:
      if (isign3 == ENC_OPS2(Mem, Mem)) {
        if (ASMJIT_UNLIKELY(o0.as<X86Mem>().getBaseIndexType() !=
                            o1.as<X86Mem>().getBaseIndexType()))
          goto InvalidInstruction;

        rmRel = &o1;
        if (ASMJIT_UNLIKELY(o0.as<X86Mem>().hasOffset()))
          goto InvalidInstruction;

        uint32_t size = o1.getSize();
        if (ASMJIT_UNLIKELY(size == 0))
          goto AmbiguousOperandSize;

        if (ASMJIT_UNLIKELY(o0.getSize() != size))
          goto OperandSizeMismatch;

        ADD_PREFIX_BY_SIZE(size);
        opCode += static_cast<uint32_t>(size != 1);

        goto EmitX86OpImplicitMem;
      }
      break;

    case X86Inst::kEncodingX86Test:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        if (o0.getSize() != o1.getSize())
          goto OperandSizeMismatch;

        rbReg = o0.getId();
        opReg = o1.getId();

        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, rbReg);
          FIXUP_GPB(o1, opReg);
          goto EmitX86R;
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());
          goto EmitX86R;
        }
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;

        if (o1.getSize() == 1) {
          FIXUP_GPB(o1, opReg);
          goto EmitX86M;
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o1.getSize());
          goto EmitX86M;
        }
      }

      // The following instructions use the secondary opcode.
      opCode = commonData->getAltOpCode();
      opReg = x86ExtractO(opCode);

      if (isign3 == ENC_OPS2(Reg, Imm)) {
        rbReg = o0.getId();

        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, rbReg);

          imVal = static_cast<const Imm&>(o1).getUInt8();
          imLen = 1;
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());

          imVal = static_cast<const Imm&>(o1).getInt64();
          imLen = std::min<uint32_t>(o0.getSize(), 4);
        }

        // Alternate Form - AL, AX, EAX, RAX.
        if (o0.getId() == 0 && !(options & X86Inst::kOptionLongForm)) {
          opCode &= X86Inst::kOpCode_PP_66 | X86Inst::kOpCode_W;
          opCode |= 0xA8 + (o0.getSize() != 1);
          goto EmitX86Op;
        }

        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Mem, Imm)) {
        if (ASMJIT_UNLIKELY(o0.getSize() == 0))
          goto AmbiguousOperandSize;

        imVal = static_cast<const Imm&>(o1).getInt64();
        imLen = std::min<uint32_t>(o0.getSize(), 4);

        opCode += (o0.getSize() != 1);
        ADD_PREFIX_BY_SIZE(o0.getSize());

        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Xchg:
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;

        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, opReg);
          goto EmitX86M;
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());
          goto EmitX86M;
        }
      }
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingX86Xadd:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        rbReg = o0.getId();
        opReg = o1.getId();

        if (o0.getSize() != o1.getSize())
          goto OperandSizeMismatch;

        if (o0.getSize() == 1) {
          FIXUP_GPB(o0, rbReg);
          FIXUP_GPB(o1, opReg);
          goto EmitX86R;
        }
        else {
          opCode++;
          ADD_PREFIX_BY_SIZE(o0.getSize());

          // Special opcode for 'xchg ?ax, reg'.
          if (instId == X86Inst::kIdXchg && (opReg == 0 || rbReg == 0)) {
            opCode &= X86Inst::kOpCode_PP_66 | X86Inst::kOpCode_W;
            opCode |= 0x90;
            // One of `xchg a, b` or `xchg b, a` is AX/EAX/RAX.
            opReg += rbReg;
            goto EmitX86OpReg;
          }
          else {
            goto EmitX86R;
          }
        }
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opCode += o1.getSize() != 1;
        ADD_PREFIX_BY_SIZE(o1.getSize());

        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingX86Fence:
      rbReg = 0;
      goto EmitX86R;

    case X86Inst::kEncodingX86Bndmov:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();

        // ModRM encoding:
        if (!(options & X86Inst::kOptionModMR))
          goto EmitX86R;

        // ModMR encoding:
        opCode = commonData->getAltOpCode();
        std::swap(opReg, rbReg);
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opCode = commonData->getAltOpCode();

        rmRel = &o0;
        opReg = o1.getId();
        goto EmitX86M;
      }
      break;

    // ------------------------------------------------------------------------
    // [FPU]
    // ------------------------------------------------------------------------

    case X86Inst::kEncodingFpuOp:
      goto EmitFpuOp;

    case X86Inst::kEncodingFpuArith:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();

        // We switch to the alternative opcode if the first operand is zero.
        if (opReg == 0) {
CaseFpuArith_Reg:
          opCode = ((0xD8   << X86Inst::kOpCode_FPU_2B_Shift)       ) +
                   ((opCode >> X86Inst::kOpCode_FPU_2B_Shift) & 0xFF) + rbReg;
          goto EmitFpuOp;
        }
        else if (rbReg == 0) {
          rbReg = opReg;
          opCode = ((0xDC   << X86Inst::kOpCode_FPU_2B_Shift)       ) +
                   ((opCode                                 ) & 0xFF) + rbReg;
          goto EmitFpuOp;
        }
        else {
          goto InvalidInstruction;
        }
      }

      if (isign3 == ENC_OPS1(Mem)) {
CaseFpuArith_Mem:
        // 0xD8/0xDC, depends on the size of the memory operand; opReg is valid.
        opCode = (o0.getSize() == 4) ? 0xD8 : 0xDC;
        // Clear compressed displacement before going to EmitX86M.
        opCode &= ~static_cast<uint32_t>(X86Inst::kOpCode_CDSHL_Mask);

        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingFpuCom:
      if (isign3 == 0) {
        rbReg = 1;
        goto CaseFpuArith_Reg;
      }

      if (isign3 == ENC_OPS1(Reg)) {
        rbReg = o0.getId();
        goto CaseFpuArith_Reg;
      }

      if (isign3 == ENC_OPS1(Mem)) {
        goto CaseFpuArith_Mem;
      }
      break;

    case X86Inst::kEncodingFpuFldFst:
      if (isign3 == ENC_OPS1(Mem)) {
        rmRel = &o0;

        if (o0.getSize() == 4 && commonData->hasFlag(X86Inst::kFlagFpuM32)) {
          goto EmitX86M;
        }

        if (o0.getSize() == 8 && commonData->hasFlag(X86Inst::kFlagFpuM64)) {
          opCode += 4;
          goto EmitX86M;
        }

        if (o0.getSize() == 10 && commonData->hasFlag(X86Inst::kFlagFpuM80)) {
          opCode = commonData->getAltOpCode();
          opReg  = x86ExtractO(opCode);
          goto EmitX86M;
        }
      }

      if (isign3 == ENC_OPS1(Reg)) {
        if (instId == X86Inst::kIdFld ) { opCode = (0xD9 << X86Inst::kOpCode_FPU_2B_Shift) + 0xC0 + o0.getId(); goto EmitFpuOp; }
        if (instId == X86Inst::kIdFst ) { opCode = (0xDD << X86Inst::kOpCode_FPU_2B_Shift) + 0xD0 + o0.getId(); goto EmitFpuOp; }
        if (instId == X86Inst::kIdFstp) { opCode = (0xDD << X86Inst::kOpCode_FPU_2B_Shift) + 0xD8 + o0.getId(); goto EmitFpuOp; }
      }
      break;

    case X86Inst::kEncodingFpuM:
      if (isign3 == ENC_OPS1(Mem)) {
        // Clear compressed displacement before going to EmitX86M.
        opCode &= ~static_cast<uint32_t>(X86Inst::kOpCode_CDSHL_Mask);

        rmRel = &o0;
        if (o0.getSize() == 2 && commonData->hasFlag(X86Inst::kFlagFpuM16)) {
          opCode += 4;
          goto EmitX86M;
        }

        if (o0.getSize() == 4 && commonData->hasFlag(X86Inst::kFlagFpuM32)) {
          goto EmitX86M;
        }

        if (o0.getSize() == 8 && commonData->hasFlag(X86Inst::kFlagFpuM64)) {
          opCode = commonData->getAltOpCode() & ~static_cast<uint32_t>(X86Inst::kOpCode_CDSHL_Mask);
          opReg  = x86ExtractO(opCode);
          goto EmitX86M;
        }
      }
      break;

    case X86Inst::kEncodingFpuRDef:
      if (isign3 == 0) {
        opCode += 1;
        goto EmitFpuOp;
      }
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingFpuR:
      if (isign3 == ENC_OPS1(Reg)) {
        opCode += o0.getId();
        goto EmitFpuOp;
      }
      break;

    case X86Inst::kEncodingFpuStsw:
      if (isign3 == ENC_OPS1(Reg)) {
        if (ASMJIT_UNLIKELY(o0.getId() != X86Gp::kIdAx))
          goto InvalidInstruction;

        opCode = commonData->getAltOpCode();
        goto EmitFpuOp;
      }

      if (isign3 == ENC_OPS1(Mem)) {
        // Clear compressed displacement before going to EmitX86M.
        opCode &= ~static_cast<uint32_t>(X86Inst::kOpCode_CDSHL_Mask);

        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    // ------------------------------------------------------------------------
    // [Ext]
    // ------------------------------------------------------------------------

    case X86Inst::kEncodingExtPextrw:
      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        ADD_66H_P(X86Reg::isXmm(o1));

        imVal = static_cast<const Imm&>(o2).getInt64();
        imLen = 1;

        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS3(Mem, Reg, Imm)) {
        // Secondary opcode of 'pextrw' instruction (SSE4.1).
        opCode = commonData->getAltOpCode();
        ADD_66H_P(X86Reg::isXmm(o1));

        imVal = static_cast<const Imm&>(o2).getInt64();
        imLen = 1;

        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtExtract:
      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        ADD_66H_P(X86Reg::isXmm(o1));

        imVal = static_cast<const Imm&>(o2).getInt64();
        imLen = 1;

        opReg = o1.getId();
        rbReg = o0.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS3(Mem, Reg, Imm)) {
        ADD_66H_P(X86Reg::isXmm(o1));

        imVal = static_cast<const Imm&>(o2).getInt64();
        imLen = 1;

        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtMov:
      // GP|MMX|XMM <- GP|MMX|XMM
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();

        if (!(options & X86Inst::kOptionModMR) || !commonData->hasAltOpCode())
          goto EmitX86R;

        opCode = commonData->getAltOpCode();
        Utils::swap(opReg, rbReg);
        goto EmitX86R;
      }

      // GP|MMX|XMM <- Mem
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }

      // The following instruction uses opCode[1].
      opCode = commonData->getAltOpCode();

      // Mem <- GP|MMX|XMM
      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtMovnti:
      if (isign3 == ENC_OPS2(Mem, Reg)) {
        ADD_REX_W(X86Reg::isGpq(o1));

        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtMovbe:
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        if (o0.getSize() == 1)
          goto InvalidInstruction;

        ADD_PREFIX_BY_SIZE(o0.getSize());
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }

      // The following instruction uses the secondary opcode.
      opCode = commonData->getAltOpCode();

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        if (o1.getSize() == 1)
          goto InvalidInstruction;

        ADD_PREFIX_BY_SIZE(o1.getSize());
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtMovd:
CaseExtMovd:
      opReg = o0.getId();
      ADD_66H_P(X86Reg::isXmm(o0));

      // MMX/XMM <- Gp
      if (isign3 == ENC_OPS2(Reg, Reg) && X86Reg::isGp(o1)) {
        rbReg = o1.getId();
        goto EmitX86R;
      }

      // MMX/XMM <- Mem
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        rmRel = &o1;
        goto EmitX86M;
      }

      // The following instructions use the secondary opcode.
      opCode &= X86Inst::kOpCode_W;
      opCode |= commonData->getAltOpCode();
      opReg = o1.getId();
      ADD_66H_P(X86Reg::isXmm(o1));

      // GP <- MMX/XMM
      if (isign3 == ENC_OPS2(Reg, Reg) && X86Reg::isGp(o0)) {
        rbReg = o0.getId();
        goto EmitX86R;
      }

      // Mem <- MMX/XMM
      if (isign3 == ENC_OPS2(Mem, Reg)) {
        rmRel = &o0;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtMovq:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();

        // MMX <- MMX
        if (X86Reg::isMm(o0) && X86Reg::isMm(o1)) {
          opCode = X86Inst::kOpCode_PP_00 | X86Inst::kOpCode_MM_0F | 0x6F;

          if (!(options & X86Inst::kOptionModMR))
            goto EmitX86R;

          opCode += 0x10;
          Utils::swap(opReg, rbReg);
          goto EmitX86R;
        }

        // XMM <- XMM
        if (X86Reg::isXmm(o0) && X86Reg::isXmm(o1)) {
          opCode = X86Inst::kOpCode_PP_F3 | X86Inst::kOpCode_MM_0F | 0x7E;

          if (!(options & X86Inst::kOptionModMR))
            goto EmitX86R;

          opCode = X86Inst::kOpCode_PP_66 | X86Inst::kOpCode_MM_0F | 0xD6;
          Utils::swap(opReg, rbReg);
          goto EmitX86R;
        }

        // MMX <- XMM (MOVDQ2Q)
        if (X86Reg::isMm(o0) && X86Reg::isXmm(o1)) {
          opCode = X86Inst::kOpCode_PP_F2 | X86Inst::kOpCode_MM_0F | 0xD6;
          goto EmitX86R;
        }

        // XMM <- MMX (MOVQ2DQ)
        if (X86Reg::isXmm(o0) && X86Reg::isMm(o1)) {
          opCode = X86Inst::kOpCode_PP_F3 | X86Inst::kOpCode_MM_0F | 0xD6;
          goto EmitX86R;
        }
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;

        // MMX <- Mem
        if (X86Reg::isMm(o0)) {
          opCode = X86Inst::kOpCode_PP_00 | X86Inst::kOpCode_MM_0F | 0x6F;
          goto EmitX86M;
        }

        // XMM <- Mem
        if (X86Reg::isXmm(o0)) {
          opCode = X86Inst::kOpCode_PP_F3 | X86Inst::kOpCode_MM_0F | 0x7E;
          goto EmitX86M;
        }
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;

        // Mem <- MMX
        if (X86Reg::isMm(o1)) {
          opCode = X86Inst::kOpCode_PP_00 | X86Inst::kOpCode_MM_0F | 0x7F;
          goto EmitX86M;
        }

        // Mem <- XMM
        if (X86Reg::isXmm(o1)) {
          opCode = X86Inst::kOpCode_PP_66 | X86Inst::kOpCode_MM_0F | 0xD6;
          goto EmitX86M;
        }
      }

      // MOVQ in other case is simply a MOVD instruction promoted to 64-bit.
      opCode |= X86Inst::kOpCode_W;
      goto CaseExtMovd;

    case X86Inst::kEncodingExtRm_XMM0:
      if (ASMJIT_UNLIKELY(!o2.isNone() && !X86Reg::isXmm(o2, 0)))
        goto InvalidInstruction;

      isign3 &= 0x3F;
      goto CaseExtRm;

    case X86Inst::kEncodingExtRm_ZDI:
      if (ASMJIT_UNLIKELY(!o2.isNone() && !x86IsImplicitMem(o2, X86Gp::kIdDi)))
        goto InvalidInstruction;

      isign3 &= 0x3F;
      goto CaseExtRm;

    case X86Inst::kEncodingExtRm_Wx:
      ADD_REX_W(X86Reg::isGpq(o0) || o1.getSize() == 8);
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingExtRm:
CaseExtRm:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtRm_P:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        ADD_66H_P(X86Reg::isXmm(o0) | X86Reg::isXmm(o1));

        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        ADD_66H_P(X86Reg::isXmm(o0));

        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtRmRi:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }

      // The following instruction uses the secondary opcode.
      opCode = commonData->getAltOpCode();
      opReg  = x86ExtractO(opCode);

      if (isign3 == ENC_OPS2(Reg, Imm)) {
        imVal = static_cast<const Imm&>(o1).getInt64();
        imLen = 1;

        rbReg = o0.getId();
        goto EmitX86R;
      }
      break;

    case X86Inst::kEncodingExtRmRi_P:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        ADD_66H_P(X86Reg::isXmm(o0) | X86Reg::isXmm(o1));

        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        ADD_66H_P(X86Reg::isXmm(o0));

        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }

      // The following instruction uses the secondary opcode.
      opCode = commonData->getAltOpCode();
      opReg  = x86ExtractO(opCode);

      if (isign3 == ENC_OPS2(Reg, Imm)) {
        ADD_66H_P(X86Reg::isXmm(o0));

        imVal = static_cast<const Imm&>(o1).getInt64();
        imLen = 1;

        rbReg = o0.getId();
        goto EmitX86R;
      }
      break;

    case X86Inst::kEncodingExtRmi:
      imVal = static_cast<const Imm&>(o2).getInt64();
      imLen = 1;

      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Imm)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }
      break;

    case X86Inst::kEncodingExtRmi_P:
      imVal = static_cast<const Imm&>(o2).getInt64();
      imLen = 1;

      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        ADD_66H_P(X86Reg::isXmm(o0) | X86Reg::isXmm(o1));

        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Imm)) {
        ADD_66H_P(X86Reg::isXmm(o0));

        opReg = o0.getId();
        rmRel = &o1;
        goto EmitX86M;
      }
      break;

    // ------------------------------------------------------------------------
    // [Extrq / Insertq (SSE4A)]
    // ------------------------------------------------------------------------

    case X86Inst::kEncodingExtExtrq:
      opReg = o0.getId();
      rbReg = o1.getId();

      if (isign3 == ENC_OPS2(Reg, Reg))
        goto EmitX86R;

      // The following instruction uses the secondary opcode.
      opCode = commonData->getAltOpCode();

      if (isign3 == ENC_OPS3(Reg, Imm, Imm)) {
        imVal = (static_cast<const Imm&>(o1).getUInt32()     ) +
                (static_cast<const Imm&>(o2).getUInt32() << 8) ;
        imLen = 2;

        rbReg = x86ExtractO(opCode);
        goto EmitX86R;
      }
      break;

    case X86Inst::kEncodingExtInsertq: {
      const uint32_t isign4 = isign3 + (o3.getOp() << 9);
      opReg = o0.getId();
      rbReg = o1.getId();

      if (isign4 == ENC_OPS2(Reg, Reg))
        goto EmitX86R;

      // The following instruction uses the secondary opcode.
      opCode = commonData->getAltOpCode();

      if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
        imVal = (static_cast<const Imm&>(o2).getUInt32()     ) +
                (static_cast<const Imm&>(o3).getUInt32() << 8) ;
        imLen = 2;
        goto EmitX86R;
      }
      break;
    }

    // ------------------------------------------------------------------------
    // [3dNow]
    // ------------------------------------------------------------------------

    case X86Inst::kEncodingExt3dNow:
      // Every 3dNow instruction starts with 0x0F0F and the actual opcode is
      // stored as 8-bit immediate.
      imVal = opCode & 0xFF;
      imLen = 1;

      opCode = X86Inst::kOpCode_MM_0F | 0x0F;
      opReg = o0.getId();

      if (isign3 == ENC_OPS2(Reg, Reg)) {
        rbReg = o1.getId();
        goto EmitX86R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        rmRel = &o1;
        goto EmitX86M;
      }
      break;

    // ------------------------------------------------------------------------
    // [VEX/EVEX]
    // ------------------------------------------------------------------------

    case X86Inst::kEncodingVexOp:
      goto EmitVexEvexOp;

    case X86Inst::kEncodingVexKmov:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();

        // Form 'k, reg'.
        if (X86Reg::isGp(o1)) {
          opCode = commonData->getAltOpCode();
          goto EmitVexEvexR;
        }

        // Form 'reg, k'.
        if (X86Reg::isGp(o0)) {
          opCode = commonData->getAltOpCode() + 1;
          goto EmitVexEvexR;
        }

        // Form 'k, k'.
        if (!(options & X86Inst::kOptionModMR))
          goto EmitVexEvexR;

        opCode++;
        Utils::swap(opReg, rbReg);
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;

        goto EmitVexEvexM;
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;

        opCode++;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexM:
      if (isign3 == ENC_OPS1(Mem)) {
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexM_VM:
      if (isign3 == ENC_OPS1(Mem)) {
        opCode |= x86OpCodeLByVMem(o0);
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexMr_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());

      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o1.getId();
        rbReg = o0.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexMr_VM:
      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opCode |= std::max(x86OpCodeLByVMem(o0), x86OpCodeLBySize(o1.getSize()));

        opReg = o1.getId();
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexMri_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexMri:
      imVal = static_cast<const Imm&>(o2).getInt64();
      imLen = 1;

      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        opReg = o1.getId();
        rbReg = o0.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Mem, Reg, Imm)) {
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRm_ZDI:
      if (ASMJIT_UNLIKELY(!o2.isNone() && !x86IsImplicitMem(o2, X86Gp::kIdDi)))
        goto InvalidInstruction;

      isign3 &= 0x3F;
      goto CaseVexRm;

    case X86Inst::kEncodingVexRm_Wx:
      ADD_REX_W(X86Reg::isGpq(o0) | X86Reg::isGpq(o1));
      goto CaseVexRm;

    case X86Inst::kEncodingVexRm_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRm:
CaseVexRm:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRm_VM:
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opCode |= std::max(x86OpCodeLByVMem(o1), x86OpCodeLBySize(o0.getSize()));
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRm_T1_4X: {
      if (!(options & kOptionOp4Op5Used))
        goto InvalidInstruction;

      if (X86Reg::isZmm(o0  ) && X86Reg::isZmm(o1) &&
          X86Reg::isZmm(o2  ) && X86Reg::isZmm(o3) &&
          X86Reg::isZmm(_op4) && _op5.isMem()) {

        // Registers [o1, o2, o3, _op4] must start aligned and must be consecutive.
        uint32_t i1 = o1.getId();
        uint32_t i2 = o2.getId();
        uint32_t i3 = o3.getId();
        uint32_t i4 = _op4.getId();

        if (ASMJIT_UNLIKELY((i1 & 0x3) != 0 || i2 != i1 + 1 || i3 != i1 + 2 || i4 != i1 + 3))
          goto NotConsecutiveRegs;

        opReg = o0.getId();
        rmRel = &_op5;
        goto EmitVexEvexM;
      }
      break;
    }

    case X86Inst::kEncodingVexRmi_Wx:
      ADD_REX_W(X86Reg::isGpq(o0) | X86Reg::isGpq(o1));
      goto CaseVexRmi;

    case X86Inst::kEncodingVexRmi_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRmi:
CaseVexRmi:
      imVal = static_cast<const Imm&>(o2).getInt64();
      imLen = 1;

      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Imm)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvm:
CaseVexRvm:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
CaseVexRvm_R:
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Reg, Mem)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvm_ZDX_Wx:
      if (ASMJIT_UNLIKELY(!o3.isNone() && !X86Reg::isGp(o3, X86Gp::kIdDx)))
        goto InvalidInstruction;
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRvm_Wx:
      ADD_REX_W(X86Reg::isGpq(o0) | (o2.getSize() == 8));
      goto CaseVexRvm;

    case X86Inst::kEncodingVexRvm_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      goto CaseVexRvm;

    case X86Inst::kEncodingVexRvmr_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRvmr: {
      const uint32_t isign4 = isign3 + (o3.getOp() << 9);
      imVal = o3.getId() << 4;
      imLen = 1;

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();
        goto EmitVexEvexR;
      }

      if (isign4 == ENC_OPS4(Reg, Reg, Mem, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;
        goto EmitVexEvexM;
      }
      break;
    }

    case X86Inst::kEncodingVexRvmi_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRvmi: {
      const uint32_t isign4 = isign3 + (o3.getOp() << 9);
      imVal = static_cast<const Imm&>(o3).getInt64();
      imLen = 1;

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();
        goto EmitVexEvexR;
      }

      if (isign4 == ENC_OPS4(Reg, Reg, Mem, Imm)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;
        goto EmitVexEvexM;
      }
      break;
    }

    case X86Inst::kEncodingVexRmv_Wx:
      ADD_REX_W(X86Reg::isGpq(o0) | X86Reg::isGpq(o2));
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRmv:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRmvRm_VM:
      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opCode  = commonData->getAltOpCode();
        opCode |= std::max(x86OpCodeLByVMem(o1), x86OpCodeLBySize(o0.getSize()));

        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }

      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRmv_VM:
      if (isign3 == ENC_OPS3(Reg, Mem, Reg)) {
        opCode |= std::max(x86OpCodeLByVMem(o1), x86OpCodeLBySize(o0.getSize() | o2.getSize()));

        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;


    case X86Inst::kEncodingVexRmvi: {
      const uint32_t isign4 = isign3 + (o3.getOp() << 9);
      imVal = static_cast<const Imm&>(o3).getInt64();
      imLen = 1;

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign4 == ENC_OPS4(Reg, Mem, Reg, Imm)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;
    }

    case X86Inst::kEncodingVexMovdMovq:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        if (X86Reg::isGp(o0)) {
          opCode = commonData->getAltOpCode();
          ADD_REX_W_BY_SIZE(o0.getSize());
          opReg = o1.getId();
          rbReg = o0.getId();
          goto EmitVexEvexR;
        }

        if (X86Reg::isGp(o1)) {
          ADD_REX_W_BY_SIZE(o1.getSize());
          opReg = o0.getId();
          rbReg = o1.getId();
          goto EmitVexEvexR;
        }

        // If this is a 'W' version (movq) then allow also vmovq 'xmm|xmm' form.
        if (opCode & X86Inst::kOpCode_EW) {
          opCode &= ~(X86Inst::kOpCode_PP_VEXMask | X86Inst::kOpCode_MM_Mask | 0xFF);
          opCode |=  (X86Inst::kOpCode_PP_F3      | X86Inst::kOpCode_MM_0F   | 0x7E);

          opReg = o0.getId();
          rbReg = o1.getId();
          goto EmitVexEvexR;
        }
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        if (opCode & X86Inst::kOpCode_EW) {
          opCode &= ~(X86Inst::kOpCode_PP_VEXMask | X86Inst::kOpCode_MM_Mask | 0xFF);
          opCode |=  (X86Inst::kOpCode_PP_F3      | X86Inst::kOpCode_MM_0F   | 0x7E);
        }

        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }

      // The following instruction uses the secondary opcode.
      opCode = commonData->getAltOpCode();

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        if (opCode & X86Inst::kOpCode_EW) {
          opCode &= ~(X86Inst::kOpCode_PP_VEXMask | X86Inst::kOpCode_MM_Mask | 0xFF);
          opCode |=  (X86Inst::kOpCode_PP_66      | X86Inst::kOpCode_MM_0F   | 0xD6);
        }

        opReg = o1.getId();
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRmMr_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRmMr:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }

      // The following instruction uses the secondary opcode.
      opCode &= X86Inst::kOpCode_LL_Mask;
      opCode |= commonData->getAltOpCode();

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvmRmv:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rbReg = o1.getId();

        if (!(options & X86Inst::kOptionModMR))
          goto EmitVexEvexR;

        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();

        ADD_VEX_W(true);
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rmRel = &o1;
        goto EmitVexEvexM;
      }

      if (isign3 == ENC_OPS3(Reg, Reg, Mem)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;

        ADD_VEX_W(true);
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvmRmi_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRvmRmi:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Reg, Mem)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;
        goto EmitVexEvexM;
      }

      // The following instructions use the secondary opcode.
      opCode &= X86Inst::kOpCode_LL_Mask;
      opCode |= commonData->getAltOpCode();

      imVal = static_cast<const Imm&>(o2).getInt64();
      imLen = 1;

      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Imm)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvmRmvRmi:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rbReg = o1.getId();

        if (!(options & X86Inst::kOptionModMR))
          goto EmitVexEvexR;

        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();

        ADD_VEX_W(true);
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o2.getId());
        rmRel = &o1;

        goto EmitVexEvexM;
      }

      if (isign3 == ENC_OPS3(Reg, Reg, Mem)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;

        ADD_VEX_W(true);
        goto EmitVexEvexM;
      }

      // The following instructions use the secondary opcode.
      opCode = commonData->getAltOpCode();

      imVal = static_cast<const Imm&>(o2).getInt64();
      imLen = 1;

      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        opReg = o0.getId();
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Imm)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvmMr:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Reg, Mem)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;
        goto EmitVexEvexM;
      }

      // The following instructions use the secondary opcode.
      opCode = commonData->getAltOpCode();

      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = o1.getId();
        rbReg = o0.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvmMvr_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRvmMvr:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Reg, Mem)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;
        goto EmitVexEvexM;
      }

      // The following instruction uses the secondary opcode.
      opCode &= X86Inst::kOpCode_LL_Mask;
      opCode |= commonData->getAltOpCode();

      if (isign3 == ENC_OPS3(Mem, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o2.getId(), o1.getId());
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvmVmi_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRvmVmi:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Reg, Mem)) {
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;
        goto EmitVexEvexM;
      }

      // The following instruction uses the secondary opcode.
      opCode &= X86Inst::kOpCode_LL_Mask;
      opCode |= commonData->getAltOpCode();
      opReg = x86ExtractO(opCode);

      imVal = static_cast<const Imm&>(o2).getInt64();
      imLen = 1;

      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        opReg = x86PackRegAndVvvvv(opReg, o0.getId());
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Imm)) {
        opReg = x86PackRegAndVvvvv(opReg, o0.getId());
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexVm_Wx:
      ADD_REX_W(X86Reg::isGpq(o0) | X86Reg::isGpq(o1));
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexVm:
      if (isign3 == ENC_OPS2(Reg, Reg)) {
        opReg = x86PackRegAndVvvvv(opReg, o0.getId());
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = x86PackRegAndVvvvv(opReg, o0.getId());
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexEvexVmi_Lx:
      if (isign3 == ENC_OPS3(Reg, Mem, Imm))
        opCode |= X86Inst::kOpCode_MM_ForceEvex;
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexVmi_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexVmi:
      imVal = static_cast<const Imm&>(o2).getInt64();
      imLen = 1;

      if (isign3 == ENC_OPS3(Reg, Reg, Imm)) {
        opReg = x86PackRegAndVvvvv(opReg, o0.getId());
        rbReg = o1.getId();
        goto EmitVexEvexR;
      }

      if (isign3 == ENC_OPS3(Reg, Mem, Imm)) {
        opReg = x86PackRegAndVvvvv(opReg, o0.getId());
        rmRel = &o1;
        goto EmitVexEvexM;
      }
      break;

    case X86Inst::kEncodingVexRvrmRvmr_Lx:
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingVexRvrmRvmr: {
      const uint32_t isign4 = isign3 + (o3.getOp() << 9);

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
        imVal = o3.getId() << 4;
        imLen = 1;

        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();

        goto EmitVexEvexR;
      }

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Mem)) {
        imVal = o2.getId() << 4;
        imLen = 1;

        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o3;

        ADD_VEX_W(true);
        goto EmitVexEvexM;
      }

      if (isign4 == ENC_OPS4(Reg, Reg, Mem, Reg)) {
        imVal = o3.getId() << 4;
        imLen = 1;

        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;

        goto EmitVexEvexM;
      }
      break;
    }

    case X86Inst::kEncodingVexRvrmiRvmri_Lx: {
      if (!(options & CodeEmitter::kOptionOp4Op5Used) || !_op4.isImm())
        goto InvalidInstruction;

      const uint32_t isign4 = isign3 + (o3.getOp() << 9);
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize() | o2.getSize() | o3.getSize());

      imVal = static_cast<const Imm&>(_op4).getUInt8() & 0x0F;
      imLen = 1;

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
        imVal |= o3.getId() << 4;
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();

        goto EmitVexEvexR;
      }

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Mem)) {
        imVal |= o2.getId() << 4;
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o3;

        ADD_VEX_W(true);
        goto EmitVexEvexM;
      }

      if (isign4 == ENC_OPS4(Reg, Reg, Mem, Reg)) {
        imVal |= o3.getId() << 4;
        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;

        goto EmitVexEvexM;
      }
      break;
    }

    case X86Inst::kEncodingVexMovssMovsd:
      if (isign3 == ENC_OPS3(Reg, Reg, Reg)) {
        goto CaseVexRvm_R;
      }

      if (isign3 == ENC_OPS2(Reg, Mem)) {
        opReg = o0.getId();
        rmRel = &o1;
        goto EmitVexEvexM;
      }

      if (isign3 == ENC_OPS2(Mem, Reg)) {
        opCode = commonData->getAltOpCode();
        opReg = o1.getId();
        rmRel = &o0;
        goto EmitVexEvexM;
      }
      break;

    // ------------------------------------------------------------------------
    // [FMA4]
    // ------------------------------------------------------------------------

    case X86Inst::kEncodingFma4_Lx:
      // It's fine to just check the first operand, second is just for sanity.
      opCode |= x86OpCodeLBySize(o0.getSize() | o1.getSize());
      ASMJIT_FALLTHROUGH;

    case X86Inst::kEncodingFma4: {
      const uint32_t isign4 = isign3 + (o3.getOp() << 9);

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
        imVal = o3.getId() << 4;
        imLen = 1;

        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rbReg = o2.getId();

        goto EmitVexEvexR;
      }

      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Mem)) {
        imVal = o2.getId() << 4;
        imLen = 1;

        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o3;

        ADD_VEX_W(true);
        goto EmitVexEvexM;
      }

      if (isign4 == ENC_OPS4(Reg, Reg, Mem, Reg)) {
        imVal = o3.getId() << 4;
        imLen = 1;

        opReg = x86PackRegAndVvvvv(o0.getId(), o1.getId());
        rmRel = &o2;

        goto EmitVexEvexM;
      }
      break;
    }
  }
  goto InvalidInstruction;

  // --------------------------------------------------------------------------
  // [Emit - X86]
  // --------------------------------------------------------------------------

EmitX86OpMovAbs:
  imLen = getGpSize();

  // Segment-override prefix.
  if (rmRel->as<X86Mem>().hasSegment())
    EMIT_BYTE(x86SegmentPrefix[rmRel->as<X86Mem>().getSegmentId()]);

EmitX86Op:
  // Emit mandatory instruction prefix.
  EMIT_PP(opCode);

  // Emit REX prefix (64-bit only).
  {
    uint32_t rex = x86ExtractREX(opCode, options);
    if (rex) {
      if (options & X86Inst::_kOptionInvalidRex)
        goto InvalidRexPrefix;
      EMIT_BYTE(rex | kX86ByteRex);
    }
  }

  // Emit instruction opcodes.
  EMIT_MM_OP(opCode);

  if (imLen != 0)
    goto EmitImm;
  else
    goto EmitDone;

EmitX86OpReg:
  // Emit mandatory instruction prefix.
  EMIT_PP(opCode);

  // Emit REX prefix (64-bit only).
  {
    uint32_t rex = x86ExtractREX(opCode, options) |
                   (opReg >> 3); // Rex.B (0x01).
    if (rex) {
      EMIT_BYTE(rex | kX86ByteRex);
      if (options & X86Inst::_kOptionInvalidRex)
        goto InvalidRexPrefix;
      opReg &= 0x7;
    }
  }

  // Emit instruction opcodes.
  opCode += opReg;
  EMIT_MM_OP(opCode);

  if (imLen != 0)
    goto EmitImm;
  else
    goto EmitDone;

EmitX86OpImplicitMem:
  // NOTE: Don't change the emit order here, it's compatible with KeyStone/LLVM.
  rmInfo = x86MemInfo[rmRel->as<X86Mem>().getBaseIndexType()];
  if (ASMJIT_UNLIKELY(rmRel->as<X86Mem>().hasOffset() || (rmInfo & kX86MemInfo_Index)))
    goto InvalidInstruction;

  // Emit mandatory instruction prefix.
  EMIT_PP(opCode);

  // Emit REX prefix (64-bit only).
  {
    uint32_t rex = x86ExtractREX(opCode, options);
    if (rex) {
      if (options & X86Inst::_kOptionInvalidRex)
        goto InvalidRexPrefix;
      EMIT_BYTE(rex | kX86ByteRex);
    }
  }

  // Segment-override prefix.
  if (rmRel->as<X86Mem>().hasSegment())
    EMIT_BYTE(x86SegmentPrefix[rmRel->as<X86Mem>().getSegmentId()]);

  // Address-override prefix.
  if (rmInfo & _getAddressOverrideMask())
    EMIT_BYTE(0x67);

  // Emit instruction opcodes.
  EMIT_MM_OP(opCode);

  if (imLen != 0)
    goto EmitImm;
  else
    goto EmitDone;

EmitX86R:
  // Mandatory instruction prefix.
  EMIT_PP(opCode);

  // Rex prefix (64-bit only).
  {
    uint32_t rex = x86ExtractREX(opCode, options) |
                   ((opReg & 0x08) >> 1) | // REX.R (0x04).
                   ((rbReg       ) >> 3) ; // REX.B (0x01).
    if (rex) {
      if (options & X86Inst::_kOptionInvalidRex)
        goto InvalidRexPrefix;
      EMIT_BYTE(rex | kX86ByteRex);
      opReg &= 0x07;
      rbReg &= 0x07;
    }
  }

  // Instruction opcodes.
  EMIT_MM_OP(opCode);
  // ModR.
  EMIT_BYTE(x86EncodeMod(3, opReg, rbReg));

  if (imLen != 0)
    goto EmitImm;
  else
    goto EmitDone;

EmitX86M:
  ASMJIT_ASSERT(rmRel != nullptr);
  ASMJIT_ASSERT(rmRel->getOp() == Operand::kOpMem);
  rmInfo = x86MemInfo[rmRel->as<X86Mem>().getBaseIndexType()];

  // GP instructions have never compressed displacement specified.
  ASMJIT_ASSERT((opCode & X86Inst::kOpCode_CDSHL_Mask) == 0);

  // Segment-override prefix.
  if (rmRel->as<X86Mem>().hasSegment())
    EMIT_BYTE(x86SegmentPrefix[rmRel->as<X86Mem>().getSegmentId()]);

  // Address-override prefix.
  if (rmInfo & _getAddressOverrideMask())
    EMIT_BYTE(0x67);

  // Mandatory instruction prefix.
  EMIT_PP(opCode);

  rbReg = rmRel->as<X86Mem>().getBaseId();
  rxReg = rmRel->as<X86Mem>().getIndexId();

  // REX prefix (64-bit only).
  {
    uint32_t rex;

    rex  = (rbReg >> 3) & 0x01; // REX.B (0x01).
    rex |= (rxReg >> 2) & 0x02; // REX.X (0x02).
    rex |= (opReg >> 1) & 0x04; // REX.R (0x04).

    rex &= rmInfo;
    rex |= x86ExtractREX(opCode, options);

    if (rex) {
      if (options & X86Inst::_kOptionInvalidRex)
        goto InvalidRexPrefix;
      EMIT_BYTE(rex | kX86ByteRex);
      opReg &= 0x07;
    }
  }

  // Instruction opcodes.
  EMIT_MM_OP(opCode);
  // ... Fall through ...

  // --------------------------------------------------------------------------
  // [Emit - MOD/SIB]
  // --------------------------------------------------------------------------

EmitModSib:
  if (!(rmInfo & (kX86MemInfo_Index | kX86MemInfo_67H_X86))) {
    // ==========|> [BASE + DISP8|DISP32].
    if (rmInfo & kX86MemInfo_BaseGp) {
      rbReg &= 0x7;
      relOffset = rmRel->as<X86Mem>().getOffsetLo32();

      uint32_t mod = x86EncodeMod(0, opReg, rbReg);
      if (rbReg == X86Gp::kIdSp) {
        // [XSP|R12].
        if (relOffset == 0) {
          EMIT_BYTE(mod);
          EMIT_BYTE(x86EncodeSib(0, 4, 4));
        }
        // [XSP|R12 + DISP8|DISP32].
        else {
          uint32_t cdShift = (opCode & X86Inst::kOpCode_CDSHL_Mask) >> X86Inst::kOpCode_CDSHL_Shift;
          int32_t cdOffset = relOffset >> cdShift;

          if (Utils::isInt8(cdOffset) && relOffset == (cdOffset << cdShift)) {
            EMIT_BYTE(mod + 0x40); // <- MOD(1, opReg, rbReg).
            EMIT_BYTE(x86EncodeSib(0, 4, 4));
            EMIT_BYTE(cdOffset & 0xFF);
          }
          else {
            EMIT_BYTE(mod + 0x80); // <- MOD(2, opReg, rbReg).
            EMIT_BYTE(x86EncodeSib(0, 4, 4));
            EMIT_32(relOffset);
          }
        }
      }
      else if (rbReg != X86Gp::kIdBp && relOffset == 0) {
        // [BASE].
        EMIT_BYTE(mod);
      }
      else {
        // [BASE + DISP8|DISP32].
        uint32_t cdShift = (opCode & X86Inst::kOpCode_CDSHL_Mask) >> X86Inst::kOpCode_CDSHL_Shift;
        int32_t cdOffset = relOffset >> cdShift;

        if (Utils::isInt8(cdOffset) && relOffset == (cdOffset << cdShift)) {
          EMIT_BYTE(mod + 0x40);
          EMIT_BYTE(cdOffset & 0xFF);
        }
        else {
          EMIT_BYTE(mod + 0x80);
          EMIT_32(relOffset);
        }
      }
    }
    // ==========|> [ABSOLUTE | DISP32].
    else if (!(rmInfo & (kX86MemInfo_BaseLabel | kX86MemInfo_BaseRip))) {
      if (is32Bit()) {
        relOffset = rmRel->as<X86Mem>().getOffsetLo32();
        EMIT_BYTE(x86EncodeMod(0, opReg, 5));
        EMIT_32(relOffset);
      }
      else {
        uint64_t baseAddress = getCodeInfo().getBaseAddress();
        relOffset = rmRel->as<X86Mem>().getOffsetLo32();

        // Prefer absolute addressing mode if FS|GS segment override is present.
        bool absoluteValid = rmRel->as<X86Mem>().getOffsetHi32() == (relOffset >> 31);
        bool preferAbsolute = (rmRel->as<X86Mem>().getSegmentId() >= X86Seg::kIdFs) || rmRel->as<X86Mem>().isAbs();

        // If we know the base address and the memory operand points to an
        // absolute address it's possible to calculate REL32 that can be
        // be used as [RIP+REL32] in 64-bit mode.
        if (baseAddress != Globals::kNoBaseAddress && !preferAbsolute) {
          const uint32_t kModRel32Size = 5;
          uint64_t rip64 = baseAddress +
            static_cast<uint64_t>((uintptr_t)(cursor - _bufferData)) + imLen + kModRel32Size;

          uint64_t rel64 = static_cast<uint64_t>(rmRel->as<X86Mem>().getOffset()) - rip64;
          if (Utils::isInt32(static_cast<int64_t>(rel64))) {
            EMIT_BYTE(x86EncodeMod(0, opReg, 5));
            EMIT_32(static_cast<uint32_t>(rel64 & 0xFFFFFFFFU));
            if (imLen != 0)
              goto EmitImm;
            else
              goto EmitDone;
          }
        }

        if (ASMJIT_UNLIKELY(!absoluteValid))
          goto InvalidAddress64Bit;

        EMIT_BYTE(x86EncodeMod(0, opReg, 4));
        EMIT_BYTE(x86EncodeSib(0, 4, 5));
        EMIT_32(relOffset);
      }
    }
    // ==========|> [LABEL|RIP + DISP32]
    else {
      EMIT_BYTE(x86EncodeMod(0, opReg, 5));

      if (is32Bit()) {
EmitModSib_LabelRip_X86:
        if (ASMJIT_UNLIKELY(_code->_relocations.willGrow(&_code->_baseHeap) != kErrorOk))
          goto NoHeapMemory;

        relOffset = rmRel->as<X86Mem>().getOffsetLo32();
        if (rmInfo & kX86MemInfo_BaseLabel) {
          // [LABEL->ABS].
          label = _code->getLabelEntry(rmRel->as<X86Mem>().getBaseId());
          if (!label) goto InvalidLabel;

          err = _code->newRelocEntry(&re, RelocEntry::kTypeRelToAbs, 4);
          if (ASMJIT_UNLIKELY(err)) goto Failed;

          re->_sourceSectionId = _section->getId();
          re->_sourceOffset = static_cast<uint64_t>((uintptr_t)(cursor - _bufferData));
          re->_data = static_cast<int64_t>(relOffset);

          if (label->isBound()) {
            // Bound label.
            re->_data += static_cast<uint64_t>(label->getOffset());
            EMIT_32(0);
          }
          else {
            // Non-bound label.
            relOffset = -4 - imLen;
            relSize = 4;
            goto EmitRel;
          }
        }
        else {
          // [RIP->ABS].
          err = _code->newRelocEntry(&re, RelocEntry::kTypeRelToAbs, 4);
          if (ASMJIT_UNLIKELY(err)) goto Failed;

          re->_sourceSectionId = _section->getId();
          re->_sourceOffset = static_cast<uint64_t>((uintptr_t)(cursor - _bufferData));
          re->_data = re->_sourceOffset + static_cast<uint64_t>(static_cast<int64_t>(relOffset));
          EMIT_32(0);
        }
      }
      else {
        relOffset = rmRel->as<X86Mem>().getOffsetLo32();
        if (rmInfo & kX86MemInfo_BaseLabel) {
          // [RIP].
          label = _code->getLabelEntry(rmRel->as<X86Mem>().getBaseId());
          if (!label) goto InvalidLabel;

          relOffset -= (4 + imLen);
          if (label->isBound()) {
            // Bound label.
            relOffset += label->getOffset() - static_cast<int32_t>((intptr_t)(cursor - _bufferData));
            EMIT_32(static_cast<int32_t>(relOffset));
          }
          else {
            // Non-bound label.
            relSize = 4;
            goto EmitRel;
          }
        }
        else {
          // [RIP].
          EMIT_32(static_cast<int32_t>(relOffset));
        }
      }
    }
  }
  else if (!(rmInfo & kX86MemInfo_67H_X86)) {
    // ESP|RSP can't be used as INDEX in pure SIB mode, however, VSIB mode
    // allows XMM4|YMM4|ZMM4 (that's why the check is before the label).
    if (ASMJIT_UNLIKELY(rxReg == X86Gp::kIdSp))
      goto InvalidAddressIndex;

EmitModVSib:
    rxReg &= 0x7;

    // ==========|> [BASE + INDEX + DISP8|DISP32].
    if (rmInfo & kX86MemInfo_BaseGp) {
      rbReg &= 0x7;
      relOffset = rmRel->as<X86Mem>().getOffsetLo32();

      uint32_t mod = x86EncodeMod(0, opReg, 4);
      uint32_t sib = x86EncodeSib(rmRel->as<X86Mem>().getShift(), rxReg, rbReg);

      if (relOffset == 0 && rbReg != X86Gp::kIdBp) {
        // [BASE + INDEX << SHIFT].
        EMIT_BYTE(mod);
        EMIT_BYTE(sib);
      }
      else {
        uint32_t cdShift = (opCode & X86Inst::kOpCode_CDSHL_Mask) >> X86Inst::kOpCode_CDSHL_Shift;
        int32_t cdOffset = relOffset >> cdShift;

        if (Utils::isInt8(cdOffset) && relOffset == (cdOffset << cdShift)) {
          // [BASE + INDEX << SHIFT + DISP8].
          EMIT_BYTE(mod + 0x40); // <- MOD(1, opReg, 4).
          EMIT_BYTE(sib);
          EMIT_BYTE(cdOffset);
        }
        else {
          // [BASE + INDEX << SHIFT + DISP32].
          EMIT_BYTE(mod + 0x80); // <- MOD(2, opReg, 4).
          EMIT_BYTE(sib);
          EMIT_32(relOffset);
        }
      }
    }
    // ==========|> [INDEX + DISP32].
    else if (!(rmInfo & (kX86MemInfo_BaseLabel | kX86MemInfo_BaseRip))) {
      // [INDEX << SHIFT + DISP32].
      EMIT_BYTE(x86EncodeMod(0, opReg, 4));
      EMIT_BYTE(x86EncodeSib(rmRel->as<X86Mem>().getShift(), rxReg, 5));

      relOffset = rmRel->as<X86Mem>().getOffsetLo32();
      EMIT_32(relOffset);
    }
    // ==========|> [LABEL|RIP + INDEX + DISP32].
    else {
      if (is32Bit()) {
        EMIT_BYTE(x86EncodeMod(0, opReg, 4));
        EMIT_BYTE(x86EncodeSib(rmRel->as<X86Mem>().getShift(), rxReg, 5));
        goto EmitModSib_LabelRip_X86;
      }
      else {
        // NOTE: This also handles VSIB+RIP, which is not allowed in 64-bit mode.
        goto InvalidAddress;
      }
    }
  }
  else {
    // 16-bit address mode (32-bit mode with 67 override prefix).
    relOffset = (static_cast<int32_t>(rmRel->as<X86Mem>().getOffsetLo32()) << 16) >> 16;

    // NOTE: 16-bit addresses don't use SIB byte and their encoding differs. We
    // use a table-based approach to calculate the proper MOD byte as it's easier.
    // Also, not all BASE [+ INDEX] combinations are supported in 16-bit mode, so
    // this may fail.
    const uint32_t kBaseGpIdx = (kX86MemInfo_BaseGp | kX86MemInfo_Index);

    if (rmInfo & kBaseGpIdx) {
      // ==========|> [BASE + INDEX + DISP16].
      uint32_t mod;

      rbReg &= 0x7;
      rxReg &= 0x7;

      if ((rmInfo & kBaseGpIdx) == kBaseGpIdx) {
        uint32_t shf = rmRel->as<X86Mem>().getShift();
        if (ASMJIT_UNLIKELY(shf != 0))
          goto InvalidAddress;
        mod = x86Mod16BaseIndexTable[(rbReg << 3) + rxReg];
      }
      else {
        if (rmInfo & kX86MemInfo_Index)
          rbReg = rxReg;
        mod = x86Mod16BaseTable[rbReg];
      }

      if (ASMJIT_UNLIKELY(mod == 0xFF))
        goto InvalidAddress;

      mod += opReg << 3;
      if (relOffset == 0 && mod != 0x06) {
        EMIT_BYTE(mod);
      }
      else if (Utils::isInt8(relOffset)) {
        EMIT_BYTE(mod + 0x40);
        EMIT_BYTE(relOffset);
      }
      else {
        EMIT_BYTE(mod + 0x80);
        EMIT_16(relOffset);
      }
    }
    else {
      // Not supported in 16-bit addresses.
      if (rmInfo & (kX86MemInfo_BaseRip | kX86MemInfo_BaseLabel))
        goto InvalidAddress;

      // ==========|> [DISP16].
      EMIT_BYTE(opReg | 0x06);
      EMIT_16(relOffset);
    }
  }

  if (imLen != 0)
    goto EmitImm;
  else
    goto EmitDone;

  // --------------------------------------------------------------------------
  // [Emit - FPU]
  // --------------------------------------------------------------------------

EmitFpuOp:
  // Mandatory instruction prefix.
  EMIT_PP(opCode);

  // FPU instructions consist of two opcodes.
  EMIT_BYTE(opCode >> X86Inst::kOpCode_FPU_2B_Shift);
  EMIT_BYTE(opCode);
  goto EmitDone;

  // --------------------------------------------------------------------------
  // [Emit - VEX / EVEX]
  // --------------------------------------------------------------------------

EmitVexEvexOp:
  {
    // These don't use immediate.
    ASMJIT_ASSERT(imLen == 0);

    // Only 'vzeroall' and 'vzeroupper' instructions use this encoding, they
    // don't define 'W' to be '1' so we can just check the 'mmmmm' field. Both
    // functions can encode by using VEV2 prefix so VEV3 is basically only used
    // when forced from outside.
    ASMJIT_ASSERT((opCode & X86Inst::kOpCode_W) == 0);

    uint32_t x = ((opCode & X86Inst::kOpCode_MM_Mask   ) >> (X86Inst::kOpCode_MM_Shift     )) |
                 ((opCode & X86Inst::kOpCode_LL_Mask   ) >> (X86Inst::kOpCode_LL_Shift - 10)) |
                 ((opCode & X86Inst::kOpCode_PP_VEXMask) >> (X86Inst::kOpCode_PP_Shift -  8)) |
                 ((options & X86Inst::kOptionVex3      ) >> (X86Inst::kOpCode_MM_Shift     )) ;
    if (x & 0x04U) {
      x  = (x & (0x4 ^ 0xFFFF)) << 8;                    // [00000000|00000Lpp|0000m0mm|00000000].
      x ^= (kX86ByteVex3) |                              // [........|00000Lpp|0000m0mm|__VEX3__].
           (0x07U  << 13) |                              // [........|00000Lpp|1110m0mm|__VEX3__].
           (0x0FU  << 19) |                              // [........|01111Lpp|1110m0mm|__VEX3__].
           (opCode << 24) ;                              // [_OPCODE_|01111Lpp|1110m0mm|__VEX3__].

      EMIT_32(x);
      goto EmitDone;
    }
    else {
      x = ((x >> 8) ^ x) ^ 0xF9;
      EMIT_BYTE(kX86ByteVex2);
      EMIT_BYTE(x);
      EMIT_BYTE(opCode);
      goto EmitDone;
    }
  }

EmitVexEvexR:
  {
    // VEX instructions use only 0-1 BYTE immediate.
    ASMJIT_ASSERT(imLen <= 1);

    // Construct `x` - a complete EVEX|VEX prefix.
    uint32_t x = ((opReg << 4) & 0xF980U) |              // [........|........|Vvvvv..R|R.......].
                 ((rbReg << 2) & 0x0060U) |              // [........|........|........|.BB.....].
                 (x86ExtractLLMM(opCode, options)) |     // [........|.LL.....|Vvvvv..R|RBBmmmmm].
                 (_extraReg.getId() << 16);              // [........|.LL..aaa|Vvvvv..R|RBBmmmmm].
    opReg &= 0x7;

    // Mark invalid VEX (force EVEX) case:               // [@.......|.LL..aaa|Vvvvv..R|RBBmmmmm].
    x |= (~commonData->getFlags() & X86Inst::kFlagVex) << (31 - Utils::firstBitOfT<X86Inst::kFlagVex>());

    // Handle AVX512 options by a single branch.
    const uint32_t kAvx512Options = X86Inst::kOptionZMask   |
                                    X86Inst::kOption1ToX    |
                                    X86Inst::kOptionSAE     |
                                    X86Inst::kOptionER      ;
    if (options & kAvx512Options) {
      // Memory broadcast without a memory operand is invalid.
      if (ASMJIT_UNLIKELY(options & X86Inst::kOption1ToX))
        goto InvalidBroadcast;

      // TODO: {sae} and {er}
      x |= options & X86Inst::kOptionZMask;              // [@.......|zLL..aaa|Vvvvv..R|RBBmmmmm].
    }

    // Check if EVEX is required by checking bits in `x` :  [@.......|xx...xxx|x......x|.x.x....].
    if (x & 0x80C78150U) {
      uint32_t y = ((x << 4) & 0x00080000U) |            // [@.......|....V...|........|........].
                   ((x >> 4) & 0x00000010U) ;            // [@.......|....V...|........|...R....].
      x  = (x & 0x00FF78E3U) | y;                        // [........|zLL.Vaaa|0vvvv000|RBBR00mm].
      x  = (x << 8) |                                    // [zLL.Vaaa|0vvvv000|RBBR00mm|00000000].
           ((opCode >> kSHR_W_PP) & 0x00830000U) |       // [zLL.Vaaa|Wvvvv0pp|RBBR00mm|00000000].
           ((opCode >> kSHR_W_EW) & 0x00800000U) ;       // [zLL.Vaaa|Wvvvv0pp|RBBR00mm|00000000] (added EVEX.W).
                                                         //      _     ____    ____
      x ^= 0x087CF000U | kX86ByteEvex;                   // [zLL.Vaaa|Wvvvv1pp|RBBR00mm|01100010].

      EMIT_32(x);
      EMIT_BYTE(opCode);

      rbReg &= 0x7;
      EMIT_BYTE(x86EncodeMod(3, opReg, rbReg));

      if (imLen == 0) goto EmitDone;
      EMIT_BYTE(imVal & 0xFF);
      goto EmitDone;
    }

    // Not EVEX, prepare `x` for VEX2 or VEX3:          x = [........|00L00000|0vvvv000|R0B0mmmm].
    x |= ((opCode >> (kSHR_W_PP + 8)) & 0x8300U) |       // [00000000|00L00000|Wvvvv0pp|R0B0mmmm].
         ((x      >> 11             ) & 0x0400U) ;       // [00000000|00L00000|WvvvvLpp|R0B0mmmm].

    // Check if VEX3 is required / forced:                  [........|........|x.......|..x..x..].
    if (x & 0x0008024U) {
      uint32_t xorMsk = x86VEXPrefix[x & 0xF] | (opCode << 24);

      // Clear 'FORCE-VEX3' bit and all high bits.
      x  = (x & (0x4 ^ 0xFFFF)) << 8;                    // [00000000|WvvvvLpp|R0B0m0mm|00000000].
                                                         //            ____    _ _
      x ^= xorMsk;                                       // [_OPCODE_|WvvvvLpp|R1Bmmmmm|VEX3|XOP].
      EMIT_32(x);

      rbReg &= 0x7;
      EMIT_BYTE(x86EncodeMod(3, opReg, rbReg));

      if (imLen == 0) goto EmitDone;
      EMIT_BYTE(imVal & 0xFF);
      goto EmitDone;
    }
    else {
      // 'mmmmm' must be '00001'.
      ASMJIT_ASSERT((x & 0x1F) == 0x01);

      x = ((x >> 8) ^ x) ^ 0xF9;
      EMIT_BYTE(kX86ByteVex2);
      EMIT_BYTE(x);
      EMIT_BYTE(opCode);

      rbReg &= 0x7;
      EMIT_BYTE(x86EncodeMod(3, opReg, rbReg));

      if (imLen == 0) goto EmitDone;
      EMIT_BYTE(imVal & 0xFF);
      goto EmitDone;
    }
  }

EmitVexEvexM:
  ASMJIT_ASSERT(rmRel != nullptr);
  ASMJIT_ASSERT(rmRel->getOp() == Operand::kOpMem);
  rmInfo = x86MemInfo[rmRel->as<X86Mem>().getBaseIndexType()];

  // Segment-override prefix.
  if (rmRel->as<X86Mem>().hasSegment())
    EMIT_BYTE(x86SegmentPrefix[rmRel->as<X86Mem>().getSegmentId()]);

  // Address-override prefix.
  if (rmInfo & _getAddressOverrideMask())
    EMIT_BYTE(0x67);

  rbReg = rmRel->as<X86Mem>().hasBaseReg()  ? rmRel->as<X86Mem>().getBaseId()  : uint32_t(0);
  rxReg = rmRel->as<X86Mem>().hasIndexReg() ? rmRel->as<X86Mem>().getIndexId() : uint32_t(0);

  {
    // VEX instructions use only 0-1 BYTE immediate.
    ASMJIT_ASSERT(imLen <= 1);

    // Construct `x` - a complete EVEX|VEX prefix.
    uint32_t x = ((opReg << 4 ) & 0x0000F980U) |         // [........|........|Vvvvv..R|R.......].
                 ((rxReg << 3 ) & 0x00000040U) |         // [........|........|........|.X......].
                 ((rxReg << 15) & 0x00080000U) |         // [........|....X...|........|........].
                 ((rbReg << 2 ) & 0x00000020U) |         // [........|........|........|..B.....].
                 (x86ExtractLLMM(opCode, options)) |     // [........|.LL.X...|Vvvvv..R|RXBmmmmm].
                 (_extraReg.getId() << 16)         ;     // [........|.LL.Xaaa|Vvvvv..R|RXBmmmmm].
    opReg &= 0x07U;

    // Mark invalid VEX (force EVEX) case:               // [@.......|.LL.Xaaa|Vvvvv..R|RXBmmmmm].
    x |= (~commonData->getFlags() & X86Inst::kFlagVex) << (31 - Utils::firstBitOfT<X86Inst::kFlagVex>());

    // Handle AVX512 options by a single branch.
    const uint32_t kAvx512Options = X86Inst::kOption1ToX    |
                                    X86Inst::kOptionZMask   |
                                    X86Inst::kOptionSAE     |
                                    X86Inst::kOptionER      ;
    if (options & kAvx512Options) {
      // {er} and {sae} are both invalid if memory operand is used.
      if (ASMJIT_UNLIKELY(options & (X86Inst::kOptionSAE | X86Inst::kOptionER)))
        goto InvalidEROrSAE;

      x |= options & (X86Inst::kOption1ToX |             // [@.......|.LLbXaaa|Vvvvv..R|RXBmmmmm].
                      X86Inst::kOptionZMask);            // [@.......|zLLbXaaa|Vvvvv..R|RXBmmmmm].
    }

    // Check if EVEX is required by checking bits in `x` :  [@.......|xx.xxxxx|x......x|...x....].
    if (x & 0x80DF8110U) {
      uint32_t y = ((x << 4) & 0x00080000U) |            // [@.......|....V...|........|........].
                   ((x >> 4) & 0x00000010U) ;            // [@.......|....V...|........|...R....].
      x  = (x & 0x00FF78E3U) | y;                        // [........|zLLbVaaa|0vvvv000|RXBR00mm].
      x  = (x << 8) |                                    // [zLLbVaaa|0vvvv000|RBBR00mm|00000000].
           ((opCode >> kSHR_W_PP) & 0x00830000U) |       // [zLLbVaaa|Wvvvv0pp|RBBR00mm|00000000].
           ((opCode >> kSHR_W_EW) & 0x00800000U) ;       // [zLLbVaaa|Wvvvv0pp|RBBR00mm|00000000] (added EVEX.W).
                                                         //      _     ____    ____
      x ^= 0x087CF000U | kX86ByteEvex;                   // [zLLbVaaa|Wvvvv1pp|RBBR00mm|01100010].

      EMIT_32(x);
      EMIT_BYTE(opCode);

      if (opCode & 0x10000000U) {
        // Broadcast, change the compressed displacement scale to either x4 (SHL 2) or x8 (SHL 3)
        // depending on instruction's W. If 'W' is 1 'SHL' must be 3, otherwise it must be 2.
        opCode &=~static_cast<uint32_t>(X86Inst::kOpCode_CDSHL_Mask);
        opCode |= ((x & 0x00800000U) ? 3 : 2) << X86Inst::kOpCode_CDSHL_Shift;
      }
      else {
        // Add the compressed displacement 'SHF' to the opcode based on 'TTWLL'.
        uint32_t TTWLL = ((opCode >> (X86Inst::kOpCode_CDTT_Shift - 3)) & 0x18) +
                         ((opCode >> (X86Inst::kOpCode_W_Shift    - 2)) & 0x04) +
                         ((x >> 29) & 0x3);
        opCode += x86CDisp8SHL[TTWLL];
      }
    }
    else {
      // Not EVEX, prepare `x` for VEX2 or VEX3:        x = [........|00L00000|0vvvv000|RXB0mmmm].
      x |= ((opCode >> (kSHR_W_PP + 8)) & 0x8300U) |     // [00000000|00L00000|Wvvvv0pp|RXB0mmmm].
           ((x      >> 11             ) & 0x0400U) ;     // [00000000|00L00000|WvvvvLpp|RXB0mmmm].

      // Clear a possible CDisp specified by EVEX.
      opCode &= ~X86Inst::kOpCode_CDSHL_Mask;

      // Check if VEX3 is required / forced:                [........|........|x.......|.xx..x..].
      if (x & 0x0008064U) {
        uint32_t xorMsk = x86VEXPrefix[x & 0xF] | (opCode << 24);

        // Clear 'FORCE-VEX3' bit and all high bits.
        x  = (x & (0x4 ^ 0xFFFF)) << 8;                  // [00000000|WvvvvLpp|RXB0m0mm|00000000].
                                                         //            ____    ___
        x ^= xorMsk;                                     // [_OPCODE_|WvvvvLpp|RXBmmmmm|VEX3_XOP].
        EMIT_32(x);
      }
      else {
        // 'mmmmm' must be '00001'.
        ASMJIT_ASSERT((x & 0x1F) == 0x01);

        x = ((x >> 8) ^ x) ^ 0xF9;
        EMIT_BYTE(kX86ByteVex2);
        EMIT_BYTE(x);
        EMIT_BYTE(opCode);
      }
    }
  }

  // MOD|SIB address.
  if (!commonData->hasFlag(X86Inst::kFlagVsib))
    goto EmitModSib;

  // MOD|VSIB address without INDEX is invalid.
  if (rmInfo & kX86MemInfo_Index)
    goto EmitModVSib;
  goto InvalidInstruction;

  // --------------------------------------------------------------------------
  // [Emit - Jmp/Jcc/Call]
  // --------------------------------------------------------------------------

  // TODO: Should be adjusted after the support for multiple sections feature is added.
EmitJmpCall:
  {
    // Emit REX prefix if asked for (64-bit only).
    uint32_t rex = x86ExtractREX(opCode, options);
    if (rex) {
      if (options & X86Inst::_kOptionInvalidRex)
        goto InvalidRexPrefix;
      EMIT_BYTE(rex | kX86ByteRex);
    }

    uint64_t ip = static_cast<uint64_t>((intptr_t)(cursor - _bufferData));
    uint32_t rel32 = 0;
    uint32_t opCode8 = commonData->getAltOpCode();

    uint32_t inst8Size  = 1 + 1; //          OPCODE + REL8 .
    uint32_t inst32Size = 1 + 4; // [PREFIX] OPCODE + REL32.

    // Jcc instructions with 32-bit displacement use 0x0F prefix,
    // other instructions don't. No other prefixes are used by X86.
    ASMJIT_ASSERT((opCode8 & X86Inst::kOpCode_MM_Mask) == 0);
    ASMJIT_ASSERT((opCode  & X86Inst::kOpCode_MM_Mask) == 0 ||
                  (opCode  & X86Inst::kOpCode_MM_Mask) == X86Inst::kOpCode_MM_0F);

    // Only one of these should be used at the same time.
    inst32Size += static_cast<uint32_t>(opReg != 0);
    inst32Size += static_cast<uint32_t>((opCode & X86Inst::kOpCode_MM_Mask) == X86Inst::kOpCode_MM_0F);

    if (rmRel->isLabel()) {
      label = _code->getLabelEntry(rmRel->as<Label>());
      if (!label) goto InvalidLabel;

      if (label->isBound()) {
        // Bound label.
        rel32 = static_cast<uint32_t>((static_cast<uint64_t>(label->getOffset()) - ip - inst32Size) & 0xFFFFFFFFU);
        goto EmitJmpCallRel;
      }
      else {
        // Non-bound label.
        if (opCode8 && (!opCode || (options & X86Inst::kOptionShortForm))) {
          EMIT_BYTE(opCode8);
          relOffset = -1;
          relSize = 1;
          goto EmitRel;
        }
        else {
          // Refuse also 'short' prefix, if specified.
          if (ASMJIT_UNLIKELY(!opCode || (options & X86Inst::kOptionShortForm) != 0))
            goto InvalidDisplacement;

          // Emit [PREFIX] OPCODE [/X] <DISP32>.
          if (opCode & X86Inst::kOpCode_MM_Mask)
            EMIT_BYTE(0x0F);

          EMIT_BYTE(opCode);
          if (opReg)
            EMIT_BYTE(x86EncodeMod(3, opReg, 0));

          relOffset = -4;
          relSize = 4;
          goto EmitRel;
        }
      }
    }

    if (rmRel->isImm()) {
      uint64_t baseAddress = getCodeInfo().getBaseAddress();
      uint64_t jumpAddress = rmRel->as<Imm>().getUInt64();

      // If the base-address is known calculate a relative displacement and
      // check if it fits in 32 bits (which is always true in 32-bit mode).
      // Emit relative displacement as it was a bound label if all checks ok.
      if (baseAddress != Globals::kNoBaseAddress) {
        uint64_t rel64 = jumpAddress - (ip + baseAddress) - inst32Size;
        if (getArchType() == ArchInfo::kTypeX86 || Utils::isInt32(static_cast<int64_t>(rel64))) {
          rel32 = static_cast<uint32_t>(rel64 & 0xFFFFFFFFU);
          goto EmitJmpCallRel;
        }
        else {
          // Relative displacement exceeds 32-bits - relocator can only
          // insert trampoline for jmp/call, but not for jcc/jecxz.
          if (ASMJIT_UNLIKELY(!x86IsJmpOrCall(instId)))
            goto InvalidDisplacement;
        }
      }

      if (ASMJIT_UNLIKELY(_code->_relocations.willGrow(&_code->_baseHeap) != kErrorOk))
        goto NoHeapMemory;

      err = _code->newRelocEntry(&re, RelocEntry::kTypeAbsToRel, 0);
      if (ASMJIT_UNLIKELY(err)) goto Failed;

      re->_sourceSectionId = _section->getId();
      re->_data = static_cast<int64_t>(jumpAddress);

      if (ASMJIT_LIKELY(opCode)) {
        // 64-bit: Emit REX prefix so the instruction can be patched later.
        // REX prefix does nothing if not patched, but allows to patch the
        // instruction to use MOD/M and to point to a memory where the final
        // 64-bit address is stored.
        re->_size = 4;
        re->_sourceOffset = ip + inst32Size - 4;

        if (getArchType() != ArchInfo::kTypeX86 && x86IsJmpOrCall(instId)) {
          if (!rex) {
            re->_sourceOffset++;
            EMIT_BYTE(kX86ByteRex);
          }

          re->_type = RelocEntry::kTypeTrampoline;
          _code->_trampolinesSize += 8;
        }

        // Emit [PREFIX] OPCODE [/X] DISP32.
        if (opCode & X86Inst::kOpCode_MM_Mask)
          EMIT_BYTE(0x0F);

        EMIT_BYTE(opCode);
        if (opReg)
          EMIT_BYTE(x86EncodeMod(3, opReg, 0));

        EMIT_32(0);
      }
      else {
        re->_size = 1;
        re->_sourceOffset = ip + inst8Size - 1;

        // Emit OPCODE + DISP8.
        EMIT_BYTE(opCode8);
        EMIT_BYTE(0);
      }
      goto EmitDone;
    }

    // Not Label|Imm -> Invalid.
    goto InvalidInstruction;

    // Emit jmp/call with relative displacement known at assembly-time. Decide
    // between 8-bit and 32-bit displacement encoding. Some instructions only
    // allow either 8-bit or 32-bit encoding, others allow both encodings.
EmitJmpCallRel:
    if (Utils::isInt8(static_cast<int32_t>(rel32 + inst32Size - inst8Size)) && opCode8 && !(options & X86Inst::kOptionLongForm)) {
      options |= X86Inst::kOptionShortForm;
      EMIT_BYTE(opCode8);
      EMIT_BYTE(rel32 + inst32Size - inst8Size);
      goto EmitDone;
    }
    else {
      if (ASMJIT_UNLIKELY(!opCode || (options & X86Inst::kOptionShortForm) != 0))
        goto InvalidDisplacement;

      options &= ~X86Inst::kOptionShortForm;
      if (opCode & X86Inst::kOpCode_MM_Mask)
        EMIT_BYTE(0x0F);

      EMIT_BYTE(opCode);
      if (opReg)
        EMIT_BYTE(x86EncodeMod(3, opReg, 0));

      EMIT_32(rel32);
      goto EmitDone;
    }
  }

  // --------------------------------------------------------------------------
  // [Emit - Relative]
  // --------------------------------------------------------------------------

EmitRel:
  {
    ASMJIT_ASSERT(!label->isBound());
    ASMJIT_ASSERT(relSize == 1 || relSize == 4);

    // Chain with label.
    size_t offset = (size_t)(cursor - _bufferData);
    LabelLink* link = _code->newLabelLink(label, _section->getId(), offset, relOffset);

    if (ASMJIT_UNLIKELY(!link))
      goto NoHeapMemory;

    if (re)
      link->relocId = re->getId();

    // Emit label size as dummy data.
    if (relSize == 1)
      EMIT_BYTE(0x01);
    else // if (relSize == 4)
      EMIT_32(0x04040404);
  }

  if (imLen == 0)
    goto EmitDone;

  // --------------------------------------------------------------------------
  // [Emit - Immediate]
  // --------------------------------------------------------------------------

EmitImm:
  {
#if ASMJIT_ARCH_64BIT
    uint32_t i = imLen;
    uint64_t imm = static_cast<uint64_t>(imVal);
#else
    uint32_t i = imLen;
    uint32_t imm = static_cast<uint32_t>(imVal & 0xFFFFFFFFU);
#endif

    // Many instructions just use a single byte immediate, so make it fast.
    EMIT_BYTE(imm & 0xFFU); if (--i == 0) goto EmitDone;
    imm >>= 8;
    EMIT_BYTE(imm & 0xFFU); if (--i == 0) goto EmitDone;
    imm >>= 8;
    EMIT_BYTE(imm & 0xFFU); if (--i == 0) goto EmitDone;
    imm >>= 8;
    EMIT_BYTE(imm & 0xFFU); if (--i == 0) goto EmitDone;

    // Can be 1-4 or 8 bytes, this handles the remaining high DWORD of an 8-byte immediate.
    ASMJIT_ASSERT(i == 4);

#if ASMJIT_ARCH_64BIT
    imm >>= 8;
    EMIT_32(static_cast<uint32_t>(imm));
#else
    EMIT_32(static_cast<uint32_t>((static_cast<uint64_t>(imVal) >> 32) & 0xFFFFFFFFU));
#endif
  }

  // --------------------------------------------------------------------------
  // [Done]
  // --------------------------------------------------------------------------

EmitDone:
#if !defined(ASMJIT_DISABLE_LOGGING)
  // Logging is a performance hit anyway, so make it the unlikely case.
  if (ASMJIT_UNLIKELY(options & CodeEmitter::kOptionLoggingEnabled))
    _emitLog(instId, options, o0, o1, o2, o3, relSize, imLen, cursor);
#endif // !ASMJIT_DISABLE_LOGGING

  resetOptions();
  resetExtraReg();
  resetInlineComment();

  _bufferPtr = cursor;
  return kErrorOk;

  // --------------------------------------------------------------------------
  // [Error Cases]
  // --------------------------------------------------------------------------

#define ERROR_HANDLER(ERROR)                \
ERROR:                                      \
  err = DebugUtils::errored(kError##ERROR); \
  goto Failed;

ERROR_HANDLER(NoHeapMemory)
ERROR_HANDLER(InvalidArgument)
ERROR_HANDLER(InvalidLabel)
ERROR_HANDLER(InvalidInstruction)
ERROR_HANDLER(InvalidLockPrefix)
ERROR_HANDLER(InvalidXAcquirePrefix)
ERROR_HANDLER(InvalidXReleasePrefix)
ERROR_HANDLER(InvalidRepPrefix)
ERROR_HANDLER(InvalidRexPrefix)
ERROR_HANDLER(InvalidBroadcast)
ERROR_HANDLER(InvalidEROrSAE)
ERROR_HANDLER(InvalidAddress)
ERROR_HANDLER(InvalidAddressIndex)
ERROR_HANDLER(InvalidAddress64Bit)
ERROR_HANDLER(InvalidDisplacement)
ERROR_HANDLER(InvalidSegment)
ERROR_HANDLER(InvalidImmediate)
ERROR_HANDLER(OperandSizeMismatch)
ERROR_HANDLER(AmbiguousOperandSize)
ERROR_HANDLER(NotConsecutiveRegs)

Failed:
  return _emitFailed(err, instId, options, o0, o1, o2, o3);
}

// ============================================================================
// [asmjit::X86Assembler - Align]
// ============================================================================

Error X86Assembler::align(uint32_t mode, uint32_t alignment) {
#if !defined(ASMJIT_DISABLE_LOGGING)
  if (_globalOptions & kOptionLoggingEnabled)
    _code->_logger->logf("%s.align %u\n", _code->_logger->getIndentation(), alignment);
#endif // !ASMJIT_DISABLE_LOGGING

  if (mode >= kAlignCount)
    return setLastError(DebugUtils::errored(kErrorInvalidArgument));

  if (alignment <= 1)
    return kErrorOk;

  if (!Utils::isPowerOf2(alignment) || alignment > Globals::kMaxAlignment)
    return setLastError(DebugUtils::errored(kErrorInvalidArgument));

  uint32_t i = static_cast<uint32_t>(Utils::alignDiff<size_t>(getOffset(), alignment));
  if (i == 0)
    return kErrorOk;

  if (getRemainingSpace() < i) {
    Error err = _code->growBuffer(&_section->_buffer, i);
    if (ASMJIT_UNLIKELY(err)) return setLastError(err);
  }

  uint8_t* cursor = _bufferPtr;
  uint8_t pattern = 0x00;

  switch (mode) {
    case kAlignCode: {
      if (_globalHints & kHintOptimizedAlign) {
        // Intel 64 and IA-32 Architectures Software Developer's Manual - Volume 2B (NOP).
        enum { kMaxNopSize = 9 };

        static const uint8_t nopData[kMaxNopSize][kMaxNopSize] = {
          { 0x90 },
          { 0x66, 0x90 },
          { 0x0F, 0x1F, 0x00 },
          { 0x0F, 0x1F, 0x40, 0x00 },
          { 0x0F, 0x1F, 0x44, 0x00, 0x00 },
          { 0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00 },
          { 0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00 },
          { 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 },
          { 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }
        };

        do {
          uint32_t n = std::min<uint32_t>(i, kMaxNopSize);
          const uint8_t* src = nopData[n - 1];

          i -= n;
          do {
            EMIT_BYTE(*src++);
          } while (--n);
        } while (i);
      }

      pattern = 0x90;
      break;
    }

    case kAlignData:
      pattern = 0xCC;
      break;

    case kAlignZero:
      // Pattern already set to zero.
      break;
  }

  while (i) {
    EMIT_BYTE(pattern);
    i--;
  }

  _bufferPtr = cursor;
  return kErrorOk;
}

} // asmjit namespace
} // namespace PLMD

// [Api-End]
#include "./asmjit_apiend.h"

// [Guard]
#endif // ASMJIT_BUILD_X86
#pragma GCC diagnostic pop
#endif // __PLUMED_HAS_ASMJIT
