Merge branch 'master' of github.com:/pandegroup/openmm into feature/cuda-docker

libraries/asmjit/x86/x86instimpl.cpp

+// [AsmJit]
+// Complete x86/x64 JIT and Remote Assembler for C++.
+//
+// [License]
+// Zlib - See LICENSE.md file in the package.
+
+// [Export]
+#define ASMJIT_EXPORTS
+
+// [Dependencies]
+#include "../base/misc_p.h"
+#include "../base/utils.h"
+#include "../x86/x86instimpl_p.h"
+#include "../x86/x86operand.h"
+
+// [Api-Begin]
+#include "../asmjit_apibegin.h"
+
+namespace asmjit {
+
+// ============================================================================
+// [asmjit::X86InstImpl - Validate]
+// ============================================================================
+
+#if !defined(ASMJIT_DISABLE_VALIDATION)
+template<uint32_t RegType>
+struct X86OpTypeFromRegTypeT {
+  enum {
+    kValue = (RegType == X86Reg::kRegGpbLo) ? X86Inst::kOpGpbLo :
+             (RegType == X86Reg::kRegGpbHi) ? X86Inst::kOpGpbHi :
+             (RegType == X86Reg::kRegGpw  ) ? X86Inst::kOpGpw   :
+             (RegType == X86Reg::kRegGpd  ) ? X86Inst::kOpGpd   :
+             (RegType == X86Reg::kRegGpq  ) ? X86Inst::kOpGpq   :
+             (RegType == X86Reg::kRegXmm  ) ? X86Inst::kOpXmm   :
+             (RegType == X86Reg::kRegYmm  ) ? X86Inst::kOpYmm   :
+             (RegType == X86Reg::kRegZmm  ) ? X86Inst::kOpZmm   :
+             (RegType == X86Reg::kRegRip  ) ? X86Inst::kOpNone  :
+             (RegType == X86Reg::kRegSeg  ) ? X86Inst::kOpSeg   :
+             (RegType == X86Reg::kRegFp   ) ? X86Inst::kOpFp    :
+             (RegType == X86Reg::kRegMm   ) ? X86Inst::kOpMm    :
+             (RegType == X86Reg::kRegK    ) ? X86Inst::kOpK     :
+             (RegType == X86Reg::kRegBnd  ) ? X86Inst::kOpBnd   :
+             (RegType == X86Reg::kRegCr   ) ? X86Inst::kOpCr    :
+             (RegType == X86Reg::kRegDr   ) ? X86Inst::kOpDr    : X86Inst::kOpNone
+  };
+};
+
+template<uint32_t RegType>
+struct X86RegMaskFromRegTypeT {
+  enum {
+    kMask = (RegType == X86Reg::kRegGpbLo) ? 0x0000000FU :
+            (RegType == X86Reg::kRegGpbHi) ? 0x0000000FU :
+            (RegType == X86Reg::kRegGpw  ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegGpd  ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegGpq  ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegXmm  ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegYmm  ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegZmm  ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegRip  ) ? 0x00000001U :
+            (RegType == X86Reg::kRegSeg  ) ? 0x0000007EU : // [ES|CS|SS|DS|FS|GS]
+            (RegType == X86Reg::kRegFp   ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegMm   ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegK    ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegBnd  ) ? 0x0000000FU :
+            (RegType == X86Reg::kRegCr   ) ? 0x0000FFFFU :
+            (RegType == X86Reg::kRegDr   ) ? 0x000000FFU : X86Inst::kOpNone
+  };
+};
+
+template<uint32_t RegType>
+struct X64RegMaskFromRegTypeT {
+  enum {
+    kMask = (RegType == X86Reg::kRegGpbLo) ? 0x0000FFFFU :
+            (RegType == X86Reg::kRegGpbHi) ? 0x0000000FU :
+            (RegType == X86Reg::kRegGpw  ) ? 0x0000FFFFU :
+            (RegType == X86Reg::kRegGpd  ) ? 0x0000FFFFU :
+            (RegType == X86Reg::kRegGpq  ) ? 0x0000FFFFU :
+            (RegType == X86Reg::kRegXmm  ) ? 0xFFFFFFFFU :
+            (RegType == X86Reg::kRegYmm  ) ? 0xFFFFFFFFU :
+            (RegType == X86Reg::kRegZmm  ) ? 0xFFFFFFFFU :
+            (RegType == X86Reg::kRegRip  ) ? 0x00000001U :
+            (RegType == X86Reg::kRegSeg  ) ? 0x0000007EU : // [ES|CS|SS|DS|FS|GS]
+            (RegType == X86Reg::kRegFp   ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegMm   ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegK    ) ? 0x000000FFU :
+            (RegType == X86Reg::kRegBnd  ) ? 0x0000000FU :
+            (RegType == X86Reg::kRegCr   ) ? 0x0000FFFFU :
+            (RegType == X86Reg::kRegDr   ) ? 0x0000FFFFU : X86Inst::kOpNone
+  };
+};
+
+struct X86ValidationData {
+  //! Allowed registers by reg-type (X86::kReg...).
+  uint32_t allowedRegMask[X86Reg::kRegMax + 1];
+  uint32_t allowedMemBaseRegs;
+  uint32_t allowedMemIndexRegs;
+};
+
+static const uint32_t _x86OpFlagFromRegType[X86Reg::kRegMax + 1] = {
+  ASMJIT_TABLE_T_32(X86OpTypeFromRegTypeT, kValue, 0)
+};
+
+static const X86ValidationData _x86ValidationData = {
+  { ASMJIT_TABLE_T_32(X86RegMaskFromRegTypeT, kMask, 0) },
+  (1U << X86Reg::kRegGpw) | (1U << X86Reg::kRegGpd) | (1U << X86Reg::kRegRip) | (1U << Label::kLabelTag),
+  (1U << X86Reg::kRegGpw) | (1U << X86Reg::kRegGpd) | (1U << X86Reg::kRegXmm) | (1U << X86Reg::kRegYmm) | (1U << X86Reg::kRegZmm)
+};
+
+static const X86ValidationData _x64ValidationData = {
+  { ASMJIT_TABLE_T_32(X64RegMaskFromRegTypeT, kMask, 0) },
+  (1U << X86Reg::kRegGpd) | (1U << X86Reg::kRegGpq) | (1U << X86Reg::kRegRip) | (1U << Label::kLabelTag),
+  (1U << X86Reg::kRegGpd) | (1U << X86Reg::kRegGpq) | (1U << X86Reg::kRegXmm) | (1U << X86Reg::kRegYmm) | (1U << X86Reg::kRegZmm)
+};
+
+static ASMJIT_INLINE bool x86CheckOSig(const X86Inst::OSignature& op, const X86Inst::OSignature& ref, bool& immOutOfRange) noexcept {
+  // Fail if operand types are incompatible.
+  uint32_t opFlags = op.flags;
+  if ((opFlags & ref.flags) == 0) {
+    // Mark temporarily `immOutOfRange` so we can return a more descriptive error.
+    if ((opFlags & X86Inst::kOpAllImm) && (ref.flags & X86Inst::kOpAllImm)) {
+      immOutOfRange = true;
+      return true;
+    }
+
+    return false;
+  }
+
+  // Fail if memory specific flags and sizes are incompatibles.
+  uint32_t opMemFlags = op.memFlags;
+  if (opMemFlags != 0) {
+    uint32_t refMemFlags = ref.memFlags;
+    if ((refMemFlags & opMemFlags) == 0)
+      return false;
+
+    if ((refMemFlags & X86Inst::kMemOpBaseOnly) && !(opMemFlags & X86Inst::kMemOpBaseOnly))
+      return false;
+  }
+
+  // Specific register index.
+  if (opFlags & X86Inst::kOpAllRegs) {
+    uint32_t refRegMask = ref.regMask;
+    if (refRegMask && !(op.regMask & refRegMask))
+      return false;
+  }
+
+  return true;
+}
+
+ASMJIT_FAVOR_SIZE Error X86InstImpl::validate(uint32_t archType, const Inst::Detail& detail, const Operand_* operands, uint32_t count) noexcept {
+  uint32_t i;
+  uint32_t archMask;
+  const X86ValidationData* vd;
+
+  if (!ArchInfo::isX86Family(archType))
+    return DebugUtils::errored(kErrorInvalidArch);
+
+  if (archType == ArchInfo::kTypeX86) {
+    vd = &_x86ValidationData;
+    archMask = X86Inst::kArchMaskX86;
+  }
+  else {
+    vd = &_x64ValidationData;
+    archMask = X86Inst::kArchMaskX64;
+  }
+
+  // Get the instruction data.
+  uint32_t instId = detail.instId;
+  uint32_t options = detail.options;
+
+  if (ASMJIT_UNLIKELY(instId >= X86Inst::_kIdCount))
+    return DebugUtils::errored(kErrorInvalidArgument);
+
+  const X86Inst* iData = &X86InstDB::instData[instId];
+  uint32_t iFlags = iData->getFlags();
+
+  // Validate LOCK, XACQUIRE, and XRELEASE prefixes.
+  const uint32_t kLockXAcqRel = X86Inst::kOptionXAcquire | X86Inst::kOptionXRelease;
+  if (options & (X86Inst::kOptionLock | kLockXAcqRel)) {
+    if (options & X86Inst::kOptionLock) {
+      if (ASMJIT_UNLIKELY(!(iFlags & X86Inst::kFlagLock) && !(options & kLockXAcqRel)))
+        return DebugUtils::errored(kErrorInvalidLockPrefix);
+
+      if (ASMJIT_UNLIKELY(count < 1 || !operands[0].isMem()))
+        return DebugUtils::errored(kErrorInvalidLockPrefix);
+    }
+
+    if (options & kLockXAcqRel) {
+      if (ASMJIT_UNLIKELY(!(options & X86Inst::kOptionLock) || (options & kLockXAcqRel) == kLockXAcqRel))
+        return DebugUtils::errored(kErrorInvalidPrefixCombination);
+
+      if (ASMJIT_UNLIKELY((options & X86Inst::kOptionXAcquire) && !(iFlags & X86Inst::kFlagXAcquire)))
+        return DebugUtils::errored(kErrorInvalidXAcquirePrefix);
+
+      if (ASMJIT_UNLIKELY((options & X86Inst::kOptionXRelease) && !(iFlags & X86Inst::kFlagXRelease)))
+        return DebugUtils::errored(kErrorInvalidXReleasePrefix);
+    }
+  }
+
+  // Validate REP and REPNZ prefixes.
+  const uint32_t kRepRepRepnz = X86Inst::kOptionRep | X86Inst::kOptionRepnz;
+  if (options & kRepRepRepnz) {
+    if (ASMJIT_UNLIKELY((options & kRepRepRepnz) == kRepRepRepnz))
+      return DebugUtils::errored(kErrorInvalidPrefixCombination);
+
+    if (ASMJIT_UNLIKELY((options & X86Inst::kOptionRep) && !(iFlags & X86Inst::kFlagRep)))
+      return DebugUtils::errored(kErrorInvalidRepPrefix);
+
+    if (ASMJIT_UNLIKELY((options & X86Inst::kOptionRepnz) && !(iFlags & X86Inst::kFlagRepnz)))
+      return DebugUtils::errored(kErrorInvalidRepPrefix);
+
+    // TODO: Validate extraReg {cx|ecx|rcx}.
+  }
+
+  // Translate the given operands to `X86Inst::OSignature`.
+  X86Inst::OSignature oSigTranslated[6];
+  uint32_t combinedOpFlags = 0;
+  uint32_t combinedRegMask = 0;
+
+  const X86Mem* memOp = nullptr;
+
+  for (i = 0; i < count; i++) {
+    const Operand_& op = operands[i];
+    if (op.getOp() == Operand::kOpNone) break;
+
+    uint32_t opFlags = 0;
+    uint32_t memFlags = 0;
+    uint32_t regMask = 0;
+
+    switch (op.getOp()) {
+      case Operand::kOpReg: {
+        uint32_t regType = op.as<Reg>().getType();
+        if (ASMJIT_UNLIKELY(regType >= X86Reg::kRegCount))
+          return DebugUtils::errored(kErrorInvalidRegType);
+
+        opFlags = _x86OpFlagFromRegType[regType];
+        if (ASMJIT_UNLIKELY(opFlags == 0))
+          return DebugUtils::errored(kErrorInvalidRegType);
+
+        // If `regId` is equal or greater than Operand::kPackedIdMin it means
+        // that the register is virtual and its index will be assigned later
+        // by the register allocator. We must pass unless asked to disallow
+        // virtual registers.
+        // TODO: We need an option to refuse virtual regs here.
+        uint32_t regId = op.getId();
+        if (regId < Operand::kPackedIdMin) {
+          if (ASMJIT_UNLIKELY(regId >= 32))
+            return DebugUtils::errored(kErrorInvalidPhysId);
+
+          regMask = Utils::mask(regId);
+          if (ASMJIT_UNLIKELY((vd->allowedRegMask[regType] & regMask) == 0))
+            return DebugUtils::errored(kErrorInvalidPhysId);
+
+          combinedRegMask |= regMask;
+        }
+        else {
+          regMask = 0xFFFFFFFFU;
+        }
+        break;
+      }
+
+      // TODO: Validate base and index and combine with `combinedRegMask`.
+      case Operand::kOpMem: {
+        const X86Mem& m = op.as<X86Mem>();
+
+        uint32_t baseType = m.getBaseType();
+        uint32_t indexType = m.getIndexType();
+
+        memOp = &m;
+
+        if (m.getSegmentId() > 6)
+          return DebugUtils::errored(kErrorInvalidSegment);
+
+        if (baseType) {
+          uint32_t baseId = m.getBaseId();
+
+          if (m.isRegHome()) {
+            // Home address of virtual register. In such case we don't want to
+            // validate the type of the base register as it will always be patched
+            // to ESP|RSP.
+          }
+          else {
+            if (ASMJIT_UNLIKELY((vd->allowedMemBaseRegs & (1U << baseType)) == 0))
+              return DebugUtils::errored(kErrorInvalidAddress);
+          }
+
+          // Create information that will be validated only if this is an implicit
+          // memory operand. Basically only usable for string instructions and other
+          // instructions where memory operand is implicit and has 'seg:[reg]' form.
+          if (baseId < Operand::kPackedIdMin) {
+            // Physical base id.
+            regMask = Utils::mask(baseId);
+            combinedRegMask |= regMask;
+          }
+          else {
+            // Virtual base id - will the whole mask for implicit mem validation.
+            // The register is not assigned yet, so we cannot predict the phys id.
+            regMask = 0xFFFFFFFFU;
+          }
+
+          if (!indexType && !m.getOffsetLo32())
+            memFlags |= X86Inst::kMemOpBaseOnly;
+        }
+        else {
+          // Base is an address, make sure that the address doesn't overflow 32-bit
+          // integer (either int32_t or uint32_t) in 32-bit targets.
+          int64_t offset = m.getOffset();
+          if (archMask == X86Inst::kArchMaskX86 && !Utils::isInt32(offset) && !Utils::isUInt32(offset))
+            return DebugUtils::errored(kErrorInvalidAddress);
+        }
+
+        if (indexType) {
+          if (ASMJIT_UNLIKELY((vd->allowedMemIndexRegs & (1U << indexType)) == 0))
+            return DebugUtils::errored(kErrorInvalidAddress);
+
+          if (indexType == X86Reg::kRegXmm) {
+            opFlags |= X86Inst::kOpVm;
+            memFlags |= X86Inst::kMemOpVm32x | X86Inst::kMemOpVm64x;
+          }
+          else if (indexType == X86Reg::kRegYmm) {
+            opFlags |= X86Inst::kOpVm;
+            memFlags |= X86Inst::kMemOpVm32y | X86Inst::kMemOpVm64y;
+          }
+          else if (indexType == X86Reg::kRegZmm) {
+            opFlags |= X86Inst::kOpVm;
+            memFlags |= X86Inst::kMemOpVm32z | X86Inst::kMemOpVm64z;
+          }
+          else {
+            opFlags |= X86Inst::kOpMem;
+            if (baseType)
+              memFlags |= X86Inst::kMemOpMib;
+          }
+
+          // [RIP + {XMM|YMM|ZMM}] is not allowed.
+          if (baseType == X86Reg::kRegRip && (opFlags & X86Inst::kOpVm))
+            return DebugUtils::errored(kErrorInvalidAddress);
+
+          uint32_t indexId = m.getIndexId();
+          if (indexId < Operand::kPackedIdMin)
+            combinedRegMask |= Utils::mask(indexId);
+
+          // Only used for implicit memory operands having 'seg:[reg]' form, so clear it.
+          regMask = 0;
+        }
+        else {
+          opFlags |= X86Inst::kOpMem;
+        }
+
+        switch (m.getSize()) {
+          case  0: memFlags |= X86Inst::kMemOpAny ; break;
+          case  1: memFlags |= X86Inst::kMemOpM8  ; break;
+          case  2: memFlags |= X86Inst::kMemOpM16 ; break;
+          case  4: memFlags |= X86Inst::kMemOpM32 ; break;
+          case  6: memFlags |= X86Inst::kMemOpM48 ; break;
+          case  8: memFlags |= X86Inst::kMemOpM64 ; break;
+          case 10: memFlags |= X86Inst::kMemOpM80 ; break;
+          case 16: memFlags |= X86Inst::kMemOpM128; break;
+          case 32: memFlags |= X86Inst::kMemOpM256; break;
+          case 64: memFlags |= X86Inst::kMemOpM512; break;
+          default:
+            return DebugUtils::errored(kErrorInvalidOperandSize);
+        }
+
+        break;
+      }
+
+      case Operand::kOpImm: {
+        uint64_t immValue = op.as<Imm>().getUInt64();
+        uint32_t immFlags = 0;
+
+        if (static_cast<int64_t>(immValue) >= 0) {
+          const uint32_t k32AndMore = X86Inst::kOpI32 | X86Inst::kOpU32 |
+                                      X86Inst::kOpI64 | X86Inst::kOpU64 ;
+
+          if (immValue <= 0xFU)
+            immFlags = X86Inst::kOpU4 | X86Inst::kOpI8 | X86Inst::kOpU8 | X86Inst::kOpI16 | X86Inst::kOpU16 | k32AndMore;
+          else if (immValue <= 0x7FU)
+            immFlags = X86Inst::kOpI8 | X86Inst::kOpU8 | X86Inst::kOpI16 | X86Inst::kOpU16 | k32AndMore;
+          else if (immValue <= 0xFFU)
+            immFlags = X86Inst::kOpU8 | X86Inst::kOpI16 | X86Inst::kOpU16 | k32AndMore;
+          else if (immValue <= 0x7FFFU)
+            immFlags = X86Inst::kOpI16 | X86Inst::kOpU16 | k32AndMore;
+          else if (immValue <= 0xFFFFU)
+            immFlags = X86Inst::kOpU16 | k32AndMore;
+          else if (immValue <= 0x7FFFFFFFU)
+            immFlags = k32AndMore;
+          else if (immValue <= 0xFFFFFFFFU)
+            immFlags = X86Inst::kOpU32 | X86Inst::kOpI64 | X86Inst::kOpU64;
+          else if (immValue <= ASMJIT_UINT64_C(0x7FFFFFFFFFFFFFFF))
+            immFlags = X86Inst::kOpI64 | X86Inst::kOpU64;
+          else
+            immFlags = X86Inst::kOpU64;
+        }
+        else {
+          // 2s complement negation, as our number is unsigned...
+          immValue = (~immValue + 1);
+
+          if (immValue <= 0x80U)
+            immFlags = X86Inst::kOpI8 | X86Inst::kOpI16 | X86Inst::kOpI32 | X86Inst::kOpI64;
+          else if (immValue <= 0x8000U)
+            immFlags = X86Inst::kOpI16 | X86Inst::kOpI32 | X86Inst::kOpI64;
+          else if (immValue <= 0x80000000U)
+            immFlags = X86Inst::kOpI32 | X86Inst::kOpI64;
+          else
+            immFlags = X86Inst::kOpI64;
+        }
+        opFlags |= immFlags;
+        break;
+      }
+
+      case Operand::kOpLabel: {
+        opFlags |= X86Inst::kOpRel8 | X86Inst::kOpRel32;
+        break;
+      }
+
+      default:
+        return DebugUtils::errored(kErrorInvalidState);
+    }
+
+    X86Inst::OSignature& tod = oSigTranslated[i];
+    tod.flags = opFlags;
+    tod.memFlags = static_cast<uint16_t>(memFlags);
+    tod.regMask = static_cast<uint8_t>(regMask & 0xFFU);
+    combinedOpFlags |= opFlags;
+  }
+
+  // Decrease the number of operands of those that are none. This is important
+  // as Assembler and CodeCompiler may just pass more operands where some of
+  // them are none (it means that no operand is given at that index). However,
+  // validate that there are no gaps (like [reg, none, reg] or [none, reg]).
+  if (i < count) {
+    while (--count > i)
+      if (ASMJIT_UNLIKELY(!operands[count].isNone()))
+        return DebugUtils::errored(kErrorInvalidState);
+  }
+
+  // Validate X86 and X64 specific cases.
+  if (archMask == X86Inst::kArchMaskX86) {
+    // Illegal use of 64-bit register in 32-bit mode.
+    if (ASMJIT_UNLIKELY((combinedOpFlags & X86Inst::kOpGpq) != 0))
+      return DebugUtils::errored(kErrorInvalidUseOfGpq);
+  }
+  else {
+    // Illegal use of a high 8-bit register with REX prefix.
+    if (ASMJIT_UNLIKELY((combinedOpFlags & X86Inst::kOpGpbHi) != 0 && (combinedRegMask & 0xFFFFFF00U) != 0))
+      return DebugUtils::errored(kErrorInvalidUseOfGpbHi);
+  }
+
+  // Validate instruction operands.
+  const X86Inst::CommonData* commonData = &iData->getCommonData();
+  const X86Inst::ISignature* iSig = X86InstDB::iSignatureData + commonData->_iSignatureIndex;
+  const X86Inst::ISignature* iEnd = iSig                      + commonData->_iSignatureCount;
+
+  if (iSig != iEnd) {
+    const X86Inst::OSignature* oSigData = X86InstDB::oSignatureData;
+
+    // If set it means that we matched a signature where only immediate value
+    // was out of bounds. We can return a more descriptive error if we know this.
+    bool globalImmOutOfRange = false;
+
+    do {
+      // Check if the architecture is compatible.
+      if ((iSig->archMask & archMask) == 0) continue;
+
+      // Compare the operands table with reference operands.
+      uint32_t j = 0;
+      uint32_t iSigCount = iSig->opCount;
+      bool localImmOutOfRange = false;
+
+      if (iSigCount == count) {
+        for (j = 0; j < count; j++)
+          if (!x86CheckOSig(oSigTranslated[j], oSigData[iSig->operands[j]], localImmOutOfRange))
+            break;
+      }
+      else if (iSigCount - iSig->implicit == count) {
+        uint32_t r = 0;
+        for (j = 0; j < count && r < iSigCount; j++, r++) {
+          const X86Inst::OSignature* oChk = oSigTranslated + j;
+          const X86Inst::OSignature* oRef;
+Next:
+          oRef = oSigData + iSig->operands[r];
+          // Skip implicit.
+          if ((oRef->flags & X86Inst::kOpImplicit) != 0) {
+            if (++r >= iSigCount)
+              break;
+            else
+              goto Next;
+          }
+
+          if (!x86CheckOSig(*oChk, *oRef, localImmOutOfRange))
+            break;
+        }
+      }
+
+      if (j == count) {
+        if (!localImmOutOfRange) {
+          // Match, must clear possible `globalImmOutOfRange`.
+          globalImmOutOfRange = false;
+          break;
+        }
+        globalImmOutOfRange = localImmOutOfRange;
+      }
+    } while (++iSig != iEnd);
+
+    if (iSig == iEnd) {
+      if (globalImmOutOfRange)
+        return DebugUtils::errored(kErrorInvalidImmediate);
+      else
+        return DebugUtils::errored(kErrorInvalidInstruction);
+    }
+  }
+
+  // Validate AVX-512 options:
+  const RegOnly& extraReg = detail.extraReg;
+  const uint32_t kAvx512Options = X86Inst::kOptionZMask   |
+                                  X86Inst::kOption1ToX    |
+                                  X86Inst::kOptionER      |
+                                  X86Inst::kOptionSAE     ;
+
+  if (!extraReg.isNone() || (options & kAvx512Options)) {
+    if (commonData->hasFlag(X86Inst::kFlagEvex)) {
+      // Validate AVX-512 {k} and {k}{z}.
+      if (!extraReg.isNone()) {
+        // Mask can only be specified by a 'k' register.
+        if (ASMJIT_UNLIKELY(extraReg.getType() != X86Reg::kRegK))
+          return DebugUtils::errored(kErrorInvalidKMaskReg);
+
+        if (ASMJIT_UNLIKELY(!commonData->hasAvx512K()))
+          return DebugUtils::errored(kErrorInvalidKMaskUse);
+      }
+
+      if ((options & X86Inst::kOptionZMask)) {
+        if (ASMJIT_UNLIKELY((options & X86Inst::kOptionZMask) != 0 && !commonData->hasAvx512Z()))
+          return DebugUtils::errored(kErrorInvalidKZeroUse);
+      }
+
+      // Validate AVX-512 broadcast {1tox}.
+      if (options & X86Inst::kOption1ToX) {
+        if (ASMJIT_UNLIKELY(!memOp))
+          return DebugUtils::errored(kErrorInvalidBroadcast);
+
+        uint32_t size = memOp->getSize();
+        if (size != 0) {
+          // The the size is specified it has to match the broadcast size.
+          if (ASMJIT_UNLIKELY(commonData->hasAvx512B32() && size != 4))
+            return DebugUtils::errored(kErrorInvalidBroadcast);
+
+          if (ASMJIT_UNLIKELY(commonData->hasAvx512B64() && size != 8))
+            return DebugUtils::errored(kErrorInvalidBroadcast);
+        }
+      }
+
+      // Validate AVX-512 {sae} and {er}.
+      if (options & (X86Inst::kOptionSAE | X86Inst::kOptionER)) {
+        // Rounding control is impossible if the instruction is not reg-to-reg.
+        if (ASMJIT_UNLIKELY(memOp))
+          return DebugUtils::errored(kErrorInvalidEROrSAE);
+
+        // Check if {sae} or {er} is supported by the instruction.
+        if (options & X86Inst::kOptionER) {
+          // NOTE: if both {sae} and {er} are set, we don't care, as {sae} is implied.
+          if (ASMJIT_UNLIKELY(!commonData->hasAvx512ER()))
+            return DebugUtils::errored(kErrorInvalidEROrSAE);
+
+          // {er} is defined for scalar ops or vector ops using zmm (LL = 10). We
+          // don't need any more bits in the instruction database to be able to
+          // validate this, as each AVX512 instruction that has broadcast is vector
+          // instruction (in this case we require zmm registers), otherwise it's a
+          // scalar instruction, which is valid.
+          if (commonData->hasAvx512B()) {
+            // Supports broadcast, thus we require LL to be '10', which means there
+            // have to be zmm registers used. We don't calculate LL here, but we know
+            // that it would be '10' if there is at least one ZMM register used.
+
+            // There is no 'ER' enabled instruction with less than two operands.
+            ASMJIT_ASSERT(count >= 2);
+            if (ASMJIT_UNLIKELY(!X86Reg::isZmm(operands[0]) && !X86Reg::isZmm(operands[1])))
+              return DebugUtils::errored(kErrorInvalidEROrSAE);
+          }
+        }
+        else {
+          // {sae} doesn't have the same limitations as {er}, this is enough.
+          if (ASMJIT_UNLIKELY(!commonData->hasAvx512SAE()))
+            return DebugUtils::errored(kErrorInvalidEROrSAE);
+        }
+      }
+    }
+    else {
+      // Not AVX512 instruction - maybe OpExtra is xCX register used by REP/REPNZ prefix. Otherwise the instruction is invalid.
+      if ((options & kAvx512Options) || (options & (X86Inst::kOptionRep | X86Inst::kOptionRepnz)) == 0)
+        return DebugUtils::errored(kErrorInvalidInstruction);
+    }
+  }
+
+  return kErrorOk;
+}
+#endif
+
+// ============================================================================
+// [asmjit::X86InstImpl - CheckFeatures]
+// ============================================================================
+
+#if !defined(ASMJIT_DISABLE_EXTENSIONS)
+ASMJIT_FAVOR_SIZE static uint32_t x86GetRegTypesMask(const Operand_* operands, uint32_t count) noexcept {
+  uint32_t mask = 0;
+  for (uint32_t i = 0; i < count; i++) {
+    const Operand_& op = operands[i];
+    if (op.isReg()) {
+      const Reg& reg = op.as<Reg>();
+      mask |= Utils::mask(reg.getType());
+    }
+    else if (op.isMem()) {
+      const Mem& mem = op.as<Mem>();
+      if (mem.hasBaseReg()) mask |= Utils::mask(mem.getBaseType());
+      if (mem.hasIndexReg()) mask |= Utils::mask(mem.getIndexType());
+    }
+  }
+  return mask;
+}
+
+ASMJIT_FAVOR_SIZE Error X86InstImpl::checkFeatures(uint32_t archType, const Inst::Detail& detail, const Operand_* operands, uint32_t count, CpuFeatures& out) noexcept {
+  if (!ArchInfo::isX86Family(archType))
+    return DebugUtils::errored(kErrorInvalidArch);
+
+  // Get the instruction data.
+  uint32_t instId = detail.instId;
+  if (ASMJIT_UNLIKELY(instId >= X86Inst::_kIdCount))
+    return DebugUtils::errored(kErrorInvalidArgument);
+
+  const X86Inst* iData = &X86InstDB::instData[instId];
+  const X86Inst::OperationData& od = iData->getOperationData();
+
+  const uint8_t* fData = od.getFeaturesData();
+  const uint8_t* fEnd = od.getFeaturesEnd();
+
+  // Copy all features to `out`.
+  out.reset();
+  do {
+    uint32_t feature = fData[0];
+    if (!feature)
+      break;
+    out.add(feature);
+  } while (++fData != fEnd);
+
+  // Since AsmJit merges all instructions that share the same name we have to
+  // deal with some special cases and also with MMX/SSE and AVX/AVX2 overlaps.
+
+  // Only proceed if there were some CPU flags set.
+  if (fData != od.getFeaturesData()) {
+    uint32_t mask = x86GetRegTypesMask(operands, count);
+
+    // Check for MMX vs SSE overlap.
+    if (out.has(CpuInfo::kX86FeatureMMX) || out.has(CpuInfo::kX86FeatureMMX2)) {
+      // Only instructions defined by SSE and SSE2 overlap. Instructions introduced
+      // by newer instruction sets like SSE3+ don't state MMX as they require SSE3+.
+      if (out.has(CpuInfo::kX86FeatureSSE) || out.has(CpuInfo::kX86FeatureSSE2)) {
+        if (!(mask & Utils::mask(X86Reg::kRegXmm))) {
+          // The instruction doesn't use XMM register(s), thus it's MMX/MMX2 only.
+          out.remove(CpuInfo::kX86FeatureSSE);
+          out.remove(CpuInfo::kX86FeatureSSE2);
+        }
+        else {
+          out.remove(CpuInfo::kX86FeatureMMX);
+          out.remove(CpuInfo::kX86FeatureMMX2);
+        }
+
+        // Special case: PEXTRW instruction is MMX/SSE2 instruction. However, this
+        // instruction couldn't access memory (only register to register extract) so
+        // when SSE4.1 introduced the whole family of PEXTR/PINSR instructions they
+        // also introduced PEXTRW with a new opcode 0x15 that can extract directly to
+        // memory. This instruction is, of course, not compatible with MMX/SSE2 one.
+        if (instId == X86Inst::kIdPextrw && count > 0 && !operands[0].isMem()) {
+          out.remove(CpuInfo::kX86FeatureSSE4_1);
+        }
+      }
+    }
+
+    // Check for AVX vs AVX2 overlap.
+    if (out.has(CpuInfo::kX86FeatureAVX) && out.has(CpuInfo::kX86FeatureAVX2)) {
+      bool isAVX2 = true;
+      // Special case: VBROADCASTSS and VBROADCASTSD were introduced in AVX, but
+      // only version that uses memory as a source operand. AVX2 then added support
+      // for register source operand.
+      if (instId == X86Inst::kIdVbroadcastss || instId == X86Inst::kIdVbroadcastsd) {
+        if (count > 1 && operands[0].isMem())
+          isAVX2 = false;
+      }
+      else {
+        // AVX instruction set doesn't support integer operations on YMM registers
+        // as these were later introcuced by AVX2. In our case we have to check if
+        // YMM register(s) are in use and if that is the case this is an AVX2 instruction.
+        if (!(mask & Utils::mask(X86Reg::kRegYmm, X86Reg::kRegZmm)))
+          isAVX2 = false;
+      }
+
+      if (isAVX2)
+        out.remove(CpuInfo::kX86FeatureAVX);
+      else
+        out.remove(CpuInfo::kX86FeatureAVX2);
+    }
+
+    // Check for AVX|AVX2|FMA|F16C vs AVX512 overlap.
+    if (out.has(CpuInfo::kX86FeatureAVX) || out.has(CpuInfo::kX86FeatureAVX2) || out.has(CpuInfo::kX86FeatureFMA) || out.has(CpuInfo::kX86FeatureF16C)) {
+      // Only AVX512-F|BW|DQ allow to encode AVX/AVX2 instructions
+      if (out.has(CpuInfo::kX86FeatureAVX512_F) || out.has(CpuInfo::kX86FeatureAVX512_BW) || out.has(CpuInfo::kX86FeatureAVX512_DQ)) {
+        uint32_t options = detail.options;
+        uint32_t kAvx512Options = X86Inst::kOptionEvex | X86Inst::_kOptionAvx512Mask;
+
+        if (!(mask & Utils::mask(X86Reg::kRegZmm, X86Reg::kRegK)) && !(options & (kAvx512Options)) && detail.extraReg.getType() != X86Reg::kRegK) {
+          out.remove(CpuInfo::kX86FeatureAVX512_F)
+             .remove(CpuInfo::kX86FeatureAVX512_BW)
+             .remove(CpuInfo::kX86FeatureAVX512_DQ)
+             .remove(CpuInfo::kX86FeatureAVX512_VL);
+        }
+      }
+    }
+
+    // Remove or keep AVX512_VL feature.
+    if (out.has(CpuInfo::kX86FeatureAVX512_VL)) {
+      if (!(mask & Utils::mask(X86Reg::kRegZmm)))
+        out.remove(CpuInfo::kX86FeatureAVX512_VL);
+    }
+  }
+
+  return kErrorOk;
+}
+#endif
+
+} // asmjit namespace
+
+// [Api-End]
+#include "../asmjit_apiend.h"

libraries/asmjit/x86/x86instimpl_p.h

+// [AsmJit]
+// Complete x86/x64 JIT and Remote Assembler for C++.
+//
+// [License]
+// Zlib - See LICENSE.md file in the package.
+
+// [Guard]
+#ifndef _ASMJIT_X86_X86INSTIMPL_P_H
+#define _ASMJIT_X86_X86INSTIMPL_P_H
+
+// [Dependencies]
+#include "../x86/x86inst.h"
+
+// [Api-Begin]
+#include "../asmjit_apibegin.h"
+
+namespace asmjit {
+
+//! \addtogroup asmjit_x86
+//! \{
+
+//! \internal
+//!
+//! Contains X86/X64 specific implementation of APIs provided by `asmjit::Inst`.
+//!
+//! The purpose of `X86InstImpl` is to move most of the logic out of `X86Inst`.
+struct X86InstImpl {
+  #if !defined(ASMJIT_DISABLE_VALIDATION)
+  static Error validate(uint32_t archType, const Inst::Detail& detail, const Operand_* operands, uint32_t count) noexcept;
+  #endif
+
+  #if !defined(ASMJIT_DISABLE_EXTENSIONS)
+  static Error checkFeatures(uint32_t archType, const Inst::Detail& detail, const Operand_* operands, uint32_t count, CpuFeatures& out) noexcept;
+  #endif
+};
+
+//! \}
+
+} // asmjit namespace
+
+// [Api-End]
+#include "../asmjit_apiend.h"
+
+// [Guard]
+#endif // _ASMJIT_X86_X86INSTIMPL_P_H

libraries/asmjit/x86/x86internal.cpp

+// [AsmJit]
+// Complete x86/x64 JIT and Remote Assembler for C++.
+//
+// [License]
+// Zlib - See LICENSE.md file in the package.
+
+// [Export]
+#define ASMJIT_EXPORTS
+
+// [Guard]
+#include "../asmjit_build.h"
+#if defined(ASMJIT_BUILD_X86)
+
+// [Dependencies]
+#include "../x86/x86internal_p.h"
+
+// [Api-Begin]
+#include "../asmjit_apibegin.h"
+
+namespace asmjit {
+
+// ============================================================================
+// [asmjit::X86Internal - Helpers]
+// ============================================================================
+
+static ASMJIT_INLINE uint32_t x86GetXmmMovInst(const FuncFrameLayout& layout) {
+  bool avx = layout.isAvxEnabled();
+  bool aligned = layout.hasAlignedVecSR();
+
+  return aligned ? (avx ? X86Inst::kIdVmovaps : X86Inst::kIdMovaps)
+                 : (avx ? X86Inst::kIdVmovups : X86Inst::kIdMovups);
+}
+
+static ASMJIT_INLINE uint32_t x86VecTypeIdToRegType(uint32_t typeId) noexcept {
+  return typeId <= TypeId::_kVec128End ? X86Reg::kRegXmm :
+         typeId <= TypeId::_kVec256End ? X86Reg::kRegYmm :
+                                         X86Reg::kRegZmm ;
+}
+
+// ============================================================================
+// [asmjit::X86FuncArgsContext]
+// ============================================================================
+
+// Used by both, `Utils::argsToFrameInfo()` and `Utils::allocArgs()`.
+class X86FuncArgsContext {
+public:
+  typedef FuncDetail::Value SrcArg;
+  typedef FuncArgsMapper::Value DstArg;
+
+  enum { kMaxVRegKinds = Globals::kMaxVRegKinds };
+
+  struct WorkData {
+    uint32_t archRegs;                   //!< Architecture provided and allocable regs.
+    uint32_t workRegs;                   //!< Registers that can be used by shuffler.
+    uint32_t usedRegs;                   //!< Only registers used to pass arguments.
+    uint32_t srcRegs;                    //!< Source registers that need shuffling.
+    uint32_t dstRegs;                    //!< Destination registers that need shuffling.
+    uint8_t numOps;                      //!< Number of operations to finish.
+    uint8_t numSwaps;                    //!< Number of register swaps.
+    uint8_t numStackArgs;                //!< Number of stack loads.
+    uint8_t reserved[9];                 //!< Reserved (only used as padding).
+    uint8_t argIndex[32];                //!< Only valid if a corresponding bit in `userRegs` is true.
+  };
+
+  X86FuncArgsContext() noexcept;
+  Error initWorkData(const FuncArgsMapper& args, const uint32_t* dirtyRegs, bool preservedFP) noexcept;
+
+  Error markRegsForSwaps(FuncFrameInfo& ffi) noexcept;
+  Error markDstRegsDirty(FuncFrameInfo& ffi) noexcept;
+  Error markStackArgsReg(FuncFrameInfo& ffi) noexcept;
+
+  // --------------------------------------------------------------------------
+  // [Members]
+  // --------------------------------------------------------------------------
+
+  WorkData _workData[kMaxVRegKinds];
+  bool _hasStackArgs;
+  bool _hasRegSwaps;
+};
+
+X86FuncArgsContext::X86FuncArgsContext() noexcept {
+  ::memset(_workData, 0, sizeof(_workData));
+  _hasStackArgs = false;
+  _hasRegSwaps = false;
+}
+
+ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::initWorkData(const FuncArgsMapper& args, const uint32_t* dirtyRegs, bool preservedFP) noexcept {
+  // This code has to be updated if this changes.
+  ASMJIT_ASSERT(kMaxVRegKinds == 4);
+
+  uint32_t i;
+  const FuncDetail& func = *args.getFuncDetail();
+
+  uint32_t archType = func.getCallConv().getArchType();
+  uint32_t count = (archType == ArchInfo::kTypeX86) ? 8 : 16;
+
+  // Initialize WorkData::archRegs.
+  _workData[X86Reg::kKindGp ].archRegs = Utils::bits(count) & ~Utils::mask(X86Gp::kIdSp);
+  _workData[X86Reg::kKindMm ].archRegs = Utils::bits(8);
+  _workData[X86Reg::kKindK  ].archRegs = Utils::bits(8);
+  _workData[X86Reg::kKindVec].archRegs = Utils::bits(count);
+
+  if (preservedFP)
+    _workData[X86Reg::kKindGp].archRegs &= ~Utils::mask(X86Gp::kIdBp);
+
+  // Initialize WorkData::workRegs.
+  for (i = 0; i < kMaxVRegKinds; i++)
+    _workData[i].workRegs = _workData[i].archRegs & (dirtyRegs[i] | ~func.getCallConv().getPreservedRegs(i));
+
+  // Build WorkData.
+  for (i = 0; i < kFuncArgCountLoHi; i++) {
+    const DstArg& dstArg = args.getArg(i);
+    if (!dstArg.isAssigned()) continue;
+
+    const SrcArg& srcArg = func.getArg(i);
+    if (ASMJIT_UNLIKELY(!srcArg.isAssigned()))
+      return DebugUtils::errored(kErrorInvalidState);
+
+    uint32_t dstRegType = dstArg.getRegType();
+    if (ASMJIT_UNLIKELY(dstRegType >= X86Reg::kRegCount))
+      return DebugUtils::errored(kErrorInvalidRegType);
+
+    uint32_t dstRegKind = X86Reg::kindOf(dstRegType);
+    if (ASMJIT_UNLIKELY(dstRegKind >= kMaxVRegKinds))
+      return DebugUtils::errored(kErrorInvalidState);
+
+    WorkData& dstData = _workData[dstRegKind];
+    uint32_t dstRegId = dstArg.getRegId();
+    if (ASMJIT_UNLIKELY(dstRegId >= 32 || !(dstData.archRegs & Utils::mask(dstRegId))))
+      return DebugUtils::errored(kErrorInvalidPhysId);
+
+    uint32_t dstRegMask = Utils::mask(dstRegId);
+    if (ASMJIT_UNLIKELY(dstData.usedRegs & dstRegMask))
+      return DebugUtils::errored(kErrorOverlappedRegs);
+
+    dstData.usedRegs |= dstRegMask;
+    dstData.argIndex[dstRegId] = static_cast<uint8_t>(i);
+
+    if (srcArg.byReg()) {
+      uint32_t srcRegKind = X86Reg::kindOf(srcArg.getRegType());
+      uint32_t srcRegId = srcArg.getRegId();
+      uint32_t srcRegMask = Utils::mask(srcRegId);
+
+      if (dstRegKind == srcRegKind) {
+        // The best case, register is allocated where it is expected to be.
+        if (dstRegId == srcRegId) continue;
+
+        // Detect a register swap.
+        if (dstData.usedRegs & srcRegMask) {
+          const SrcArg& ref = func.getArg(dstData.argIndex[srcRegId]);
+          if (ref.byReg() && X86Reg::kindOf(ref.getRegType()) == dstRegKind && ref.getRegId() == dstRegId) {
+            dstData.numSwaps++;
+            _hasRegSwaps = true;
+          }
+        }
+        dstData.srcRegs |= srcRegMask;
+      }
+      else {
+        if (ASMJIT_UNLIKELY(srcRegKind >= kMaxVRegKinds))
+          return DebugUtils::errored(kErrorInvalidState);
+
+        WorkData& srcData = _workData[srcRegKind];
+        srcData.srcRegs |= srcRegMask;
+      }
+    }
+    else {
+      dstData.numStackArgs++;
+      _hasStackArgs = true;
+    }
+
+    dstData.numOps++;
+    dstData.dstRegs |= dstRegMask;
+  }
+
+  return kErrorOk;
+}
+
+ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markDstRegsDirty(FuncFrameInfo& ffi) noexcept {
+  for (uint32_t i = 0; i < kMaxVRegKinds; i++) {
+    WorkData& wd = _workData[i];
+    uint32_t regs = wd.usedRegs | wd.dstRegs;
+
+    wd.workRegs |= regs;
+    ffi.addDirtyRegs(i, regs);
+  }
+
+  return kErrorOk;
+}
+
+ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markRegsForSwaps(FuncFrameInfo& ffi) noexcept {
+  if (!_hasRegSwaps)
+    return kErrorOk;
+
+  // If some registers require swapping then select one dirty register that
+  // can be used as a temporary. We can do it also without it (by using xors),
+  // but using temporary is always safer and also faster approach.
+  for (uint32_t i = 0; i < kMaxVRegKinds; i++) {
+    // Skip all register kinds where swapping is natively supported (GP regs).
+    if (i == X86Reg::kKindGp) continue;
+
+    // Skip all register kinds that don't require swapping.
+    WorkData& wd = _workData[i];
+    if (!wd.numSwaps) continue;
+
+    // Initially, pick some clobbered or dirty register.
+    uint32_t workRegs = wd.workRegs;
+    uint32_t regs = workRegs & ~(wd.usedRegs | wd.dstRegs);
+
+    // If that didn't work out pick some register which is not in 'used'.
+    if (!regs) regs = workRegs & ~wd.usedRegs;
+
+    // If that didn't work out pick any other register that is allocable.
+    // This last resort case will, however, result in marking one more
+    // register dirty.
+    if (!regs) regs = wd.archRegs & ~workRegs;
+
+    // If that didn't work out we will have to use xors instead of moves.
+    if (!regs) continue;
+
+    uint32_t regMask = Utils::mask(Utils::findFirstBit(regs));
+    wd.workRegs |= regMask;
+    ffi.addDirtyRegs(i, regMask);
+  }
+
+  return kErrorOk;
+}
+
+ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markStackArgsReg(FuncFrameInfo& ffi) noexcept {
+  if (!_hasStackArgs)
+    return kErrorOk;
+
+  // Decide which register to use to hold the stack base address.
+  if (!ffi.hasPreservedFP()) {
+    WorkData& wd = _workData[X86Reg::kKindGp];
+    uint32_t saRegId = ffi.getStackArgsRegId();
+    uint32_t usedRegs = wd.usedRegs;
+
+    if (saRegId != Globals::kInvalidRegId) {
+      // Check if the user chosen SA register doesn't overlap with others.
+      // However, it's fine if it overlaps with some 'dstMove' register.
+      if (usedRegs & Utils::mask(saRegId))
+        return DebugUtils::errored(kErrorOverlappingStackRegWithRegArg);
+    }
+    else {
+      // Initially, pick some clobbered or dirty register that is neither
+      // in 'used' and neither in 'dstMove'. That's the safest bet as the
+      // register won't collide with anything right now.
+      uint32_t regs = wd.workRegs & ~(usedRegs | wd.dstRegs);
+
+      // If that didn't work out pick some register which is not in 'used'.
+      if (!regs) regs = wd.workRegs & ~usedRegs;
+
+      // If that didn't work out then we have to make one more register dirty.
+      if (!regs) regs = wd.archRegs & ~wd.workRegs;
+
+      // If that didn't work out we can't continue.
+      if (ASMJIT_UNLIKELY(!regs))
+        return DebugUtils::errored(kErrorNoMorePhysRegs);
+
+      saRegId = Utils::findFirstBit(regs);
+      ffi.setStackArgsRegId(saRegId);
+    }
+  }
+  else {
+    ffi.setStackArgsRegId(X86Gp::kIdBp);
+  }
+
+  return kErrorOk;
+}
+
+// ============================================================================
+// [asmjit::X86Internal - CallConv]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Internal::initCallConv(CallConv& cc, uint32_t ccId) noexcept {
+  const uint32_t kKindGp  = X86Reg::kKindGp;
+  const uint32_t kKindVec = X86Reg::kKindVec;
+  const uint32_t kKindMm  = X86Reg::kKindMm;
+  const uint32_t kKindK   = X86Reg::kKindK;
+
+  const uint32_t kZax = X86Gp::kIdAx;
+  const uint32_t kZbx = X86Gp::kIdBx;
+  const uint32_t kZcx = X86Gp::kIdCx;
+  const uint32_t kZdx = X86Gp::kIdDx;
+  const uint32_t kZsp = X86Gp::kIdSp;
+  const uint32_t kZbp = X86Gp::kIdBp;
+  const uint32_t kZsi = X86Gp::kIdSi;
+  const uint32_t kZdi = X86Gp::kIdDi;
+
+  switch (ccId) {
+    case CallConv::kIdX86StdCall:
+      cc.setFlags(CallConv::kFlagCalleePopsStack);
+      goto X86CallConv;
+
+    case CallConv::kIdX86MsThisCall:
+      cc.setFlags(CallConv::kFlagCalleePopsStack);
+      cc.setPassedOrder(kKindGp, kZcx);
+      goto X86CallConv;
+
+    case CallConv::kIdX86MsFastCall:
+    case CallConv::kIdX86GccFastCall:
+      cc.setFlags(CallConv::kFlagCalleePopsStack);
+      cc.setPassedOrder(kKindGp, kZcx, kZdx);
+      goto X86CallConv;
+
+    case CallConv::kIdX86GccRegParm1:
+      cc.setPassedOrder(kKindGp, kZax);
+      goto X86CallConv;
+
+    case CallConv::kIdX86GccRegParm2:
+      cc.setPassedOrder(kKindGp, kZax, kZdx);
+      goto X86CallConv;
+
+    case CallConv::kIdX86GccRegParm3:
+      cc.setPassedOrder(kKindGp, kZax, kZdx, kZcx);
+      goto X86CallConv;
+
+    case CallConv::kIdX86CDecl:
+X86CallConv:
+      cc.setNaturalStackAlignment(4);
+      cc.setArchType(ArchInfo::kTypeX86);
+      cc.setPreservedRegs(kKindGp, Utils::mask(kZbx, kZsp, kZbp, kZsi, kZdi));
+      break;
+
+    case CallConv::kIdX86Win64:
+      cc.setArchType(ArchInfo::kTypeX64);
+      cc.setAlgorithm(CallConv::kAlgorithmWin64);
+      cc.setFlags(CallConv::kFlagPassFloatsByVec | CallConv::kFlagIndirectVecArgs);
+      cc.setNaturalStackAlignment(16);
+      cc.setSpillZoneSize(32);
+      cc.setPassedOrder(kKindGp, kZcx, kZdx, 8, 9);
+      cc.setPassedOrder(kKindVec, 0, 1, 2, 3);
+      cc.setPreservedRegs(kKindGp, Utils::mask(kZbx, kZsp, kZbp, kZsi, kZdi, 12, 13, 14, 15));
+      cc.setPreservedRegs(kKindVec, Utils::mask(6, 7, 8, 9, 10, 11, 12, 13, 14, 15));
+      break;
+
+    case CallConv::kIdX86SysV64:
+      cc.setArchType(ArchInfo::kTypeX64);
+      cc.setFlags(CallConv::kFlagPassFloatsByVec);
+      cc.setNaturalStackAlignment(16);
+      cc.setRedZoneSize(128);
+      cc.setPassedOrder(kKindGp, kZdi, kZsi, kZdx, kZcx, 8, 9);
+      cc.setPassedOrder(kKindVec, 0, 1, 2, 3, 4, 5, 6, 7);
+      cc.setPreservedRegs(kKindGp, Utils::mask(kZbx, kZsp, kZbp, 12, 13, 14, 15));
+      break;
+
+    case CallConv::kIdX86FastEval2:
+    case CallConv::kIdX86FastEval3:
+    case CallConv::kIdX86FastEval4: {
+      uint32_t n = ccId - CallConv::kIdX86FastEval2;
+
+      cc.setArchType(ArchInfo::kTypeX86);
+      cc.setFlags(CallConv::kFlagPassFloatsByVec);
+      cc.setNaturalStackAlignment(16);
+      cc.setPassedOrder(kKindGp, kZax, kZdx, kZcx, kZsi, kZdi);
+      cc.setPassedOrder(kKindMm, 0, 1, 2, 3, 4, 5, 6, 7);
+      cc.setPassedOrder(kKindVec, 0, 1, 2, 3, 4, 5, 6, 7);
+
+      cc.setPreservedRegs(kKindGp , Utils::bits(8));
+      cc.setPreservedRegs(kKindVec, Utils::bits(8) & ~Utils::bits(n));
+      cc.setPreservedRegs(kKindMm , Utils::bits(8));
+      cc.setPreservedRegs(kKindK  , Utils::bits(8));
+      break;
+    }
+
+    case CallConv::kIdX64FastEval2:
+    case CallConv::kIdX64FastEval3:
+    case CallConv::kIdX64FastEval4: {
+      uint32_t n = ccId - CallConv::kIdX64FastEval2;
+
+      cc.setArchType(ArchInfo::kTypeX64);
+      cc.setFlags(CallConv::kFlagPassFloatsByVec);
+      cc.setNaturalStackAlignment(16);
+      cc.setPassedOrder(kKindGp, kZax, kZdx, kZcx, kZsi, kZdi);
+      cc.setPassedOrder(kKindMm, 0, 1, 2, 3, 4, 5, 6, 7);
+      cc.setPassedOrder(kKindVec, 0, 1, 2, 3, 4, 5, 6, 7);
+
+      cc.setPreservedRegs(kKindGp , Utils::bits(16));
+      cc.setPreservedRegs(kKindVec,~Utils::bits(n));
+      cc.setPreservedRegs(kKindMm , Utils::bits(8));
+      cc.setPreservedRegs(kKindK  , Utils::bits(8));
+      break;
+    }
+
+    default:
+      return DebugUtils::errored(kErrorInvalidArgument);
+  }
+
+  cc.setId(ccId);
+  return kErrorOk;
+}
+
+// ============================================================================
+// [asmjit::X86Internal - FuncDetail]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Internal::initFuncDetail(FuncDetail& func, const FuncSignature& sign, uint32_t gpSize) noexcept {
+  const CallConv& cc = func.getCallConv();
+  uint32_t archType = cc.getArchType();
+
+  uint32_t i;
+  uint32_t argCount = func.getArgCount();
+
+  if (func.getRetCount() != 0) {
+    uint32_t typeId = func._rets[0].getTypeId();
+    switch (typeId) {
+      case TypeId::kI64:
+      case TypeId::kU64: {
+        if (archType == ArchInfo::kTypeX86) {
+          // Convert a 64-bit return to two 32-bit returns.
+          func._retCount = 2;
+          typeId -= 2;
+
+          // 64-bit value is returned in EDX:EAX on X86.
+          func._rets[0].initReg(typeId, X86Gp::kRegGpd, X86Gp::kIdAx);
+          func._rets[1].initReg(typeId, X86Gp::kRegGpd, X86Gp::kIdDx);
+          break;
+        }
+        else {
+          func._rets[0].initReg(typeId, X86Gp::kRegGpq, X86Gp::kIdAx);
+        }
+        break;
+      }
+
+      case TypeId::kI8:
+      case TypeId::kU8:
+      case TypeId::kI16:
+      case TypeId::kU16:
+      case TypeId::kI32:
+      case TypeId::kU32: {
+        func._rets[0].assignToReg(X86Gp::kRegGpd, X86Gp::kIdAx);
+        break;
+      }
+
+      case TypeId::kF32:
+      case TypeId::kF64: {
+        uint32_t regType = (archType == ArchInfo::kTypeX86) ? X86Reg::kRegFp : X86Reg::kRegXmm;
+        func._rets[0].assignToReg(regType, 0);
+        break;
+      }
+
+      case TypeId::kF80: {
+        // 80-bit floats are always returned by FP0.
+        func._rets[0].assignToReg(X86Reg::kRegFp, 0);
+        break;
+      }
+
+      case TypeId::kMmx32:
+      case TypeId::kMmx64: {
+        // On X64 MM register(s) are returned through XMM or GPQ (Win64).
+        uint32_t regType = X86Reg::kRegMm;
+        if (archType != ArchInfo::kTypeX86)
+          regType = cc.getAlgorithm() == CallConv::kAlgorithmDefault ? X86Reg::kRegXmm : X86Reg::kRegGpq;
+
+        func._rets[0].assignToReg(regType, 0);
+        break;
+      }
+
+      default: {
+        func._rets[0].assignToReg(x86VecTypeIdToRegType(typeId), 0);
+        break;
+      }
+    }
+  }
+
+  uint32_t stackBase = gpSize;
+  uint32_t stackOffset = stackBase + cc._spillZoneSize;
+
+  if (cc.getAlgorithm() == CallConv::kAlgorithmDefault) {
+    uint32_t gpzPos = 0;
+    uint32_t vecPos = 0;
+
+    for (i = 0; i < argCount; i++) {
+      FuncDetail::Value& arg = func._args[i];
+      uint32_t typeId = arg.getTypeId();
+
+      if (TypeId::isInt(typeId)) {
+        uint32_t regId = gpzPos < CallConv::kNumRegArgsPerKind ? cc._passedOrder[X86Reg::kKindGp].id[gpzPos] : Globals::kInvalidRegId;
+        if (regId != Globals::kInvalidRegId) {
+          uint32_t regType = (typeId <= TypeId::kU32)
+            ? X86Reg::kRegGpd
+            : X86Reg::kRegGpq;
+          arg.assignToReg(regType, regId);
+          func.addUsedRegs(X86Reg::kKindGp, Utils::mask(regId));
+          gpzPos++;
+        }
+        else {
+          uint32_t size = std::max<uint32_t>(TypeId::sizeOf(typeId), gpSize);
+          arg.assignToStack(stackOffset);
+          stackOffset += size;
+        }
+        continue;
+      }
+
+      if (TypeId::isFloat(typeId) || TypeId::isVec(typeId)) {
+        uint32_t regId = vecPos < CallConv::kNumRegArgsPerKind ? cc._passedOrder[X86Reg::kKindVec].id[vecPos] : Globals::kInvalidRegId;
+
+        // If this is a float, but `floatByVec` is false, we have to pass by stack.
+        if (TypeId::isFloat(typeId) && !cc.hasFlag(CallConv::kFlagPassFloatsByVec))
+          regId = Globals::kInvalidRegId;
+
+        if (regId != Globals::kInvalidRegId) {
+          arg.initReg(typeId, x86VecTypeIdToRegType(typeId), regId);
+          func.addUsedRegs(X86Reg::kKindVec, Utils::mask(regId));
+          vecPos++;
+        }
+        else {
+          int32_t size = TypeId::sizeOf(typeId);
+          arg.assignToStack(stackOffset);
+          stackOffset += size;
+        }
+        continue;
+      }
+    }
+  }
+
+  if (cc.getAlgorithm() == CallConv::kAlgorithmWin64) {
+    for (i = 0; i < argCount; i++) {
+      FuncDetail::Value& arg = func._args[i];
+
+      uint32_t typeId = arg.getTypeId();
+      uint32_t size = TypeId::sizeOf(typeId);
+
+      if (TypeId::isInt(typeId) || TypeId::isMmx(typeId)) {
+        uint32_t regId = i < CallConv::kNumRegArgsPerKind ? cc._passedOrder[X86Reg::kKindGp].id[i] : Globals::kInvalidRegId;
+        if (regId != Globals::kInvalidRegId) {
+          uint32_t regType = (size <= 4 && !TypeId::isMmx(typeId))
+            ? X86Reg::kRegGpd
+            : X86Reg::kRegGpq;
+
+          arg.assignToReg(regType, regId);
+          func.addUsedRegs(X86Reg::kKindGp, Utils::mask(regId));
+        }
+        else {
+          arg.assignToStack(stackOffset);
+          stackOffset += gpSize;
+        }
+        continue;
+      }
+
+      if (TypeId::isFloat(typeId) || TypeId::isVec(typeId)) {
+        uint32_t regId = i < CallConv::kNumRegArgsPerKind ? cc._passedOrder[X86Reg::kKindVec].id[i] : Globals::kInvalidRegId;
+        if (regId != Globals::kInvalidRegId && (TypeId::isFloat(typeId) || cc.hasFlag(CallConv::kFlagVectorCall))) {
+          uint32_t regType = x86VecTypeIdToRegType(typeId);
+          uint32_t regId = cc._passedOrder[X86Reg::kKindVec].id[i];
+
+          arg.assignToReg(regType, regId);
+          func.addUsedRegs(X86Reg::kKindVec, Utils::mask(regId));
+        }
+        else {
+          arg.assignToStack(stackOffset);
+          stackOffset += 8; // Always 8 bytes (float/double).
+        }
+        continue;
+      }
+    }
+  }
+
+  func._argStackSize = stackOffset - stackBase;
+  return kErrorOk;
+}
+
+// ============================================================================
+// [asmjit::X86Internal - FrameLayout]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Internal::initFrameLayout(FuncFrameLayout& layout, const FuncDetail& func, const FuncFrameInfo& ffi) noexcept {
+  layout.reset();
+
+  uint32_t kind;
+  uint32_t gpSize = (func.getCallConv().getArchType() == ArchInfo::kTypeX86) ? 4 : 8;
+
+  // Calculate a bit-mask of all registers that must be saved & restored.
+  for (kind = 0; kind < Globals::kMaxVRegKinds; kind++)
+    layout._savedRegs[kind] = (ffi.getDirtyRegs(kind) & ~func.getPassedRegs(kind)) & func.getPreservedRegs(kind);
+
+  // Include EBP|RBP if the function preserves the frame-pointer.
+  if (ffi.hasPreservedFP()) {
+    layout._preservedFP = true;
+    layout._savedRegs[X86Reg::kKindGp] |= Utils::mask(X86Gp::kIdBp);
+  }
+
+  // Exclude ESP/RSP - this register is never included in saved-regs.
+  layout._savedRegs[X86Reg::kKindGp] &= ~Utils::mask(X86Gp::kIdSp);
+
+  // Calculate the final stack alignment.
+  uint32_t stackAlignment =
+    std::max<uint32_t>(
+      std::max<uint32_t>(
+        ffi.getStackFrameAlignment(),
+        ffi.getCallFrameAlignment()),
+      func.getCallConv().getNaturalStackAlignment());
+  layout._stackAlignment = static_cast<uint8_t>(stackAlignment);
+
+  // Calculate if dynamic stack alignment is required. If true the function has
+  // to align stack dynamically to match `_stackAlignment` and would require to
+  // access its stack-based arguments through `_stackArgsRegId`.
+  bool dsa = stackAlignment > func.getCallConv().getNaturalStackAlignment() && stackAlignment >= 16;
+  layout._dynamicAlignment = dsa;
+
+  // This flag describes if the prolog inserter must store the previous ESP|RSP
+  // to stack so the epilog inserter can load the stack from it before returning.
+  bool dsaSlotUsed = dsa && !ffi.hasPreservedFP();
+  layout._dsaSlotUsed = dsaSlotUsed;
+
+  // These two are identical if the function doesn't align its stack dynamically.
+  uint32_t stackArgsRegId = ffi.getStackArgsRegId();
+  if (stackArgsRegId == Globals::kInvalidRegId)
+    stackArgsRegId = X86Gp::kIdSp;
+
+  // Fix stack arguments base-register from ESP|RSP to EBP|RBP in case it was
+  // not picked before and the function performs dynamic stack alignment.
+  if (dsa && stackArgsRegId == X86Gp::kIdSp)
+    stackArgsRegId = X86Gp::kIdBp;
+
+  if (stackArgsRegId != X86Gp::kIdSp)
+    layout._savedRegs[X86Reg::kKindGp] |= Utils::mask(stackArgsRegId) & func.getPreservedRegs(X86Gp::kKindGp);
+
+  layout._stackBaseRegId = X86Gp::kIdSp;
+  layout._stackArgsRegId = static_cast<uint8_t>(stackArgsRegId);
+
+  // Setup stack size used to save preserved registers.
+  layout._gpStackSize  = Utils::bitCount(layout.getSavedRegs(X86Reg::kKindGp )) * gpSize;
+  layout._vecStackSize = Utils::bitCount(layout.getSavedRegs(X86Reg::kKindVec)) * 16 +
+                         Utils::bitCount(layout.getSavedRegs(X86Reg::kKindMm )) *  8 ;
+
+  uint32_t v = 0;                        // The beginning of the stack frame, aligned to CallFrame alignment.
+  v += ffi._callFrameSize;               // Count '_callFrameSize'  <- This is used to call functions.
+  v  = Utils::alignTo(v, stackAlignment);// Align to function's SA
+
+  layout._stackBaseOffset = v;           // Store '_stackBaseOffset'<- Function's own stack starts here..
+  v += ffi._stackFrameSize;              // Count '_stackFrameSize' <- Function's own stack ends here.
+
+  // If the function is aligned, calculate the alignment necessary to store
+  // vector registers, and set `FuncFrameInfo::kX86FlagAlignedVecSR` to inform
+  // PrologEpilog inserter that it can use instructions to perform aligned
+  // stores/loads to save/restore VEC registers.
+  if (stackAlignment >= 16 && layout._vecStackSize) {
+    v = Utils::alignTo(v, 16);           // Align '_vecStackOffset'.
+    layout._alignedVecSR = true;
+  }
+
+  layout._vecStackOffset = v;            // Store '_vecStackOffset' <- Functions VEC Save|Restore starts here.
+  v += layout._vecStackSize;             // Count '_vecStackSize'   <- Functions VEC Save|Restore ends here.
+
+  if (dsaSlotUsed) {
+    layout._dsaSlot = v;                 // Store '_dsaSlot'        <- Old stack pointer is stored here.
+    v += gpSize;
+  }
+
+  // The return address should be stored after GP save/restore regs. It has
+  // the same size as `gpSize` (basically the native register/pointer size).
+  // We don't adjust it now as `v` now contains the exact size that the
+  // function requires to adjust (call frame + stack frame, vec stack size).
+  // The stack (if we consider this size) is misaligned now, as it's always
+  // aligned before the function call - when `call()` is executed it pushes
+  // the current EIP|RIP onto the stack, and misaligns it by 12 or 8 bytes
+  // (depending on the architecture). So count number of bytes needed to align
+  // it up to the function's CallFrame (the beginning).
+  if (v || ffi.hasCalls())
+    v += Utils::alignDiff(v + layout._gpStackSize + gpSize, stackAlignment);
+
+  layout._stackAdjustment = v;           // Store '_stackAdjustment'<- SA used by 'add zsp, SA' and 'sub zsp, SA'.
+  layout._gpStackOffset = v;             // Store '_gpStackOffset'  <- Functions GP Save|Restore starts here.
+  v += layout._gpStackSize;              // Count '_gpStackSize'    <- Functions GP Save|Restore ends here.
+
+  v += gpSize;                           // Count 'ReturnAddress'.
+  v += func.getSpillZoneSize();          // Count 'SpillZoneSize'.
+
+  // Calculate where function arguments start, relative to the stackArgsRegId.
+  // If the register that will be used to access arguments passed by stack is
+  // ESP|RSP then it's exactly where we are now, otherwise we must calculate
+  // how many 'push regs' we did and adjust it based on that.
+  uint32_t stackArgsOffset = v;
+  if (stackArgsRegId != X86Gp::kIdSp) {
+    if (ffi.hasPreservedFP())
+      stackArgsOffset = gpSize;
+    else
+      stackArgsOffset = layout._gpStackSize;
+  }
+  layout._stackArgsOffset = stackArgsOffset;
+
+  // If the function does dynamic stack adjustment then the stack-adjustment
+  // must be aligned.
+  if (dsa)
+    layout._stackAdjustment = Utils::alignTo(layout._stackAdjustment, stackAlignment);
+
+  // Initialize variables based on CallConv flags.
+  if (func.hasFlag(CallConv::kFlagCalleePopsStack))
+    layout._calleeStackCleanup = static_cast<uint16_t>(func.getArgStackSize());
+
+  // Initialize variables based on FFI flags.
+  layout._mmxCleanup = ffi.hasMmxCleanup();
+  layout._avxEnabled = ffi.isAvxEnabled();
+  layout._avxCleanup = ffi.hasAvxCleanup();
+
+  return kErrorOk;
+}
+
+// ============================================================================
+// [asmjit::X86Internal - ArgsToFrameInfo]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Internal::argsToFrameInfo(const FuncArgsMapper& args, FuncFrameInfo& ffi) noexcept {
+  X86FuncArgsContext ctx;
+  ASMJIT_PROPAGATE(ctx.initWorkData(args, ffi._dirtyRegs, ffi.hasPreservedFP()));
+
+  ASMJIT_PROPAGATE(ctx.markDstRegsDirty(ffi));
+  ASMJIT_PROPAGATE(ctx.markRegsForSwaps(ffi));
+  ASMJIT_PROPAGATE(ctx.markStackArgsReg(ffi));
+  return kErrorOk;
+}
+
+// ============================================================================
+// [asmjit::X86Internal - Emit Helpers]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Internal::emitRegMove(X86Emitter* emitter,
+  const Operand_& dst_,
+  const Operand_& src_, uint32_t typeId, bool avxEnabled, const char* comment) {
+
+  // Invalid or abstract TypeIds are not allowed.
+  ASMJIT_ASSERT(TypeId::isValid(typeId) && !TypeId::isAbstract(typeId));
+
+  Operand dst(dst_);
+  Operand src(src_);
+
+  uint32_t instId = Inst::kIdNone;
+  uint32_t memFlags = 0;
+
+  enum MemFlags {
+    kDstMem = 0x1,
+    kSrcMem = 0x2
+  };
+
+  // Detect memory operands and patch them to have the same size as the register.
+  // CodeCompiler always sets memory size of allocs and spills, so it shouldn't
+  // be really necessary, however, after this function was separated from Compiler
+  // it's better to make sure that the size is always specified, as we can use
+  // 'movzx' and 'movsx' that rely on it.
+  if (dst.isMem()) { memFlags |= kDstMem; dst.as<X86Mem>().setSize(src.getSize()); }
+  if (src.isMem()) { memFlags |= kSrcMem; src.as<X86Mem>().setSize(dst.getSize()); }
+
+  switch (typeId) {
+    case TypeId::kI8:
+    case TypeId::kU8:
+    case TypeId::kI16:
+    case TypeId::kU16:
+      // Special case - 'movzx' load.
+      if (memFlags & kSrcMem) {
+        instId = X86Inst::kIdMovzx;
+        dst.setSignature(X86RegTraits<X86Reg::kRegGpd>::kSignature);
+      }
+      else if (!memFlags) {
+        // Change both destination and source registers to GPD (safer, no dependencies).
+        dst.setSignature(X86RegTraits<X86Reg::kRegGpd>::kSignature);
+        src.setSignature(X86RegTraits<X86Reg::kRegGpd>::kSignature);
+      }
+      ASMJIT_FALLTHROUGH;
+
+    case TypeId::kI32:
+    case TypeId::kU32:
+    case TypeId::kI64:
+    case TypeId::kU64:
+      instId = X86Inst::kIdMov;
+      break;
+
+    case TypeId::kMmx32:
+      instId = X86Inst::kIdMovd;
+      if (memFlags) break;
+      ASMJIT_FALLTHROUGH;
+    case TypeId::kMmx64 : instId = X86Inst::kIdMovq ; break;
+    case TypeId::kMask8 : instId = X86Inst::kIdKmovb; break;
+    case TypeId::kMask16: instId = X86Inst::kIdKmovw; break;
+    case TypeId::kMask32: instId = X86Inst::kIdKmovd; break;
+    case TypeId::kMask64: instId = X86Inst::kIdKmovq; break;
+
+    default: {
+      uint32_t elementTypeId = TypeId::elementOf(typeId);
+      if (TypeId::isVec32(typeId) && memFlags) {
+        if (elementTypeId == TypeId::kF32)
+          instId = avxEnabled ? X86Inst::kIdVmovss : X86Inst::kIdMovss;
+        else
+          instId = avxEnabled ? X86Inst::kIdVmovd : X86Inst::kIdMovd;
+        break;
+      }
+
+      if (TypeId::isVec64(typeId) && memFlags) {
+        if (elementTypeId == TypeId::kF64)
+          instId = avxEnabled ? X86Inst::kIdVmovsd : X86Inst::kIdMovsd;
+        else
+          instId = avxEnabled ? X86Inst::kIdVmovq : X86Inst::kIdMovq;
+        break;
+      }
+
+      if (elementTypeId == TypeId::kF32)
+        instId = avxEnabled ? X86Inst::kIdVmovaps : X86Inst::kIdMovaps;
+      else if (elementTypeId == TypeId::kF64)
+        instId = avxEnabled ? X86Inst::kIdVmovapd : X86Inst::kIdMovapd;
+      else if (typeId <= TypeId::_kVec256End)
+        instId = avxEnabled ? X86Inst::kIdVmovdqa : X86Inst::kIdMovdqa;
+      else if (elementTypeId <= TypeId::kU32)
+        instId = X86Inst::kIdVmovdqa32;
+      else
+        instId = X86Inst::kIdVmovdqa64;
+      break;
+    }
+  }
+
+  if (!instId)
+    return DebugUtils::errored(kErrorInvalidState);
+
+  emitter->setInlineComment(comment);
+  return emitter->emit(instId, dst, src);
+}
+
+ASMJIT_FAVOR_SIZE Error X86Internal::emitArgMove(X86Emitter* emitter,
+  const X86Reg& dst_, uint32_t dstTypeId,
+  const Operand_& src_, uint32_t srcTypeId, bool avxEnabled, const char* comment) {
+
+  // Deduce optional `dstTypeId`, which may be `TypeId::kVoid` in some cases.
+  if (!dstTypeId) dstTypeId = x86OpData.archRegs.regTypeToTypeId[dst_.getType()];
+
+  // Invalid or abstract TypeIds are not allowed.
+  ASMJIT_ASSERT(TypeId::isValid(dstTypeId) && !TypeId::isAbstract(dstTypeId));
+  ASMJIT_ASSERT(TypeId::isValid(srcTypeId) && !TypeId::isAbstract(srcTypeId));
+
+  X86Reg dst(dst_);
+  Operand src(src_);
+
+  uint32_t dstSize = TypeId::sizeOf(dstTypeId);
+  uint32_t srcSize = TypeId::sizeOf(srcTypeId);
+
+  int32_t instId = Inst::kIdNone;
+
+  // Not a real loop, just 'break' is nicer than 'goto'.
+  for (;;) {
+    if (TypeId::isInt(dstTypeId)) {
+      if (TypeId::isInt(srcTypeId)) {
+        instId = X86Inst::kIdMovsx;
+        uint32_t typeOp = (dstTypeId << 8) | srcTypeId;
+
+        // Sign extend by using 'movsx'.
+        if (typeOp == ((TypeId::kI16 << 8) | TypeId::kI8 ) ||
+            typeOp == ((TypeId::kI32 << 8) | TypeId::kI8 ) ||
+            typeOp == ((TypeId::kI32 << 8) | TypeId::kI16) ||
+            typeOp == ((TypeId::kI64 << 8) | TypeId::kI8 ) ||
+            typeOp == ((TypeId::kI64 << 8) | TypeId::kI16)) break;
+
+        // Sign extend by using 'movsxd'.
+        instId = X86Inst::kIdMovsxd;
+        if (typeOp == ((TypeId::kI64 << 8) | TypeId::kI32)) break;
+      }
+
+      if (TypeId::isInt(srcTypeId) || src_.isMem()) {
+        // Zero extend by using 'movzx' or 'mov'.
+        if (dstSize <= 4 && srcSize < 4) {
+          instId = X86Inst::kIdMovzx;
+          dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
+        }
+        else {
+          // We should have caught all possibilities where `srcSize` is less
+          // than 4, so we don't have to worry about 'movzx' anymore. Minimum
+          // size is enough to determine if we want 32-bit or 64-bit move.
+          instId = X86Inst::kIdMov;
+          srcSize = std::min(srcSize, dstSize);
+
+          dst.setSignature(srcSize == 4 ? X86Reg::signatureOfT<X86Reg::kRegGpd>()
+                                        : X86Reg::signatureOfT<X86Reg::kRegGpq>());
+          if (src.isReg()) src.setSignature(dst.getSignature());
+        }
+        break;
+      }
+
+      // NOTE: The previous branch caught all memory sources, from here it's
+      // always register to register conversion, so catch the remaining cases.
+      srcSize = std::min(srcSize, dstSize);
+
+      if (TypeId::isMmx(srcTypeId)) {
+        // 64-bit move.
+        instId = X86Inst::kIdMovq;
+        if (srcSize == 8) break;
+
+        // 32-bit move.
+        instId = X86Inst::kIdMovd;
+        dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
+        break;
+      }
+
+      if (TypeId::isMask(srcTypeId)) {
+        instId = X86Inst::kmovIdFromSize(srcSize);
+        dst.setSignature(srcSize <= 4 ? X86Reg::signatureOfT<X86Reg::kRegGpd>()
+                                      : X86Reg::signatureOfT<X86Reg::kRegGpq>());
+        break;
+      }
+
+      if (TypeId::isVec(srcTypeId)) {
+        // 64-bit move.
+        instId = avxEnabled ? X86Inst::kIdVmovq : X86Inst::kIdMovq;
+        if (srcSize == 8) break;
+
+        // 32-bit move.
+        instId = avxEnabled ? X86Inst::kIdVmovd : X86Inst::kIdMovd;
+        dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
+        break;
+      }
+    }
+
+    if (TypeId::isMmx(dstTypeId)) {
+      instId = X86Inst::kIdMovq;
+      srcSize = std::min(srcSize, dstSize);
+
+      if (TypeId::isInt(srcTypeId) || src.isMem()) {
+        // 64-bit move.
+        if (srcSize == 8) break;
+
+        // 32-bit move.
+        instId = X86Inst::kIdMovd;
+        if (src.isReg()) src.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
+        break;
+      }
+
+      if (TypeId::isMmx(srcTypeId)) break;
+
+      // NOTE: This will hurt if `avxEnabled`.
+      instId = X86Inst::kIdMovdq2q;
+      if (TypeId::isVec(srcTypeId)) break;
+    }
+
+    if (TypeId::isMask(dstTypeId)) {
+      srcSize = std::min(srcSize, dstSize);
+
+      if (TypeId::isInt(srcTypeId) || TypeId::isMask(srcTypeId) || src.isMem()) {
+        instId = X86Inst::kmovIdFromSize(srcSize);
+        if (X86Reg::isGp(src) && srcSize <= 4) src.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
+        break;
+      }
+    }
+
+    if (TypeId::isVec(dstTypeId)) {
+      // By default set destination to XMM, will be set to YMM|ZMM if needed.
+      dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegXmm>());
+
+      // NOTE: This will hurt if `avxEnabled`.
+      if (X86Reg::isMm(src)) {
+        // 64-bit move.
+        instId = X86Inst::kIdMovq2dq;
+        break;
+      }
+
+      // Argument conversion.
+      uint32_t dstElement = TypeId::elementOf(dstTypeId);
+      uint32_t srcElement = TypeId::elementOf(srcTypeId);
+
+      if (dstElement == TypeId::kF32 && srcElement == TypeId::kF64) {
+        srcSize = std::min(dstSize * 2, srcSize);
+        dstSize = srcSize / 2;
+
+        if (srcSize <= 8)
+          instId = avxEnabled ? X86Inst::kIdVcvtss2sd : X86Inst::kIdCvtss2sd;
+        else
+          instId = avxEnabled ? X86Inst::kIdVcvtps2pd : X86Inst::kIdCvtps2pd;
+
+        if (dstSize == 32)
+          dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegYmm>());
+        if (src.isReg())
+          src.setSignature(X86Reg::signatureOfVecBySize(srcSize));
+        break;
+      }
+
+      if (dstElement == TypeId::kF64 && srcElement == TypeId::kF32) {
+        srcSize = std::min(dstSize, srcSize * 2) / 2;
+        dstSize = srcSize * 2;
+
+        if (srcSize <= 4)
+          instId = avxEnabled ? X86Inst::kIdVcvtsd2ss : X86Inst::kIdCvtsd2ss;
+        else
+          instId = avxEnabled ? X86Inst::kIdVcvtpd2ps : X86Inst::kIdCvtpd2ps;
+
+        dst.setSignature(X86Reg::signatureOfVecBySize(dstSize));
+        if (src.isReg() && srcSize >= 32)
+          src.setSignature(X86Reg::signatureOfT<X86Reg::kRegYmm>());
+        break;
+      }
+
+      srcSize = std::min(srcSize, dstSize);
+      if (X86Reg::isGp(src) || src.isMem()) {
+        // 32-bit move.
+        if (srcSize <= 4) {
+          instId = avxEnabled ? X86Inst::kIdVmovd : X86Inst::kIdMovd;
+          if (src.isReg()) src.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
+          break;
+        }
+
+        // 64-bit move.
+        if (srcSize == 8) {
+          instId = avxEnabled ? X86Inst::kIdVmovq : X86Inst::kIdMovq;
+          break;
+        }
+      }
+
+      if (X86Reg::isVec(src) || src.isMem()) {
+        instId = avxEnabled ? X86Inst::kIdVmovaps : X86Inst::kIdMovaps;
+        uint32_t sign = X86Reg::signatureOfVecBySize(srcSize);
+
+        dst.setSignature(sign);
+        if (src.isReg()) src.setSignature(sign);
+        break;
+      }
+    }
+
+    return DebugUtils::errored(kErrorInvalidState);
+  }
+
+  if (src.isMem())
+    src.as<X86Mem>().setSize(srcSize);
+
+  emitter->setInlineComment(comment);
+  return emitter->emit(instId, dst, src);
+}
+
+// ============================================================================
+// [asmjit::X86Internal - Emit Prolog & Epilog]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Internal::emitProlog(X86Emitter* emitter, const FuncFrameLayout& layout) {
+  uint32_t gpSaved = layout.getSavedRegs(X86Reg::kKindGp);
+
+  X86Gp zsp = emitter->zsp();   // ESP|RSP register.
+  X86Gp zbp = emitter->zbp();   // EBP|RBP register.
+  X86Gp gpReg = emitter->zsp(); // General purpose register (temporary).
+  X86Gp saReg = emitter->zsp(); // Stack-arguments base register.
+
+  // Emit: 'push zbp'
+  //       'mov  zbp, zsp'.
+  if (layout.hasPreservedFP()) {
+    gpSaved &= ~Utils::mask(X86Gp::kIdBp);
+    ASMJIT_PROPAGATE(emitter->push(zbp));
+    ASMJIT_PROPAGATE(emitter->mov(zbp, zsp));
+  }
+
+  // Emit: 'push gp' sequence.
+  if (gpSaved) {
+    for (uint32_t i = gpSaved, regId = 0; i; i >>= 1, regId++) {
+      if (!(i & 0x1)) continue;
+      gpReg.setId(regId);
+      ASMJIT_PROPAGATE(emitter->push(gpReg));
+    }
+  }
+
+  // Emit: 'mov saReg, zsp'.
+  uint32_t stackArgsRegId = layout.getStackArgsRegId();
+  if (stackArgsRegId != Globals::kInvalidRegId && stackArgsRegId != X86Gp::kIdSp) {
+    saReg.setId(stackArgsRegId);
+    if (!(layout.hasPreservedFP() && stackArgsRegId == X86Gp::kIdBp))
+      ASMJIT_PROPAGATE(emitter->mov(saReg, zsp));
+  }
+
+  // Emit: 'and zsp, StackAlignment'.
+  if (layout.hasDynamicAlignment())
+    ASMJIT_PROPAGATE(emitter->and_(zsp, -static_cast<int32_t>(layout.getStackAlignment())));
+
+  // Emit: 'sub zsp, StackAdjustment'.
+  if (layout.hasStackAdjustment())
+    ASMJIT_PROPAGATE(emitter->sub(zsp, layout.getStackAdjustment()));
+
+  // Emit: 'mov [zsp + dsaSlot], saReg'.
+  if (layout.hasDynamicAlignment() && layout.hasDsaSlotUsed()) {
+    X86Mem saMem = x86::ptr(zsp, layout._dsaSlot);
+    ASMJIT_PROPAGATE(emitter->mov(saMem, saReg));
+  }
+
+  // Emit 'movaps|movups [zsp + X], xmm0..15'.
+  uint32_t xmmSaved = layout.getSavedRegs(X86Reg::kKindVec);
+  if (xmmSaved) {
+    X86Mem vecBase = x86::ptr(zsp, layout.getVecStackOffset());
+    X86Reg vecReg = x86::xmm(0);
+
+    uint32_t vecInst = x86GetXmmMovInst(layout);
+    uint32_t vecSize = 16;
+
+    for (uint32_t i = xmmSaved, regId = 0; i; i >>= 1, regId++) {
+      if (!(i & 0x1)) continue;
+      vecReg.setId(regId);
+      ASMJIT_PROPAGATE(emitter->emit(vecInst, vecBase, vecReg));
+      vecBase.addOffsetLo32(static_cast<int32_t>(vecSize));
+    }
+  }
+
+  return kErrorOk;
+}
+
+ASMJIT_FAVOR_SIZE Error X86Internal::emitEpilog(X86Emitter* emitter, const FuncFrameLayout& layout) {
+  uint32_t i;
+  uint32_t regId;
+
+  uint32_t gpSize = emitter->getGpSize();
+  uint32_t gpSaved = layout.getSavedRegs(X86Reg::kKindGp);
+
+  X86Gp zsp = emitter->zsp();   // ESP|RSP register.
+  X86Gp zbp = emitter->zbp();   // EBP|RBP register.
+  X86Gp gpReg = emitter->zsp(); // General purpose register (temporary).
+
+  // Don't emit 'pop zbp' in the pop sequence, this case is handled separately.
+  if (layout.hasPreservedFP()) gpSaved &= ~Utils::mask(X86Gp::kIdBp);
+
+  // Emit 'movaps|movups xmm0..15, [zsp + X]'.
+  uint32_t xmmSaved = layout.getSavedRegs(X86Reg::kKindVec);
+  if (xmmSaved) {
+    X86Mem vecBase = x86::ptr(zsp, layout.getVecStackOffset());
+    X86Reg vecReg = x86::xmm(0);
+
+    uint32_t vecInst = x86GetXmmMovInst(layout);
+    uint32_t vecSize = 16;
+
+    for (i = xmmSaved, regId = 0; i; i >>= 1, regId++) {
+      if (!(i & 0x1)) continue;
+      vecReg.setId(regId);
+      ASMJIT_PROPAGATE(emitter->emit(vecInst, vecReg, vecBase));
+      vecBase.addOffsetLo32(static_cast<int32_t>(vecSize));
+    }
+  }
+
+  // Emit 'emms' and 'vzeroupper'.
+  if (layout.hasMmxCleanup()) ASMJIT_PROPAGATE(emitter->emms());
+  if (layout.hasAvxCleanup()) ASMJIT_PROPAGATE(emitter->vzeroupper());
+
+  if (layout.hasPreservedFP()) {
+    // Emit 'mov zsp, zbp' or 'lea zsp, [zbp - x]'
+    int32_t count = static_cast<int32_t>(layout.getGpStackSize() - gpSize);
+    if (!count)
+      ASMJIT_PROPAGATE(emitter->mov(zsp, zbp));
+    else
+      ASMJIT_PROPAGATE(emitter->lea(zsp, x86::ptr(zbp, -count)));
+  }
+  else {
+    if (layout.hasDynamicAlignment() && layout.hasDsaSlotUsed()) {
+      // Emit 'mov zsp, [zsp + DsaSlot]'.
+      X86Mem saMem = x86::ptr(zsp, layout._dsaSlot);
+      ASMJIT_PROPAGATE(emitter->mov(zsp, saMem));
+    }
+    else if (layout.hasStackAdjustment()) {
+      // Emit 'add zsp, StackAdjustment'.
+      ASMJIT_PROPAGATE(emitter->add(zsp, static_cast<int32_t>(layout.getStackAdjustment())));
+    }
+  }
+
+  // Emit 'pop gp' sequence.
+  if (gpSaved) {
+    i = gpSaved;
+    regId = 16;
+
+    do {
+      regId--;
+      if (i & 0x8000) {
+        gpReg.setId(regId);
+        ASMJIT_PROPAGATE(emitter->pop(gpReg));
+      }
+      i <<= 1;
+    } while (regId != 0);
+  }
+
+  // Emit 'pop zbp'.
+  if (layout.hasPreservedFP()) ASMJIT_PROPAGATE(emitter->pop(zbp));
+
+  // Emit 'ret' or 'ret x'.
+  if (layout.hasCalleeStackCleanup())
+    ASMJIT_PROPAGATE(emitter->emit(X86Inst::kIdRet, static_cast<int>(layout.getCalleeStackCleanup())));
+  else
+    ASMJIT_PROPAGATE(emitter->emit(X86Inst::kIdRet));
+
+  return kErrorOk;
+}
+
+// ============================================================================
+// [asmjit::X86Internal - AllocArgs]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Internal::allocArgs(X86Emitter* emitter, const FuncFrameLayout& layout, const FuncArgsMapper& args) {
+  typedef X86FuncArgsContext::SrcArg SrcArg;
+  typedef X86FuncArgsContext::DstArg DstArg;
+  typedef X86FuncArgsContext::WorkData WorkData;
+  enum { kMaxVRegKinds = Globals::kMaxVRegKinds };
+
+  uint32_t i;
+  const FuncDetail& func = *args.getFuncDetail();
+
+  X86FuncArgsContext ctx;
+  ASMJIT_PROPAGATE(ctx.initWorkData(args, layout._savedRegs, layout.hasPreservedFP()));
+
+  // We must honor AVX if it's enabled.
+  bool avxEnabled = layout.isAvxEnabled();
+
+  // Free registers that can be used as temporaries and during shuffling.
+  // We initialize them to match all workRegs (registers that can be used
+  // by the function) except source regs, which are used to pass arguments.
+  // Free registers are changed during shuffling - when an argument is moved
+  // to the final register then the register itself is removed from freeRegs
+  // (it can't be altered anymore during shuffling).
+  uint32_t freeRegs[kMaxVRegKinds];
+  for (i = 0; i < kMaxVRegKinds; i++)
+    freeRegs[i] = ctx._workData[i].workRegs & ~ctx._workData[i].srcRegs;
+
+  // This is an iterative process that runs until there is a work to do. When
+  // one register is moved it can create space for another move. Such moves can
+  // depend on each other so the algorithm may run multiple times before all
+  // arguments are in place. This part does only register-to-register work,
+  // arguments moved from stack-to-register area handled later.
+  for (;;) {
+    bool hasWork = false; // Do we have a work to do?
+    bool didWork = false; // If we did something...
+
+    uint32_t dstRegKind = kMaxVRegKinds;
+    do {
+      WorkData& wd = ctx._workData[--dstRegKind];
+      if (wd.numOps > wd.numStackArgs) {
+        hasWork = true;
+
+        // Iterate over all destination regs and check if we can do something.
+        // We always go from destination to source, never the opposite.
+        uint32_t regsToDo = wd.dstRegs;
+        do {
+          // If there is a work to do there has to be at least one dstReg.
+          ASMJIT_ASSERT(regsToDo != 0);
+          uint32_t dstRegId = Utils::findFirstBit(regsToDo);
+          uint32_t dstRegMask = Utils::mask(dstRegId);
+
+          uint32_t argIndex = wd.argIndex[dstRegId];
+          const DstArg& dstArg = args.getArg(argIndex);
+          const SrcArg& srcArg = func.getArg(argIndex);
+
+          if (srcArg.byReg()) {
+            uint32_t srcRegType = srcArg.getRegType();
+            uint32_t srcRegKind = X86Reg::kindOf(srcRegType);
+
+            if (freeRegs[dstRegKind] & dstRegMask) {
+              X86Reg dstReg(X86Reg::fromTypeAndId(dstArg.getRegType(), dstRegId));
+              X86Reg srcReg(X86Reg::fromTypeAndId(srcRegType, srcArg.getRegId()));
+
+              ASMJIT_PROPAGATE(
+                emitArgMove(emitter,
+                  dstReg, dstArg.getTypeId(),
+                  srcReg, srcArg.getTypeId(), avxEnabled));
+              freeRegs[dstRegKind] ^= dstRegMask;                     // Make the DST reg occupied.
+              freeRegs[srcRegKind] |= Utils::mask(srcArg.getRegId()); // Make the SRC reg free.
+
+              ASMJIT_ASSERT(wd.numOps >= 1);
+              wd.numOps--;
+              didWork = true;
+            }
+            else {
+              // Check if this is a swap operation.
+              if (dstRegKind == srcRegKind) {
+                uint32_t srcRegId = srcArg.getRegId();
+
+                uint32_t otherIndex = wd.argIndex[srcRegId];
+                const DstArg& otherArg = args.getArg(otherIndex);
+
+                if (otherArg.getRegId() == srcRegId && X86Reg::kindOf(otherArg.getRegType()) == dstRegKind) {
+                  // If this is GP reg it can be handled by 'xchg'.
+                  if (dstRegKind == X86Reg::kKindGp) {
+                    uint32_t highestType = std::max(dstArg.getRegType(), srcRegType);
+
+                    X86Reg dstReg = x86::gpd(dstRegId);
+                    X86Reg srcReg = x86::gpd(srcRegId);
+
+                    if (highestType == X86Reg::kRegGpq) {
+                      dstReg.setSignature(X86RegTraits<X86Reg::kRegGpq>::kSignature);
+                      srcReg.setSignature(X86RegTraits<X86Reg::kRegGpq>::kSignature);
+                    }
+                    ASMJIT_PROPAGATE(emitter->emit(X86Inst::kIdXchg, dstReg, srcReg));
+                    regsToDo &= ~Utils::mask(srcRegId);
+                    freeRegs[dstRegKind] &= ~(Utils::mask(srcRegId) | dstRegMask);
+
+                    ASMJIT_ASSERT(wd.numOps >= 2);
+                    ASMJIT_ASSERT(wd.numSwaps >= 1);
+                    wd.numOps-=2;
+                    wd.numSwaps--;
+                    didWork = true;
+                  }
+                }
+              }
+            }
+          }
+
+          // Clear the reg in `regsToDo` and continue if there are more.
+          regsToDo ^= dstRegMask;
+        } while (regsToDo);
+      }
+    } while (dstRegKind);
+
+    if (!hasWork)
+      break;
+
+    if (!didWork)
+      return DebugUtils::errored(kErrorInvalidState);
+  }
+
+  // Load arguments passed by stack into registers. This is pretty simple and
+  // it never requires multiple iterations like the previous phase.
+  if (ctx._hasStackArgs) {
+    // Base address of all arguments passed by stack.
+    X86Mem saBase = x86::ptr(emitter->gpz(layout.getStackArgsRegId()), layout.getStackArgsOffset());
+
+    uint32_t dstRegKind = kMaxVRegKinds;
+    do {
+      WorkData& wd = ctx._workData[--dstRegKind];
+      if (wd.numStackArgs) {
+        // Iterate over all destination regs and check if we can do something.
+        // We always go from destination to source, never the opposite.
+        uint32_t regsToDo = wd.dstRegs;
+        do {
+          // If there is a work to do there has to be at least one dstReg.
+          ASMJIT_ASSERT(regsToDo != 0);
+          ASMJIT_ASSERT(wd.numOps > 0);
+
+          uint32_t dstRegId = Utils::findFirstBit(regsToDo);
+          uint32_t dstRegMask = Utils::mask(dstRegId);
+
+          uint32_t argIndex = wd.argIndex[dstRegId];
+          const DstArg& dstArg = args.getArg(argIndex);
+          const SrcArg& srcArg = func.getArg(argIndex);
+
+          // Only arguments passed by stack should remain, also the destination
+          // registers must be free now (otherwise the first part of the algorithm
+          // failed). Ideally this should be assert, but it's much safer to enforce
+          // this in release as well.
+          if (!srcArg.byStack() || !(freeRegs[dstRegKind] & dstRegMask))
+            return DebugUtils::errored(kErrorInvalidState);
+
+          X86Reg dstReg = X86Reg::fromTypeAndId(dstArg.getRegType(), dstRegId);
+          X86Mem srcMem = saBase.adjusted(srcArg.getStackOffset());
+
+          ASMJIT_PROPAGATE(
+            emitArgMove(emitter,
+              dstReg, dstArg.getTypeId(),
+              srcMem, srcArg.getTypeId(), avxEnabled));
+
+          freeRegs[dstRegKind] ^= dstRegMask;
+          regsToDo ^= dstRegMask;
+          wd.numOps--;
+        } while (regsToDo);
+      }
+    } while (dstRegKind);
+  }
+
+  return kErrorOk;
+}
+
+} // asmjit namespace
+
+// [Api-End]
+#include "../asmjit_apiend.h"
+
+// [Guard]
+#endif // ASMJIT_BUILD_X86

libraries/asmjit/x86/x86internal_p.h

+// [AsmJit]
+// Complete x86/x64 JIT and Remote Assembler for C++.
+//
+// [License]
+// Zlib - See LICENSE.md file in the package.
+
+// [Guard]
+#ifndef _ASMJIT_X86_X86INTERNAL_P_H
+#define _ASMJIT_X86_X86INTERNAL_P_H
+
+#include "../asmjit_build.h"
+
+// [Dependencies]
+#include "../base/func.h"
+#include "../x86/x86emitter.h"
+#include "../x86/x86operand.h"
+
+// [Api-Begin]
+#include "../asmjit_apibegin.h"
+
+namespace asmjit {
+
+//! \addtogroup asmjit_base
+//! \{
+
+// ============================================================================
+// [asmjit::X86Internal]
+// ============================================================================
+
+//! \internal
+//!
+//! X86 utilities used at multiple places, not part of public API, not exported.
+struct X86Internal {
+  //! Initialize `CallConv` to X86/X64 specific calling convention.
+  static Error initCallConv(CallConv& cc, uint32_t ccId) noexcept;
+
+  //! Initialize `FuncDetail` to X86/X64 specific function signature.
+  static Error initFuncDetail(FuncDetail& func, const FuncSignature& sign, uint32_t gpSize) noexcept;
+
+  //! Initialize `FuncFrameLayout` from X86/X64 specific function detail and frame information.
+  static Error initFrameLayout(FuncFrameLayout& layout, const FuncDetail& func, const FuncFrameInfo& ffi) noexcept;
+
+  static Error argsToFrameInfo(const FuncArgsMapper& args, FuncFrameInfo& ffi) noexcept;
+
+  //! Emit function prolog.
+  static Error emitProlog(X86Emitter* emitter, const FuncFrameLayout& layout);
+
+  //! Emit function epilog.
+  static Error emitEpilog(X86Emitter* emitter, const FuncFrameLayout& layout);
+
+  //! Emit a pure move operation between two registers or the same type or
+  //! between a register and its home slot. This function does not handle
+  //! register conversion.
+  static Error emitRegMove(X86Emitter* emitter,
+    const Operand_& dst_,
+    const Operand_& src_, uint32_t typeId, bool avxEnabled, const char* comment = nullptr);
+
+  //! Emit move from a function argument (either register or stack) to a register.
+  //!
+  //! This function can handle the necessary conversion from one argument to
+  //! another, and from one register type to another, if it's possible. Any
+  //! attempt of conversion that requires third register of a different kind
+  //! (for example conversion from K to MMX) will fail.
+  static Error emitArgMove(X86Emitter* emitter,
+    const X86Reg& dst_, uint32_t dstTypeId,
+    const Operand_& src_, uint32_t srcTypeId, bool avxEnabled, const char* comment = nullptr);
+
+  static Error allocArgs(X86Emitter* emitter, const FuncFrameLayout& layout, const FuncArgsMapper& args);
+};
+
+//! \}
+
+} // asmjit namespace
+
+// [Api-End]
+#include "../asmjit_apiend.h"
+
+// [Guard]
+#endif // _ASMJIT_X86_X86INTERNAL_P_H

libraries/asmjit/x86/x86logging.cpp

+// [AsmJit]
+// Complete x86/x64 JIT and Remote Assembler for C++.
+//
+// [License]
+// Zlib - See LICENSE.md file in the package.
+
+// [Export]
+#define ASMJIT_EXPORTS
+
+// [Guard]
+#include "../asmjit_build.h"
+#if !defined(ASMJIT_DISABLE_LOGGING)
+
+// [Dependencies]
+#include "../base/misc_p.h"
+#include "../x86/x86inst.h"
+#include "../x86/x86logging_p.h"
+#include "../x86/x86operand.h"
+
+#if !defined(ASMJIT_DISABLE_COMPILER)
+#include "../base/codecompiler.h"
+#endif // !ASMJIT_DISABLE_COMPILER
+
+// [Api-Begin]
+#include "../asmjit_apibegin.h"
+
+namespace asmjit {
+
+// ============================================================================
+// [asmjit::X86Logging - Constants]
+// ============================================================================
+
+struct X86RegFormatInfo {
+  uint8_t count;
+  uint8_t formatIndex;
+  uint8_t specialIndex;
+  uint8_t specialCount;
+};
+
+static const char x86RegFormatStrings[] =
+  "r%ub"  "\0" // #0
+  "r%uh"  "\0" // #5
+  "r%uw"  "\0" // #10
+  "r%ud"  "\0" // #15
+  "r%u"   "\0" // #20
+  "xmm%u" "\0" // #24
+  "ymm%u" "\0" // #30
+  "zmm%u" "\0" // #36
+  "rip%u" "\0" // #42
+  "seg%u" "\0" // #48
+  "fp%u"  "\0" // #54
+  "mm%u"  "\0" // #59
+  "k%u"   "\0" // #64
+  "bnd%u" "\0" // #68
+  "cr%u"  "\0" // #74
+  "dr%u"  "\0" // #79
+
+  "rip\0"      // #84
+  "\0\0\0\0"   // #88
+  "\0\0\0\0"   // #92
+
+  "al\0\0" "cl\0\0" "dl\0\0" "bl\0\0" "spl\0"  "bpl\0"  "sil\0"  "dil\0"  // #96
+  "ah\0\0" "ch\0\0" "dh\0\0" "bh\0\0" "n/a\0"  "n/a\0"  "n/a\0"  "n/a\0"  // #128
+  "eax\0"  "ecx\0"  "edx\0"  "ebx\0"  "esp\0"  "ebp\0"  "esi\0"  "edi\0"  // #160
+  "rax\0"  "rcx\0"  "rdx\0"  "rbx\0"  "rsp\0"  "rbp\0"  "rsi\0"  "rdi\0"  // #192
+  "n/a\0"  "es\0\0" "cs\0\0" "ss\0\0" "ds\0\0" "fs\0\0" "gs\0\0" "n/a\0"; // #224
+
+template<uint32_t X>
+struct X86RegFormatInfo_T {
+  enum {
+    kFormatIndex  = X == X86Reg::kRegGpbLo ? 0   :
+                    X == X86Reg::kRegGpbHi ? 5   :
+                    X == X86Reg::kRegGpw   ? 10  :
+                    X == X86Reg::kRegGpd   ? 15  :
+                    X == X86Reg::kRegGpq   ? 20  :
+                    X == X86Reg::kRegXmm   ? 24  :
+                    X == X86Reg::kRegYmm   ? 30  :
+                    X == X86Reg::kRegZmm   ? 36  :
+                    X == X86Reg::kRegRip   ? 42  :
+                    X == X86Reg::kRegSeg   ? 48  :
+                    X == X86Reg::kRegFp    ? 54  :
+                    X == X86Reg::kRegMm    ? 59  :
+                    X == X86Reg::kRegK     ? 64  :
+                    X == X86Reg::kRegBnd   ? 68  :
+                    X == X86Reg::kRegCr    ? 74  :
+                    X == X86Reg::kRegDr    ? 79  : 0,
+
+    kSpecialIndex = X == X86Reg::kRegGpbLo ? 96  :
+                    X == X86Reg::kRegGpbHi ? 128 :
+                    X == X86Reg::kRegGpw   ? 161 :
+                    X == X86Reg::kRegGpd   ? 160 :
+                    X == X86Reg::kRegGpq   ? 192 :
+                    X == X86Reg::kRegRip   ? 84  :
+                    X == X86Reg::kRegSeg   ? 224 : 0,
+
+    kSpecialCount = X == X86Reg::kRegGpbLo ? 8   :
+                    X == X86Reg::kRegGpbHi ? 4   :
+                    X == X86Reg::kRegGpw   ? 8   :
+                    X == X86Reg::kRegGpd   ? 8   :
+                    X == X86Reg::kRegGpq   ? 8   :
+                    X == X86Reg::kRegRip   ? 1   :
+                    X == X86Reg::kRegSeg   ? 7   : 0
+  };
+};
+
+#define ASMJIT_X86_REG_FORMAT(TYPE) {      \
+  X86RegTraits<TYPE>::kCount,              \
+  X86RegFormatInfo_T<TYPE>::kFormatIndex,  \
+  X86RegFormatInfo_T<TYPE>::kSpecialIndex, \
+  X86RegFormatInfo_T<TYPE>::kSpecialCount  \
+}
+
+static const X86RegFormatInfo x86RegFormatInfo[] = {
+  ASMJIT_TABLE_16(ASMJIT_X86_REG_FORMAT, 0 ),
+  ASMJIT_TABLE_16(ASMJIT_X86_REG_FORMAT, 16)
+};
+
+static const char* x86GetAddressSizeString(uint32_t size) noexcept {
+  switch (size) {
+    case 1 : return "byte ";
+    case 2 : return "word ";
+    case 4 : return "dword ";
+    case 6 : return "fword ";
+    case 8 : return "qword ";
+    case 10: return "tword ";
+    case 16: return "oword ";
+    case 32: return "yword ";
+    case 64: return "zword ";
+    default: return "";
+  }
+}
+
+// ============================================================================
+// [asmjit::X86Logging - Format Operand]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Logging::formatOperand(
+  StringBuilder& sb,
+  uint32_t logOptions,
+  const CodeEmitter* emitter,
+  uint32_t archType,
+  const Operand_& op) noexcept {
+
+  if (op.isReg())
+    return formatRegister(sb, logOptions, emitter, archType, op.as<Reg>().getType(), op.as<Reg>().getId());
+
+  if (op.isMem()) {
+    const X86Mem& m = op.as<X86Mem>();
+    ASMJIT_PROPAGATE(sb.appendString(x86GetAddressSizeString(m.getSize())));
+
+    // Segment override prefix.
+    uint32_t seg = m.getSegmentId();
+    if (seg != X86Seg::kIdNone && seg < X86Seg::kIdCount)
+      ASMJIT_PROPAGATE(sb.appendFormat("%s:", x86RegFormatStrings + 224 + seg * 4));
+
+    ASMJIT_PROPAGATE(sb.appendChar('['));
+    if (m.isAbs())
+      ASMJIT_PROPAGATE(sb.appendString("abs "));
+
+    if (m.hasBase()) {
+      if (m.hasBaseLabel()) {
+        ASMJIT_PROPAGATE(Logging::formatLabel(sb, logOptions, emitter, m.getBaseId()));
+      }
+      else {
+        if (m.isArgHome()) ASMJIT_PROPAGATE(sb.appendString("$"));
+        if (m.isRegHome()) ASMJIT_PROPAGATE(sb.appendString("&"));
+        ASMJIT_PROPAGATE(formatRegister(sb, logOptions, emitter, archType, m.getBaseType(), m.getBaseId()));
+      }
+    }
+
+    if (m.hasIndex()) {
+      ASMJIT_PROPAGATE(sb.appendChar('+'));
+      ASMJIT_PROPAGATE(formatRegister(sb, logOptions, emitter, archType, m.getIndexType(), m.getIndexId()));
+      if (m.hasShift())
+        ASMJIT_PROPAGATE(sb.appendFormat("*%u", 1 << m.getShift()));
+    }
+
+    uint64_t off = static_cast<uint64_t>(m.getOffset());
+    if (off) {
+      uint32_t base = 10;
+      char prefix = '+';
+
+      if (static_cast<int64_t>(off) < 0) {
+        off = ~off + 1;
+        prefix = '-';
+      }
+
+      ASMJIT_PROPAGATE(sb.appendChar(prefix));
+      if ((logOptions & Logger::kOptionHexDisplacement) != 0 && off > 9) {
+        ASMJIT_PROPAGATE(sb.appendString("0x", 2));
+        base = 16;
+      }
+      ASMJIT_PROPAGATE(sb.appendUInt(off, base));
+    }
+
+    return sb.appendChar(']');
+  }
+
+  if (op.isImm()) {
+    const Imm& i = op.as<Imm>();
+    int64_t val = i.getInt64();
+
+    if ((logOptions & Logger::kOptionHexImmediate) != 0 && static_cast<uint64_t>(val) > 9)
+      return sb.appendUInt(static_cast<uint64_t>(val), 16);
+    else
+      return sb.appendInt(val, 10);
+  }
+
+  if (op.isLabel()) {
+    return Logging::formatLabel(sb, logOptions, emitter, op.getId());
+  }
+
+  return sb.appendString("<None>");
+}
+
+// ============================================================================
+// [asmjit::X86Logging - Format Immediate (Extension)]
+// ============================================================================
+
+struct ImmBits {
+  enum Mode {
+    kModeLookup = 0x0,
+    kModeFormat = 0x1
+  };
+
+  uint8_t mask;
+  uint8_t shift;
+  uint8_t mode;
+  char text[45];
+};
+
+ASMJIT_FAVOR_SIZE static Error X86Logging_formatImmShuf(StringBuilder& sb, uint32_t u8, uint32_t bits, uint32_t count) noexcept {
+  ASMJIT_PROPAGATE(sb.appendChar('<'));
+  uint32_t mask = (1 << bits) - 1;
+
+  for (uint32_t i = 0; i < count; i++, u8 >>= bits) {
+    uint32_t value = u8 & mask;
+    if (i != 0)
+      ASMJIT_PROPAGATE(sb.appendChar('|'));
+    ASMJIT_PROPAGATE(sb.appendUInt(value));
+  }
+
+  return sb.appendChar('>');
+}
+
+ASMJIT_FAVOR_SIZE static Error X86Logging_formatImmBits(StringBuilder& sb, uint32_t u8, const ImmBits* bits, uint32_t count) noexcept {
+  uint32_t n = 0;
+  char buf[64];
+
+  for (uint32_t i = 0; i < count; i++) {
+    const ImmBits& spec = bits[i];
+
+    uint32_t value = (u8 & static_cast<uint32_t>(spec.mask)) >> spec.shift;
+    const char* str = nullptr;
+
+    switch (spec.mode) {
+      case ImmBits::kModeLookup:
+        str = Utils::findPackedString(spec.text, value);
+        break;
+
+      case ImmBits::kModeFormat:
+        snprintf(buf, sizeof(buf), spec.text, static_cast<unsigned int>(value));
+        str = buf;
+        break;
+
+      default:
+        return DebugUtils::errored(kErrorInvalidState);
+    }
+
+    if (!str[0])
+      continue;
+
+    ASMJIT_PROPAGATE(sb.appendChar(++n == 1 ? '<' : '|'));
+    ASMJIT_PROPAGATE(sb.appendString(str));
+  }
+
+  return n ? sb.appendChar('>') : static_cast<Error>(kErrorOk);
+}
+
+ASMJIT_FAVOR_SIZE static Error X86Logging_formatImmText(StringBuilder& sb, uint32_t u8, uint32_t bits, uint32_t advance, const char* text, uint32_t count = 1) noexcept {
+  ASMJIT_PROPAGATE(sb.appendChar('<'));
+
+  uint32_t mask = (1 << bits) - 1;
+  uint32_t pos = 0;
+
+  for (uint32_t i = 0; i < count; i++, u8 >>= bits, pos += advance) {
+    uint32_t value = (u8 & mask) + pos;
+    if (i != 0)
+      ASMJIT_PROPAGATE(sb.appendChar('|'));
+    ASMJIT_PROPAGATE(sb.appendString(Utils::findPackedString(text, value)));
+  }
+
+  return sb.appendChar('>');
+}
+
+ASMJIT_FAVOR_SIZE static Error X86Logging_formatImmExtended(
+  StringBuilder& sb,
+  uint32_t logOptions,
+  uint32_t instId,
+  uint32_t vecSize,
+  const Imm& imm) noexcept {
+
+  static const char vcmpx[] =
+    "eq_oq\0" "lt_os\0"  "le_os\0"  "unord_q\0"  "neq_uq\0" "nlt_us\0" "nle_us\0" "ord_q\0"
+    "eq_uq\0" "nge_us\0" "ngt_us\0" "false_oq\0" "neq_oq\0" "ge_os\0"  "gt_os\0"  "true_uq\0"
+    "eq_os\0" "lt_oq\0"  "le_oq\0"  "unord_s\0"  "neq_us\0" "nlt_uq\0" "nle_uq\0" "ord_s\0"
+    "eq_us\0" "nge_uq\0" "ngt_uq\0" "false_os\0" "neq_os\0" "ge_oq\0"  "gt_oq\0"  "true_us\0";
+
+  // Try to find 7 differences...
+  static const char vpcmpx[] = "eq\0" "lt\0" "le\0" "false\0" "neq\0" "ge\0"  "gt\0"    "true\0";
+  static const char vpcomx[] = "lt\0" "le\0" "gt\0" "ge\0"    "eq\0"  "neq\0" "false\0" "true\0";
+
+  static const char vshufpd[] = "a0\0a1\0b0\0b1\0a2\0a3\0b2\0b3\0a4\0a5\0b4\0b5\0a6\0a7\0b6\0b7\0";
+  static const char vshufps[] = "a0\0a1\0a2\0a3\0a0\0a1\0a2\0a3\0b0\0b1\0b2\0b3\0b0\0b1\0b2\0b3\0";
+
+  static const ImmBits vfpclassxx[] = {
+    { 0x07, 0, ImmBits::kModeLookup, "qnan\0" "+0\0" "-0\0" "+inf\0" "-inf\0" "denormal\0" "-finite\0" "snan\0" }
+  };
+
+  static const ImmBits vgetmantxx[] = {
+    { 0x03, 0, ImmBits::kModeLookup, "[1, 2)\0" "[1/2, 2)\0" "1/2, 1)\0" "[3/4, 3/2)\0" },
+    { 0x04, 2, ImmBits::kModeLookup, "\0" "no-sign\0" },
+    { 0x08, 3, ImmBits::kModeLookup, "\0" "qnan-if-sign\0" }
+  };
+
+  static const ImmBits vmpsadbw[] = {
+    { 0x04, 2, ImmBits::kModeLookup, "blk1[0]\0" "blk1[1]\0" },
+    { 0x03, 0, ImmBits::kModeLookup, "blk2[0]\0" "blk2[1]\0" "blk2[2]\0" "blk2[3]\0" },
+    { 0x40, 6, ImmBits::kModeLookup, "blk1[4]\0" "blk1[5]\0" },
+    { 0x30, 4, ImmBits::kModeLookup, "blk2[4]\0" "blk2[5]\0" "blk2[6]\0" "blk2[7]\0" }
+  };
+
+  static const ImmBits vpclmulqdq[] = {
+    { 0x01, 0, ImmBits::kModeLookup, "lq\0" "hq\0" },
+    { 0x10, 4, ImmBits::kModeLookup, "lq\0" "hq\0" }
+  };
+
+  static const ImmBits vperm2x128[] = {
+    { 0x0B, 0, ImmBits::kModeLookup, "a0\0" "a1\0" "b0\0" "b1\0" "\0" "\0" "\0" "\0" "0\0" "0\0" "0\0" "0\0" },
+    { 0xB0, 4, ImmBits::kModeLookup, "a0\0" "a1\0" "b0\0" "b1\0" "\0" "\0" "\0" "\0" "0\0" "0\0" "0\0" "0\0" }
+  };
+
+  static const ImmBits vrangexx[] = {
+    { 0x03, 0, ImmBits::kModeLookup, "min\0" "max\0" "min-abs\0" "max-abs\0" },
+    { 0x0C, 2, ImmBits::kModeLookup, "sign=src1\0" "sign=src2\0" "sign=0\0" "sign=1\0" }
+  };
+
+  static const ImmBits vreducexx_vrndscalexx[] = {
+    { 0x07, 0, ImmBits::kModeLookup, "\0" "\0" "\0" "\0" "round\0" "floor\0" "ceil\0" "truncate\0" },
+    { 0x08, 3, ImmBits::kModeLookup, "\0" "suppress\0" },
+    { 0xF0, 4, ImmBits::kModeFormat, "len=%d" }
+  };
+
+  static const ImmBits vroundxx[] = {
+    { 0x07, 0, ImmBits::kModeLookup, "round\0" "floor\0" "ceil\0" "truncate\0" "\0" "\0" "\0" "\0" },
+    { 0x08, 3, ImmBits::kModeLookup, "\0" "inexact\0" }
+  };
+
+  uint32_t u8 = imm.getUInt8();
+  switch (instId) {
+    case X86Inst::kIdVblendpd:
+    case X86Inst::kIdBlendpd:
+      return X86Logging_formatImmShuf(sb, u8, 1, vecSize / 8);
+
+    case X86Inst::kIdVblendps:
+    case X86Inst::kIdBlendps:
+      return X86Logging_formatImmShuf(sb, u8, 1, vecSize / 4);
+
+    case X86Inst::kIdVcmppd:
+    case X86Inst::kIdVcmpps:
+    case X86Inst::kIdVcmpsd:
+    case X86Inst::kIdVcmpss:
+      return X86Logging_formatImmText(sb, u8, 5, 0, vcmpx);
+
+    case X86Inst::kIdCmppd:
+    case X86Inst::kIdCmpps:
+    case X86Inst::kIdCmpsd:
+    case X86Inst::kIdCmpss:
+      return X86Logging_formatImmText(sb, u8, 3, 0, vcmpx);
+
+    case X86Inst::kIdVdbpsadbw:
+      return X86Logging_formatImmShuf(sb, u8, 2, 4);
+
+    case X86Inst::kIdVdppd:
+    case X86Inst::kIdVdpps:
+    case X86Inst::kIdDppd:
+    case X86Inst::kIdDpps:
+      return X86Logging_formatImmShuf(sb, u8, 1, 8);
+
+    case X86Inst::kIdVmpsadbw:
+    case X86Inst::kIdMpsadbw:
+      return X86Logging_formatImmBits(sb, u8, vmpsadbw, std::min<uint32_t>(vecSize / 8, 4));
+
+    case X86Inst::kIdVpblendw:
+    case X86Inst::kIdPblendw:
+      return X86Logging_formatImmShuf(sb, u8, 1, 8);
+
+    case X86Inst::kIdVpblendd:
+      return X86Logging_formatImmShuf(sb, u8, 1, std::min<uint32_t>(vecSize / 4, 8));
+
+    case X86Inst::kIdVpclmulqdq:
+    case X86Inst::kIdPclmulqdq:
+      return X86Logging_formatImmBits(sb, u8, vpclmulqdq, ASMJIT_ARRAY_SIZE(vpclmulqdq));
+
+    case X86Inst::kIdVroundpd:
+    case X86Inst::kIdVroundps:
+    case X86Inst::kIdVroundsd:
+    case X86Inst::kIdVroundss:
+    case X86Inst::kIdRoundpd:
+    case X86Inst::kIdRoundps:
+    case X86Inst::kIdRoundsd:
+    case X86Inst::kIdRoundss:
+      return X86Logging_formatImmBits(sb, u8, vroundxx, ASMJIT_ARRAY_SIZE(vroundxx));
+
+    case X86Inst::kIdVshufpd:
+    case X86Inst::kIdShufpd:
+      return X86Logging_formatImmText(sb, u8, 1, 2, vshufpd, std::min<uint32_t>(vecSize / 8, 8));
+
+    case X86Inst::kIdVshufps:
+    case X86Inst::kIdShufps:
+      return X86Logging_formatImmText(sb, u8, 2, 4, vshufps, 4);
+
+    case X86Inst::kIdVcvtps2ph:
+      return X86Logging_formatImmBits(sb, u8, vroundxx, 1);
+
+    case X86Inst::kIdVperm2f128:
+    case X86Inst::kIdVperm2i128:
+      return X86Logging_formatImmBits(sb, u8, vperm2x128, ASMJIT_ARRAY_SIZE(vperm2x128));
+
+    case X86Inst::kIdVpermilpd:
+      return X86Logging_formatImmShuf(sb, u8, 1, vecSize / 8);
+
+    case X86Inst::kIdVpermilps:
+      return X86Logging_formatImmShuf(sb, u8, 2, 4);
+
+    case X86Inst::kIdVpshufd:
+    case X86Inst::kIdPshufd:
+      return X86Logging_formatImmShuf(sb, u8, 2, 4);
+
+    case X86Inst::kIdVpshufhw:
+    case X86Inst::kIdVpshuflw:
+    case X86Inst::kIdPshufhw:
+    case X86Inst::kIdPshuflw:
+    case X86Inst::kIdPshufw:
+      return X86Logging_formatImmShuf(sb, u8, 2, 4);
+
+    // TODO: Maybe?
+    case X86Inst::kIdVfixupimmpd:
+    case X86Inst::kIdVfixupimmps:
+    case X86Inst::kIdVfixupimmsd:
+    case X86Inst::kIdVfixupimmss:
+      return kErrorOk;
+
+    case X86Inst::kIdVfpclasspd:
+    case X86Inst::kIdVfpclassps:
+    case X86Inst::kIdVfpclasssd:
+    case X86Inst::kIdVfpclassss:
+      return X86Logging_formatImmBits(sb, u8, vfpclassxx, ASMJIT_ARRAY_SIZE(vfpclassxx));
+
+    case X86Inst::kIdVgetmantpd:
+    case X86Inst::kIdVgetmantps:
+    case X86Inst::kIdVgetmantsd:
+    case X86Inst::kIdVgetmantss:
+      return X86Logging_formatImmBits(sb, u8, vgetmantxx, ASMJIT_ARRAY_SIZE(vgetmantxx));
+
+    case X86Inst::kIdVpcmpb:
+    case X86Inst::kIdVpcmpd:
+    case X86Inst::kIdVpcmpq:
+    case X86Inst::kIdVpcmpw:
+    case X86Inst::kIdVpcmpub:
+    case X86Inst::kIdVpcmpud:
+    case X86Inst::kIdVpcmpuq:
+    case X86Inst::kIdVpcmpuw:
+      return X86Logging_formatImmText(sb, u8, 2, 4, vpcmpx, 4);
+
+    case X86Inst::kIdVpcomb:
+    case X86Inst::kIdVpcomd:
+    case X86Inst::kIdVpcomq:
+    case X86Inst::kIdVpcomw:
+    case X86Inst::kIdVpcomub:
+    case X86Inst::kIdVpcomud:
+    case X86Inst::kIdVpcomuq:
+    case X86Inst::kIdVpcomuw:
+      return X86Logging_formatImmText(sb, u8, 2, 4, vpcomx, 4);
+
+    case X86Inst::kIdVpermq:
+    case X86Inst::kIdVpermpd:
+      return X86Logging_formatImmShuf(sb, u8, 2, 4);
+
+    case X86Inst::kIdVpternlogd:
+    case X86Inst::kIdVpternlogq:
+      return X86Logging_formatImmShuf(sb, u8, 1, 8);
+
+    case X86Inst::kIdVrangepd:
+    case X86Inst::kIdVrangeps:
+    case X86Inst::kIdVrangesd:
+    case X86Inst::kIdVrangess:
+      return X86Logging_formatImmBits(sb, u8, vrangexx, ASMJIT_ARRAY_SIZE(vrangexx));
+
+    case X86Inst::kIdVreducepd:
+    case X86Inst::kIdVreduceps:
+    case X86Inst::kIdVreducesd:
+    case X86Inst::kIdVreducess:
+    case X86Inst::kIdVrndscalepd:
+    case X86Inst::kIdVrndscaleps:
+    case X86Inst::kIdVrndscalesd:
+    case X86Inst::kIdVrndscaless:
+      return X86Logging_formatImmBits(sb, u8, vreducexx_vrndscalexx, ASMJIT_ARRAY_SIZE(vreducexx_vrndscalexx));
+
+    case X86Inst::kIdVshuff32x4:
+    case X86Inst::kIdVshuff64x2:
+    case X86Inst::kIdVshufi32x4:
+    case X86Inst::kIdVshufi64x2: {
+      uint32_t count = std::max<uint32_t>(vecSize / 16, 2);
+      uint32_t bits = count <= 2 ? 1 : 2;
+      return X86Logging_formatImmShuf(sb, u8, bits, count);
+    }
+
+    default:
+      return kErrorOk;
+  }
+}
+
+// ============================================================================
+// [asmjit::X86Logging - Format Register]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Logging::formatRegister(
+  StringBuilder& sb,
+  uint32_t logOptions,
+  const CodeEmitter* emitter,
+  uint32_t archType,
+  uint32_t rType,
+  uint32_t rId) noexcept {
+
+  ASMJIT_UNUSED(logOptions);
+  ASMJIT_UNUSED(archType);
+
+  if (Operand::isPackedId(rId)) {
+#if !defined(ASMJIT_DISABLE_COMPILER)
+    if (emitter && emitter->getType() == CodeEmitter::kTypeCompiler) {
+      const CodeCompiler* cc = static_cast<const CodeCompiler*>(emitter);
+
+      if (cc->isVirtRegValid(rId)) {
+        VirtReg* vReg = cc->getVirtRegById(rId);
+        ASMJIT_ASSERT(vReg != nullptr);
+
+        const char* name = vReg->getName();
+        if (name && name[0] != '\0')
+          return sb.appendString(name);
+        else
+          return sb.appendFormat("v%u", static_cast<unsigned int>(Operand::unpackId(rId)));
+      }
+    }
+#endif // !ASMJIT_DISABLE_COMPILER
+
+    return sb.appendFormat("VirtReg<Type=%u Id=%u>", rType, rId);
+  }
+  else {
+    if (rType < ASMJIT_ARRAY_SIZE(x86RegFormatInfo)) {
+      const X86RegFormatInfo& rfi = x86RegFormatInfo[rType];
+
+      if (rId < rfi.specialCount)
+        return sb.appendString(x86RegFormatStrings + rfi.specialIndex + rId * 4);
+
+      if (rId < rfi.count)
+        return sb.appendFormat(x86RegFormatStrings + rfi.formatIndex, static_cast<unsigned int>(rId));
+    }
+
+    return sb.appendFormat("PhysReg<Type=%u Id=%u>", rType, rId);
+  }
+}
+
+// ============================================================================
+// [asmjit::X86Logging - Format Instruction]
+// ============================================================================
+
+ASMJIT_FAVOR_SIZE Error X86Logging::formatInstruction(
+  StringBuilder& sb,
+  uint32_t logOptions,
+  const CodeEmitter* emitter,
+  uint32_t archType,
+  const Inst::Detail& detail, const Operand_* opArray, uint32_t opCount) noexcept {
+
+  uint32_t instId = detail.instId;
+  uint32_t options = detail.options;
+
+  // Format instruction options and instruction mnemonic.
+  if (instId < X86Inst::_kIdCount) {
+    const X86Inst& instInfo = X86Inst::getInst(instId);
+
+    // SHORT|LONG options.
+    if (options & X86Inst::kOptionShortForm) ASMJIT_PROPAGATE(sb.appendString("short "));
+    if (options & X86Inst::kOptionLongForm) ASMJIT_PROPAGATE(sb.appendString("long "));
+
+    // LOCK|XACQUIRE|XRELEASE options.
+    if (options & X86Inst::kOptionXAcquire) ASMJIT_PROPAGATE(sb.appendString("xacquire "));
+    if (options & X86Inst::kOptionXRelease) ASMJIT_PROPAGATE(sb.appendString("xrelease "));
+    if (options & X86Inst::kOptionLock) ASMJIT_PROPAGATE(sb.appendString("lock "));
+
+    // REP|REPNZ options.
+    if (options & (X86Inst::kOptionRep | X86Inst::kOptionRepnz)) {
+      sb.appendString((options & X86Inst::kOptionRep) ? "rep " : "repnz ");
+      if (detail.hasExtraReg()) {
+        ASMJIT_PROPAGATE(sb.appendChar('{'));
+        ASMJIT_PROPAGATE(formatOperand(sb, logOptions, emitter, archType, detail.extraReg.toReg<Reg>()));
+        ASMJIT_PROPAGATE(sb.appendString("} "));
+      }
+    }
+
+    // REX options.
+    if (options & X86Inst::kOptionRex) {
+      const uint32_t kRXBWMask = X86Inst::kOptionOpCodeR |
+                                 X86Inst::kOptionOpCodeX |
+                                 X86Inst::kOptionOpCodeB |
+                                 X86Inst::kOptionOpCodeW ;
+      if (options & kRXBWMask) {
+        sb.appendString("rex.");
+        if (options & X86Inst::kOptionOpCodeR) sb.appendChar('r');
+        if (options & X86Inst::kOptionOpCodeX) sb.appendChar('x');
+        if (options & X86Inst::kOptionOpCodeB) sb.appendChar('b');
+        if (options & X86Inst::kOptionOpCodeW) sb.appendChar('w');
+        sb.appendChar(' ');
+      }
+      else {
+        ASMJIT_PROPAGATE(sb.appendString("rex "));
+      }
+    }
+
+    // VEX|EVEX options.
+    if (options & X86Inst::kOptionVex3) ASMJIT_PROPAGATE(sb.appendString("vex3 "));
+    if (options & X86Inst::kOptionEvex) ASMJIT_PROPAGATE(sb.appendString("evex "));
+
+    ASMJIT_PROPAGATE(sb.appendString(instInfo.getName()));
+  }
+  else {
+    ASMJIT_PROPAGATE(sb.appendFormat("<unknown id=#%u>", static_cast<unsigned int>(instId)));
+  }
+
+  for (uint32_t i = 0; i < opCount; i++) {
+    const Operand_& op = opArray[i];
+    if (op.isNone()) break;
+
+    ASMJIT_PROPAGATE(sb.appendString(i == 0 ? " " : ", "));
+    ASMJIT_PROPAGATE(formatOperand(sb, logOptions, emitter, archType, op));
+
+    if (op.isImm() && (logOptions & Logger::kOptionImmExtended)) {
+      uint32_t vecSize = 16;
+      for (uint32_t j = 0; j < opCount; j++)
+        if (opArray[j].isReg())
+          vecSize = std::max<uint32_t>(vecSize, opArray[j].getSize());
+      ASMJIT_PROPAGATE(X86Logging_formatImmExtended(sb, logOptions, instId, vecSize, op.as<Imm>()));
+    }
+
+    // Support AVX-512 {k}{z}.
+    if (i == 0) {
+      if (detail.extraReg.getKind() == X86Reg::kKindK) {
+        ASMJIT_PROPAGATE(sb.appendString(" {"));
+        ASMJIT_PROPAGATE(formatOperand(sb, logOptions, emitter, archType, detail.extraReg.toReg<Reg>()));
+        ASMJIT_PROPAGATE(sb.appendChar('}'));
+
+        if (options & X86Inst::kOptionZMask)
+          ASMJIT_PROPAGATE(sb.appendString("{z}"));
+      }
+      else if (options & X86Inst::kOptionZMask) {
+        ASMJIT_PROPAGATE(sb.appendString(" {z}"));
+      }
+    }
+
+    // Support AVX-512 {1tox}.
+    if (op.isMem() && (options & X86Inst::kOption1ToX))
+      ASMJIT_PROPAGATE(sb.appendString(" {1tox}"));
+  }
+
+  return kErrorOk;
+}
+
+} // asmjit namespace
+
+// [Api-End]
+#include "../asmjit_apiend.h"
+
+// [Guard]
+#endif // !ASMJIT_DISABLE_LOGGING

libraries/asmjit/x86/x86logging_p.h

+// [AsmJit]
+// Complete x86/x64 JIT and Remote Assembler for C++.
+//
+// [License]
+// Zlib - See LICENSE.md file in the package.
+
+// [Guard]
+#ifndef _ASMJIT_X86_X86LOGGING_P_H
+#define _ASMJIT_X86_X86LOGGING_P_H
+
+#include "../asmjit_build.h"
+#if !defined(ASMJIT_DISABLE_LOGGING)
+
+// [Dependencies]
+#include "../base/logging.h"
+#include "../x86/x86globals.h"
+
+// [Api-Begin]
+#include "../asmjit_apibegin.h"
+
+namespace asmjit {
+
+//! \addtogroup asmjit_base
+//! \{
+
+// ============================================================================
+// [asmjit::X86Logging]
+// ============================================================================
+
+struct X86Logging {
+  static Error formatRegister(
+    StringBuilder& sb,
+    uint32_t logOptions,
+    const CodeEmitter* emitter,
+    uint32_t archType,
+    uint32_t regType,
+    uint32_t regId) noexcept;
+
+  static Error formatOperand(
+    StringBuilder& sb,
+    uint32_t logOptions,
+    const CodeEmitter* emitter,
+    uint32_t archType,
+    const Operand_& op) noexcept;
+
+  static Error formatInstruction(
+    StringBuilder& sb,
+    uint32_t logOptions,
+    const CodeEmitter* emitter,
+    uint32_t archType,
+    const Inst::Detail& detail, const Operand_* opArray, uint32_t opCount) noexcept;
+};
+
+//! \}
+
+} // asmjit namespace
+
+// [Api-End]
+#include "../asmjit_apiend.h"
+
+// [Guard]
+#endif // !ASMJIT_DISABLE_LOGGING
+#endif // _ASMJIT_X86_X86LOGGING_P_H

libraries/asmjit/x86/x86misc.h

+// [AsmJit]
+// Complete x86/x64 JIT and Remote Assembler for C++.
+//
+// [License]
+// Zlib - See LICENSE.md file in the package.
+
+// [Guard]
+#ifndef _ASMJIT_X86_X86MISC_H
+#define _ASMJIT_X86_X86MISC_H
+
+// [Dependencies]
+#include "../x86/x86operand.h"
+
+// [Api-Begin]
+#include "../asmjit_apibegin.h"
+
+namespace asmjit {
+
+//! \addtogroup asmjit_x86
+//! \{
+
+// ============================================================================
+// [asmjit::X86RegCount]
+// ============================================================================
+
+//! \internal
+//!
+//! X86/X64 registers count.
+//!
+//! Since the number of registers changed across CPU generations `X86RegCount`
+//! class is used by `X86Assembler` and `X86Compiler` to provide a way to get
+//! number of available registers dynamically. 32-bit mode offers always only
+//! 8 registers of all classes, however, 64-bit mode offers 16 GP registers and
+//! 16 XMM/YMM/ZMM registers. AVX512 instruction set doubles the number of SIMD
+//! registers (XMM/YMM/ZMM) to 32, this mode has to be explicitly enabled to
+//! take effect as it changes some assumptions.
+//!
+//! `X86RegCount` is also used extensively by X86Compiler's register allocator
+//! and data structures. FP registers were omitted as they are never mapped to
+//! variables, thus, not needed to be managed.
+//!
+//! NOTE: At the moment `X86RegCount` can fit into 32-bits, having 8-bits for
+//! each register kind except FP. This can change in the future after a new
+//! instruction set, which adds more registers, is introduced.
+struct X86RegCount {
+  // --------------------------------------------------------------------------
+  // [Zero]
+  // --------------------------------------------------------------------------
+
+  //! Reset all counters to zero.
+  ASMJIT_INLINE void reset() noexcept { _packed = 0; }
+
+  // --------------------------------------------------------------------------
+  // [Get]
+  // --------------------------------------------------------------------------
+
+  //! Get register count by a register `kind`.
+  ASMJIT_INLINE uint32_t get(uint32_t kind) const noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+
+    uint32_t shift = Utils::byteShiftOfDWordStruct(kind);
+    return (_packed >> shift) & static_cast<uint32_t>(0xFF);
+  }
+
+  //! Get Gp count.
+  ASMJIT_INLINE uint32_t getGp() const noexcept { return get(X86Reg::kKindGp); }
+  //! Get Mm count.
+  ASMJIT_INLINE uint32_t getMm() const noexcept { return get(X86Reg::kKindMm); }
+  //! Get K count.
+  ASMJIT_INLINE uint32_t getK() const noexcept { return get(X86Reg::kKindK); }
+  //! Get XMM/YMM/ZMM count.
+  ASMJIT_INLINE uint32_t getVec() const noexcept { return get(X86Reg::kKindVec); }
+
+  // --------------------------------------------------------------------------
+  // [Set]
+  // --------------------------------------------------------------------------
+
+  //! Set register count by a register `kind`.
+  ASMJIT_INLINE void set(uint32_t kind, uint32_t n) noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+    ASMJIT_ASSERT(n <= 0xFF);
+
+    uint32_t shift = Utils::byteShiftOfDWordStruct(kind);
+    _packed = (_packed & ~static_cast<uint32_t>(0xFF << shift)) + (n << shift);
+  }
+
+  //! Set Gp count.
+  ASMJIT_INLINE void setGp(uint32_t n) noexcept { set(X86Reg::kKindGp, n); }
+  //! Set Mm count.
+  ASMJIT_INLINE void setMm(uint32_t n) noexcept { set(X86Reg::kKindMm, n); }
+  //! Set K count.
+  ASMJIT_INLINE void setK(uint32_t n) noexcept { set(X86Reg::kKindK, n); }
+  //! Set XMM/YMM/ZMM count.
+  ASMJIT_INLINE void setVec(uint32_t n) noexcept { set(X86Reg::kKindVec, n); }
+
+  // --------------------------------------------------------------------------
+  // [Add]
+  // --------------------------------------------------------------------------
+
+  //! Add register count by a register `kind`.
+  ASMJIT_INLINE void add(uint32_t kind, uint32_t n = 1) noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+    ASMJIT_ASSERT(0xFF - static_cast<uint32_t>(_regs[kind]) >= n);
+
+    uint32_t shift = Utils::byteShiftOfDWordStruct(kind);
+    _packed += n << shift;
+  }
+
+  //! Add GP count.
+  ASMJIT_INLINE void addGp(uint32_t n) noexcept { add(X86Reg::kKindGp, n); }
+  //! Add MMX count.
+  ASMJIT_INLINE void addMm(uint32_t n) noexcept { add(X86Reg::kKindMm, n); }
+  //! Add K count.
+  ASMJIT_INLINE void addK(uint32_t n) noexcept { add(X86Reg::kKindK, n); }
+  //! Add XMM/YMM/ZMM count.
+  ASMJIT_INLINE void addVec(uint32_t n) noexcept { add(X86Reg::kKindVec, n); }
+
+  // --------------------------------------------------------------------------
+  // [Misc]
+  // --------------------------------------------------------------------------
+
+  //! Build register indexes based on the given `count` of registers.
+  ASMJIT_INLINE void indexFromRegCount(const X86RegCount& count) noexcept {
+    uint32_t x = static_cast<uint32_t>(count._regs[0]);
+    uint32_t y = static_cast<uint32_t>(count._regs[1]) + x;
+    uint32_t z = static_cast<uint32_t>(count._regs[2]) + y;
+
+    ASMJIT_ASSERT(y <= 0xFF);
+    ASMJIT_ASSERT(z <= 0xFF);
+    _packed = Utils::pack32_4x8(0, x, y, z);
+  }
+
+  // --------------------------------------------------------------------------
+  // [Members]
+  // --------------------------------------------------------------------------
+
+  union {
+    struct {
+      //! Count of GP registers.
+      uint8_t _gp;
+      //! Count of XMM|YMM|ZMM registers.
+      uint8_t _vec;
+      //! Count of MMX registers.
+      uint8_t _mm;
+      //! Count of K registers.
+      uint8_t _k;
+    };
+
+    uint8_t _regs[4];
+    uint32_t _packed;
+  };
+};
+
+// ============================================================================
+// [asmjit::X86RegMask]
+// ============================================================================
+
+//! \internal
+//!
+//! X86/X64 registers mask.
+struct X86RegMask {
+  // --------------------------------------------------------------------------
+  // [Reset]
+  // --------------------------------------------------------------------------
+
+  //! Reset all register masks to zero.
+  ASMJIT_INLINE void reset() noexcept {
+    _packed.reset();
+  }
+
+  // --------------------------------------------------------------------------
+  // [IsEmpty / Has]
+  // --------------------------------------------------------------------------
+
+  //! Get whether all register masks are zero (empty).
+  ASMJIT_INLINE bool isEmpty() const noexcept {
+    return _packed.isZero();
+  }
+
+  ASMJIT_INLINE bool has(uint32_t kind, uint32_t mask = 0xFFFFFFFFU) const noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+
+    switch (kind) {
+      case X86Reg::kKindGp : return (static_cast<uint32_t>(_gp ) & mask) != 0;
+      case X86Reg::kKindVec: return (static_cast<uint32_t>(_vec) & mask) != 0;
+      case X86Reg::kKindMm : return (static_cast<uint32_t>(_mm ) & mask) != 0;
+      case X86Reg::kKindK  : return (static_cast<uint32_t>(_k  ) & mask) != 0;
+    }
+
+    return false;
+  }
+
+  ASMJIT_INLINE bool hasGp(uint32_t mask = 0xFFFFFFFFU) const noexcept { return has(X86Reg::kKindGp, mask); }
+  ASMJIT_INLINE bool hasVec(uint32_t mask = 0xFFFFFFFFU) const noexcept { return has(X86Reg::kKindVec, mask); }
+  ASMJIT_INLINE bool hasMm(uint32_t mask = 0xFFFFFFFFU) const noexcept { return has(X86Reg::kKindMm, mask); }
+  ASMJIT_INLINE bool hasK(uint32_t mask = 0xFFFFFFFFU) const noexcept { return has(X86Reg::kKindK, mask); }
+
+  // --------------------------------------------------------------------------
+  // [Get]
+  // --------------------------------------------------------------------------
+
+  ASMJIT_INLINE uint32_t get(uint32_t kind) const noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+
+    switch (kind) {
+      case X86Reg::kKindGp : return _gp;
+      case X86Reg::kKindVec: return _vec;
+      case X86Reg::kKindMm : return _mm;
+      case X86Reg::kKindK  : return _k;
+    }
+
+    return 0;
+  }
+
+  ASMJIT_INLINE uint32_t getGp() const noexcept { return get(X86Reg::kKindGp); }
+  ASMJIT_INLINE uint32_t getVec() const noexcept { return get(X86Reg::kKindVec); }
+  ASMJIT_INLINE uint32_t getMm() const noexcept { return get(X86Reg::kKindMm); }
+  ASMJIT_INLINE uint32_t getK() const noexcept { return get(X86Reg::kKindK); }
+
+  // --------------------------------------------------------------------------
+  // [Zero]
+  // --------------------------------------------------------------------------
+
+  ASMJIT_INLINE void zero(uint32_t kind) noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+
+    switch (kind) {
+      case X86Reg::kKindGp : _gp  = 0; break;
+      case X86Reg::kKindVec: _vec = 0; break;
+      case X86Reg::kKindMm : _mm  = 0; break;
+      case X86Reg::kKindK  : _k   = 0; break;
+    }
+  }
+
+  ASMJIT_INLINE void zeroGp() noexcept { zero(X86Reg::kKindGp); }
+  ASMJIT_INLINE void zeroVec() noexcept { zero(X86Reg::kKindVec); }
+  ASMJIT_INLINE void zeroMm() noexcept { zero(X86Reg::kKindMm); }
+  ASMJIT_INLINE void zeroK() noexcept { zero(X86Reg::kKindK); }
+
+  // --------------------------------------------------------------------------
+  // [Set]
+  // --------------------------------------------------------------------------
+
+  ASMJIT_INLINE void set(const X86RegMask& other) noexcept {
+    _packed = other._packed;
+  }
+
+  ASMJIT_INLINE void set(uint32_t kind, uint32_t mask) noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+
+    switch (kind) {
+      case X86Reg::kKindGp : _gp  = static_cast<uint16_t>(mask); break;
+      case X86Reg::kKindMm : _mm  = static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindK  : _k   = static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindVec: _vec = static_cast<uint32_t>(mask); break;
+    }
+  }
+
+  ASMJIT_INLINE void setGp(uint32_t mask) noexcept { return set(X86Reg::kKindGp, mask); }
+  ASMJIT_INLINE void setVec(uint32_t mask) noexcept { return set(X86Reg::kKindVec, mask); }
+  ASMJIT_INLINE void setMm(uint32_t mask) noexcept { return set(X86Reg::kKindMm, mask); }
+  ASMJIT_INLINE void setK(uint32_t mask) noexcept { return set(X86Reg::kKindK, mask); }
+
+  // --------------------------------------------------------------------------
+  // [And]
+  // --------------------------------------------------------------------------
+
+  ASMJIT_INLINE void and_(const X86RegMask& other) noexcept {
+    _packed.and_(other._packed);
+  }
+
+  ASMJIT_INLINE void and_(uint32_t kind, uint32_t mask) noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+
+    switch (kind) {
+      case X86Reg::kKindGp : _gp  &= static_cast<uint16_t>(mask); break;
+      case X86Reg::kKindMm : _mm  &= static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindK  : _k   &= static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindVec: _vec &= static_cast<uint32_t>(mask); break;
+    }
+  }
+
+  ASMJIT_INLINE void andGp(uint32_t mask) noexcept { and_(X86Reg::kKindGp, mask); }
+  ASMJIT_INLINE void andVec(uint32_t mask) noexcept { and_(X86Reg::kKindVec, mask); }
+  ASMJIT_INLINE void andMm(uint32_t mask) noexcept { and_(X86Reg::kKindMm, mask); }
+  ASMJIT_INLINE void andK(uint32_t mask) noexcept { and_(X86Reg::kKindK, mask); }
+
+  // --------------------------------------------------------------------------
+  // [AndNot]
+  // --------------------------------------------------------------------------
+
+  ASMJIT_INLINE void andNot(const X86RegMask& other) noexcept {
+    _packed.andNot(other._packed);
+  }
+
+  ASMJIT_INLINE void andNot(uint32_t kind, uint32_t mask) noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+
+    switch (kind) {
+      case X86Reg::kKindGp : _gp  &= ~static_cast<uint16_t>(mask); break;
+      case X86Reg::kKindMm : _mm  &= ~static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindK  : _k   &= ~static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindVec: _vec &= ~static_cast<uint32_t>(mask); break;
+    }
+  }
+
+  ASMJIT_INLINE void andNotGp(uint32_t mask) noexcept { andNot(X86Reg::kKindGp, mask); }
+  ASMJIT_INLINE void andNotVec(uint32_t mask) noexcept { andNot(X86Reg::kKindVec, mask); }
+  ASMJIT_INLINE void andNotMm(uint32_t mask) noexcept { andNot(X86Reg::kKindMm, mask); }
+  ASMJIT_INLINE void andNotK(uint32_t mask) noexcept { andNot(X86Reg::kKindK, mask); }
+
+  // --------------------------------------------------------------------------
+  // [Or]
+  // --------------------------------------------------------------------------
+
+  ASMJIT_INLINE void or_(const X86RegMask& other) noexcept {
+    _packed.or_(other._packed);
+  }
+
+  ASMJIT_INLINE void or_(uint32_t kind, uint32_t mask) noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+    switch (kind) {
+      case X86Reg::kKindGp : _gp  |= static_cast<uint16_t>(mask); break;
+      case X86Reg::kKindMm : _mm  |= static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindK  : _k   |= static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindVec: _vec |= static_cast<uint32_t>(mask); break;
+    }
+  }
+
+  ASMJIT_INLINE void orGp(uint32_t mask) noexcept { return or_(X86Reg::kKindGp, mask); }
+  ASMJIT_INLINE void orVec(uint32_t mask) noexcept { return or_(X86Reg::kKindVec, mask); }
+  ASMJIT_INLINE void orMm(uint32_t mask) noexcept { return or_(X86Reg::kKindMm, mask); }
+  ASMJIT_INLINE void orK(uint32_t mask) noexcept { return or_(X86Reg::kKindK, mask); }
+
+  // --------------------------------------------------------------------------
+  // [Xor]
+  // --------------------------------------------------------------------------
+
+  ASMJIT_INLINE void xor_(const X86RegMask& other) noexcept {
+    _packed.xor_(other._packed);
+  }
+
+  ASMJIT_INLINE void xor_(uint32_t kind, uint32_t mask) noexcept {
+    ASMJIT_ASSERT(kind < Globals::kMaxVRegKinds);
+
+    switch (kind) {
+      case X86Reg::kKindGp : _gp  ^= static_cast<uint16_t>(mask); break;
+      case X86Reg::kKindMm : _mm  ^= static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindK  : _k   ^= static_cast<uint8_t >(mask); break;
+      case X86Reg::kKindVec: _vec ^= static_cast<uint32_t>(mask); break;
+    }
+  }
+
+  ASMJIT_INLINE void xorGp(uint32_t mask) noexcept { xor_(X86Reg::kKindGp, mask); }
+  ASMJIT_INLINE void xorVec(uint32_t mask) noexcept { xor_(X86Reg::kKindVec, mask); }
+  ASMJIT_INLINE void xorMm(uint32_t mask) noexcept { xor_(X86Reg::kKindMm, mask); }
+  ASMJIT_INLINE void xorK(uint32_t mask) noexcept { xor_(X86Reg::kKindK, mask); }
+
+  // --------------------------------------------------------------------------
+  // [Members]
+  // --------------------------------------------------------------------------
+
+  union {
+    struct {
+      //! GP registers mask (16 bits).
+      uint16_t _gp;
+      //! MMX registers mask (8 bits).
+      uint8_t _mm;
+      //! K registers mask (8 bits).
+      uint8_t _k;
+      //! XMM|YMM|ZMM registers mask (32 bits).
+      uint32_t _vec;
+    };
+
+    //! Packed masks.
+    UInt64 _packed;
+  };
+};
+
+//! \}
+
+} // asmjit namespace
+
+// [Api-End]
+#include "../asmjit_apiend.h"
+
+// [Guard]
+#endif // _ASMJIT_X86_X86MISC_H

libraries/asmjit/x86/x86operand.cpp

@@ -8,78 +8,169 @@
 #define ASMJIT_EXPORTS
 
 // [Guard]
-#include "../build.h"
-#if defined(ASMJIT_BUILD_X86) || defined(ASMJIT_BUILD_X64)
+#include "../asmjit_build.h"
+#if defined(ASMJIT_BUILD_X86)
 
-// [Dependencies - AsmJit]
+// [Dependencies]
 #include "../x86/x86operand.h"
 
 // [Api-Begin]
-#include "../apibegin.h"
+#include "../asmjit_apibegin.h"
 
 namespace asmjit {
-namespace x86 {
 
 // ============================================================================
-// [asmjit::X86Mem - abs[]]
+// [asmjit::X86Operand - Test]
 // ============================================================================
 
-X86Mem ptr_abs(Ptr pAbs, int32_t disp, uint32_t size) {
-  X86Mem m(NoInit);
-
-  m._init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeAbsolute, 0, kInvalidValue);
-  m._vmem.index = kInvalidValue;
-  m._vmem.displacement = static_cast<int32_t>((intptr_t)(pAbs + disp));
-
-  return m;
-}
-
-X86Mem ptr_abs(Ptr pAbs, const X86Reg& index, uint32_t shift, int32_t disp, uint32_t size) {
-  X86Mem m(NoInit);
-  uint32_t flags = shift << kX86MemShiftIndex;
-
-  if (index.isGp())
-    flags |= X86Mem::_getGpdFlags(index);
-  else if (index.isXmm())
-    flags |= kX86MemVSibXmm << kX86MemVSibIndex;
-  else if (index.isYmm())
-    flags |= kX86MemVSibYmm << kX86MemVSibIndex;
-
-  m._init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeAbsolute, flags, kInvalidValue);
-  m._vmem.index = index.getRegIndex();
-  m._vmem.displacement = static_cast<int32_t>((intptr_t)(pAbs + disp));
-
-  return m;
-}
-
-#if !defined(ASMJIT_DISABLE_COMPILER)
-X86Mem ptr_abs(Ptr pAbs, const X86Var& index, uint32_t shift, int32_t disp, uint32_t size) {
-  X86Mem m(NoInit);
-  uint32_t flags = shift << kX86MemShiftIndex;
-
-  const Var& index_ = reinterpret_cast<const Var&>(index);
-  uint32_t indexRegType = index_.getRegType();
-
-  if (indexRegType <= kX86RegTypeGpq)
-    flags |= X86Mem::_getGpdFlags(reinterpret_cast<const Var&>(index));
-  else if (indexRegType == kX86RegTypeXmm)
-    flags |= kX86MemVSibXmm << kX86MemVSibIndex;
-  else if (indexRegType == kX86RegTypeYmm)
-    flags |= kX86MemVSibYmm << kX86MemVSibIndex;
-
-  m._init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeAbsolute, flags, kInvalidValue);
-  m._vmem.index = index_.getId();
-  m._vmem.displacement = static_cast<int32_t>((intptr_t)(pAbs + disp));
-
-  return m;
+#if defined(ASMJIT_TEST)
+UNIT(x86_operand) {
+  Label L;
+
+  INFO("Checking basic properties of built-in X86 registers");
+  EXPECT(x86::gpb(X86Gp::kIdAx) == x86::al);
+  EXPECT(x86::gpb(X86Gp::kIdBx) == x86::bl);
+  EXPECT(x86::gpb(X86Gp::kIdCx) == x86::cl);
+  EXPECT(x86::gpb(X86Gp::kIdDx) == x86::dl);
+
+  EXPECT(x86::gpb_lo(X86Gp::kIdAx) == x86::al);
+  EXPECT(x86::gpb_lo(X86Gp::kIdBx) == x86::bl);
+  EXPECT(x86::gpb_lo(X86Gp::kIdCx) == x86::cl);
+  EXPECT(x86::gpb_lo(X86Gp::kIdDx) == x86::dl);
+
+  EXPECT(x86::gpb_hi(X86Gp::kIdAx) == x86::ah);
+  EXPECT(x86::gpb_hi(X86Gp::kIdBx) == x86::bh);
+  EXPECT(x86::gpb_hi(X86Gp::kIdCx) == x86::ch);
+  EXPECT(x86::gpb_hi(X86Gp::kIdDx) == x86::dh);
+
+  EXPECT(x86::gpw(X86Gp::kIdAx) == x86::ax);
+  EXPECT(x86::gpw(X86Gp::kIdBx) == x86::bx);
+  EXPECT(x86::gpw(X86Gp::kIdCx) == x86::cx);
+  EXPECT(x86::gpw(X86Gp::kIdDx) == x86::dx);
+
+  EXPECT(x86::gpd(X86Gp::kIdAx) == x86::eax);
+  EXPECT(x86::gpd(X86Gp::kIdBx) == x86::ebx);
+  EXPECT(x86::gpd(X86Gp::kIdCx) == x86::ecx);
+  EXPECT(x86::gpd(X86Gp::kIdDx) == x86::edx);
+
+  EXPECT(x86::gpq(X86Gp::kIdAx) == x86::rax);
+  EXPECT(x86::gpq(X86Gp::kIdBx) == x86::rbx);
+  EXPECT(x86::gpq(X86Gp::kIdCx) == x86::rcx);
+  EXPECT(x86::gpq(X86Gp::kIdDx) == x86::rdx);
+
+  EXPECT(x86::gpb(X86Gp::kIdAx) != x86::dl);
+  EXPECT(x86::gpw(X86Gp::kIdBx) != x86::cx);
+  EXPECT(x86::gpd(X86Gp::kIdCx) != x86::ebx);
+  EXPECT(x86::gpq(X86Gp::kIdDx) != x86::rax);
+
+  INFO("Checking if x86::reg(...) matches built-in IDs");
+  EXPECT(x86::fp(5)  == x86::fp5);
+  EXPECT(x86::mm(5)  == x86::mm5);
+  EXPECT(x86::k(5)   == x86::k5);
+  EXPECT(x86::cr(5)  == x86::cr5);
+  EXPECT(x86::dr(5)  == x86::dr5);
+  EXPECT(x86::xmm(5) == x86::xmm5);
+  EXPECT(x86::ymm(5) == x86::ymm5);
+  EXPECT(x86::zmm(5) == x86::zmm5);
+
+  INFO("Checking GP register properties");
+  EXPECT(X86Gp().isReg() == false);
+  EXPECT(x86::eax.isReg() == true);
+  EXPECT(x86::eax.getId() == 0);
+  EXPECT(x86::eax.getSize() == 4);
+  EXPECT(x86::eax.getType() == X86Reg::kRegGpd);
+  EXPECT(x86::eax.getKind() == X86Reg::kKindGp);
+
+  INFO("Checking FP register properties");
+  EXPECT(X86Fp().isReg() == false);
+  EXPECT(x86::fp1.isReg() == true);
+  EXPECT(x86::fp1.getId() == 1);
+  EXPECT(x86::fp1.getSize() == 10);
+  EXPECT(x86::fp1.getType() == X86Reg::kRegFp);
+  EXPECT(x86::fp1.getKind() == X86Reg::kKindFp);
+
+  INFO("Checking MM register properties");
+  EXPECT(X86Mm().isReg() == false);
+  EXPECT(x86::mm2.isReg() == true);
+  EXPECT(x86::mm2.getId() == 2);
+  EXPECT(x86::mm2.getSize() == 8);
+  EXPECT(x86::mm2.getType() == X86Reg::kRegMm);
+  EXPECT(x86::mm2.getKind() == X86Reg::kKindMm);
+
+  INFO("Checking K register properties");
+  EXPECT(X86KReg().isReg() == false);
+  EXPECT(x86::k3.isReg() == true);
+  EXPECT(x86::k3.getId() == 3);
+  EXPECT(x86::k3.getSize() == 0);
+  EXPECT(x86::k3.getType() == X86Reg::kRegK);
+  EXPECT(x86::k3.getKind() == X86Reg::kKindK);
+
+  INFO("Checking XMM register properties");
+  EXPECT(X86Xmm().isReg() == false);
+  EXPECT(x86::xmm4.isReg() == true);
+  EXPECT(x86::xmm4.getId() == 4);
+  EXPECT(x86::xmm4.getSize() == 16);
+  EXPECT(x86::xmm4.getType() == X86Reg::kRegXmm);
+  EXPECT(x86::xmm4.getKind() == X86Reg::kKindVec);
+  EXPECT(x86::xmm4.isVec());
+
+  INFO("Checking YMM register properties");
+  EXPECT(X86Ymm().isReg() == false);
+  EXPECT(x86::ymm5.isReg() == true);
+  EXPECT(x86::ymm5.getId() == 5);
+  EXPECT(x86::ymm5.getSize() == 32);
+  EXPECT(x86::ymm5.getType() == X86Reg::kRegYmm);
+  EXPECT(x86::ymm5.getKind() == X86Reg::kKindVec);
+  EXPECT(x86::ymm5.isVec());
+
+  INFO("Checking ZMM register properties");
+  EXPECT(X86Zmm().isReg() == false);
+  EXPECT(x86::zmm6.isReg() == true);
+  EXPECT(x86::zmm6.getId() == 6);
+  EXPECT(x86::zmm6.getSize() == 64);
+  EXPECT(x86::zmm6.getType() == X86Reg::kRegZmm);
+  EXPECT(x86::zmm6.getKind() == X86Reg::kKindVec);
+  EXPECT(x86::zmm6.isVec());
+
+  INFO("Checking VEC register properties");
+  EXPECT(X86Vec().isReg() == false);
+  // Converts a VEC register to a type of the passed register, but keeps the ID.
+  EXPECT(x86::xmm4.cloneAs(x86::ymm10) == x86::ymm4);
+  EXPECT(x86::xmm4.cloneAs(x86::zmm11) == x86::zmm4);
+  EXPECT(x86::ymm5.cloneAs(x86::xmm12) == x86::xmm5);
+  EXPECT(x86::ymm5.cloneAs(x86::zmm13) == x86::zmm5);
+  EXPECT(x86::zmm6.cloneAs(x86::xmm14) == x86::xmm6);
+  EXPECT(x86::zmm6.cloneAs(x86::ymm15) == x86::ymm6);
+
+  INFO("Checking if default constructed regs behave as expected");
+  EXPECT(X86Reg().isValid() == false);
+  EXPECT(X86Gp().isValid() == false);
+  EXPECT(X86Fp().isValid() == false);
+  EXPECT(X86Mm().isValid() == false);
+  EXPECT(X86Xmm().isValid() == false);
+  EXPECT(X86Ymm().isValid() == false);
+  EXPECT(X86Zmm().isValid() == false);
+  EXPECT(X86KReg().isValid() == false);
+
+  INFO("Checking X86Mem operand");
+  X86Mem m;
+  EXPECT(m == X86Mem(),
+    "Two default constructed X86Mem operands must be equal");
+
+  X86Mem mL = x86::ptr(L);
+  EXPECT(mL.hasBase() == true,
+    "Memory constructed from Label must hasBase()");
+  EXPECT(mL.hasBaseReg() == false,
+    "Memory constructed from Label must not report hasBaseReg()");
+  EXPECT(mL.hasBaseLabel() == true,
+    "Memory constructed from Label must report hasBaseLabel()");
 }
-#endif // !ASMJIT_DISABLE_COMPILER
+#endif // ASMJIT_TEST
 
-} // x86 namespace
 } // asmjit namespace
 
 // [Api-End]
-#include "../apiend.h"
+#include "../asmjit_apiend.h"
 
 // [Guard]
-#endif // ASMJIT_BUILD_X86 || ASMJIT_BUILD_X64
+#endif // ASMJIT_BUILD_X86

libraries/asmjit/x86/x86operand.h

@@ -8,1598 +8,669 @@
 #ifndef _ASMJIT_X86_X86OPERAND_H
 #define _ASMJIT_X86_X86OPERAND_H
 
-// [Dependencies - AsmJit]
-#include "../base/assembler.h"
-#include "../base/compiler.h"
-#include "../base/globals.h"
-#include "../base/intutil.h"
+// [Dependencies]
+#include "../base/arch.h"
 #include "../base/operand.h"
-#include "../base/vectypes.h"
+#include "../base/utils.h"
+#include "../x86/x86globals.h"
 
 // [Api-Begin]
-#include "../apibegin.h"
+#include "../asmjit_apibegin.h"
 
-//! \internal
-//!
-//! Internal macro to get an operand ID casting it to `Operand`. Basically
-//! allows to get an id of operand that has been just 'typedef'ed.
-#define ASMJIT_OP_ID(_Op_) reinterpret_cast<const Operand&>(_Op_).getId()
-
-namespace asmjit {
-
-// ============================================================================
-// [Forward Declarations]
-// ============================================================================
-
-struct X86Reg;
-struct X86RipReg;
-struct X86SegReg;
-struct X86GpReg;
-struct X86FpReg;
-struct X86MmReg;
-struct X86KReg;
-struct X86XmmReg;
-struct X86YmmReg;
-struct X86ZmmReg;
-
-#if !defined(ASMJIT_DISABLE_COMPILER)
-struct X86Var;
-struct X86GpVar;
-struct X86MmVar;
-struct X86KVar;
-struct X86XmmVar;
-struct X86YmmVar;
-struct X86ZmmVar;
-#endif // !ASMJIT_DISABLE_COMPILER
-
-//! \addtogroup asmjit_x86_general
-//! \{
-
-// ============================================================================
-// [asmjit::X86RegClass]
-// ============================================================================
-
-//! X86/X64 variable class.
-ASMJIT_ENUM(X86RegClass) {
-  // --------------------------------------------------------------------------
-  // [Regs & Vars]
-  // --------------------------------------------------------------------------
-
-  //! X86/X64 Gp register class (compatible with universal \ref kRegClassGp).
-  kX86RegClassGp = kRegClassGp,
-  //! X86/X64 Mm register class.
-  kX86RegClassMm = 1,
-  //! X86/X64 K register class.
-  kX86RegClassK = 2,
-  //! X86/X64 Xmm/Ymm/Zmm register class.
-  kX86RegClassXyz = 3,
-
-  //! \internal
-  //!
-  //! Last register class that is managed by `X86Compiler`, used by asserts.
-  _kX86RegClassManagedCount = 4,
-
-  // --------------------------------------------------------------------------
-  // [Regs Only]
-  // --------------------------------------------------------------------------
-
-  //! X86/X64 Fp register class.
-  kX86RegClassFp = 4,
-
-  //! Count of X86/X64 register classes.
-  kX86RegClassCount = 5
-};
-
-// ============================================================================
-// [asmjit::X86RegType]
-// ============================================================================
-
-//! X86/X64 register type.
-ASMJIT_ENUM(X86RegType) {
-  //! Gpb-lo register (AL, BL, CL, DL, ...).
-  kX86RegTypeGpbLo = 0x01,
-  //! Gpb-hi register (AH, BH, CH, DH only).
-  kX86RegTypeGpbHi = 0x02,
-
-  //! \internal
-  //!
-  //! Gpb-hi register patched to native index (4-7).
-  _kX86RegTypePatchedGpbHi = kX86RegTypeGpbLo | kX86RegTypeGpbHi,
-
-  //! Gpw register.
-  kX86RegTypeGpw = 0x10,
-  //! Gpd register.
-  kX86RegTypeGpd = 0x20,
-  //! Gpq register (X64).
-  kX86RegTypeGpq = 0x30,
-
-  //! Fp register.
-  kX86RegTypeFp = 0x40,
-  //! Mm register (MMX+).
-  kX86RegTypeMm = 0x50,
-
-  //! K register (AVX512+).
-  kX86RegTypeK = 0x60,
-
-  //! Xmm register (SSE+).
-  kX86RegTypeXmm = 0x70,
-  //! Ymm register (AVX+).
-  kX86RegTypeYmm = 0x80,
-  //! Zmm register (AVX512+).
-  kX86RegTypeZmm = 0x90,
-
-  //! Instruction pointer (RIP).
-  kX86RegTypeRip = 0xE0,
-  //! Segment register.
-  kX86RegTypeSeg = 0xF0
-};
-
-// ============================================================================
-// [asmjit::X86RegIndex]
-// ============================================================================
-
-//! X86/X64 register indexes.
-//!
-//! \note Register indexes have been reduced to only support general purpose
-//! registers. There is no need to have enumerations with number suffix that
-//! expands to the exactly same value as the suffix value itself.
-ASMJIT_ENUM(X86RegIndex) {
-  //! Index of Al/Ah/Ax/Eax/Rax registers.
-  kX86RegIndexAx = 0,
-  //! Index of Cl/Ch/Cx/Ecx/Rcx registers.
-  kX86RegIndexCx = 1,
-  //! Index of Dl/Dh/Dx/Edx/Rdx registers.
-  kX86RegIndexDx = 2,
-  //! Index of Bl/Bh/Bx/Ebx/Rbx registers.
-  kX86RegIndexBx = 3,
-  //! Index of Spl/Sp/Esp/Rsp registers.
-  kX86RegIndexSp = 4,
-  //! Index of Bpl/Bp/Ebp/Rbp registers.
-  kX86RegIndexBp = 5,
-  //! Index of Sil/Si/Esi/Rsi registers.
-  kX86RegIndexSi = 6,
-  //! Index of Dil/Di/Edi/Rdi registers.
-  kX86RegIndexDi = 7,
-  //! Index of R8b/R8w/R8d/R8 registers (64-bit only).
-  kX86RegIndexR8 = 8,
-  //! Index of R9B/R9w/R9d/R9 registers (64-bit only).
-  kX86RegIndexR9 = 9,
-  //! Index of R10B/R10w/R10D/R10 registers (64-bit only).
-  kX86RegIndexR10 = 10,
-  //! Index of R11B/R11w/R11d/R11 registers (64-bit only).
-  kX86RegIndexR11 = 11,
-  //! Index of R12B/R12w/R12d/R12 registers (64-bit only).
-  kX86RegIndexR12 = 12,
-  //! Index of R13B/R13w/R13d/R13 registers (64-bit only).
-  kX86RegIndexR13 = 13,
-  //! Index of R14B/R14w/R14d/R14 registers (64-bit only).
-  kX86RegIndexR14 = 14,
-  //! Index of R15B/R15w/R15d/R15 registers (64-bit only).
-  kX86RegIndexR15 = 15
-};
-
-// ============================================================================
-// [asmjit::X86Seg]
-// ============================================================================
-
-//! X86/X64 segment codes.
-ASMJIT_ENUM(X86Seg) {
-  //! No/Default segment.
-  kX86SegDefault = 0,
-  //! Es segment.
-  kX86SegEs = 1,
-  //! Cs segment.
-  kX86SegCs = 2,
-  //! Ss segment.
-  kX86SegSs = 3,
-  //! Ds segment.
-  kX86SegDs = 4,
-  //! Fs segment.
-  kX86SegFs = 5,
-  //! Gs segment.
-  kX86SegGs = 6,
-
-  //! Count of X86 segment registers supported by AsmJit.
-  //!
-  //! \note X86 architecture has 6 segment registers - ES, CS, SS, DS, FS, GS.
-  //! X64 architecture lowers them down to just FS and GS. AsmJit supports 7
-  //! segment registers - all addressable in both X86 and X64 modes and one
-  //! extra called `kX86SegDefault`, which is AsmJit specific and means that there
-  //! is no segment register specified so the segment prefix will not be emitted.
-  kX86SegCount = 7
-};
-
-// ============================================================================
-// [asmjit::X86MemVSib]
-// ============================================================================
-
-//! X86/X64 index register legacy and AVX2 (VSIB) support.
-ASMJIT_ENUM(X86MemVSib) {
-  //! Memory operand uses Gpd/Gpq index (or no index register).
-  kX86MemVSibGpz = 0,
-  //! Memory operand uses Xmm index (or no index register).
-  kX86MemVSibXmm = 1,
-  //! Memory operand uses Ymm index (or no index register).
-  kX86MemVSibYmm = 2,
-  //! Memory operand uses Zmm index (or no index register).
-  kX86MemVSibZmm = 3
-};
-
-// ============================================================================
-// [asmjit::X86MemFlags]
-// ============================================================================
-
-//! \internal
-//!
-//! X86/X64 specific memory flags.
-ASMJIT_ENUM(X86MemFlags) {
-  kX86MemSegBits    = 0x7,
-  kX86MemSegIndex   = 0,
-  kX86MemSegMask    = kX86MemSegBits << kX86MemSegIndex,
-
-  kX86MemGpdBits    = 0x1,
-  kX86MemGpdIndex   = 3,
-  kX86MemGpdMask    = kX86MemGpdBits << kX86MemGpdIndex,
-
-  kX86MemVSibBits   = 0x3,
-  kX86MemVSibIndex  = 4,
-  kX86MemVSibMask   = kX86MemVSibBits << kX86MemVSibIndex,
-
-  kX86MemShiftBits  = 0x3,
-  kX86MemShiftIndex = 6,
-  kX86MemShiftMask  = kX86MemShiftBits << kX86MemShiftIndex
-};
-
-// This is only defined by `x86operand_regs.cpp` when exporting registers.
-#if defined(ASMJIT_EXPORTS_X86OPERAND_REGS)
-
-// Remap all classes to POD structs so they can be statically initialized
-// without calling a constructor. Compiler will store these in .DATA section.
-struct X86RipReg { Operand::VRegOp data; };
-struct X86SegReg { Operand::VRegOp data; };
-struct X86GpReg  { Operand::VRegOp data; };
-struct X86FpReg  { Operand::VRegOp data; };
-struct X86KReg   { Operand::VRegOp data; };
-struct X86MmReg  { Operand::VRegOp data; };
-struct X86XmmReg { Operand::VRegOp data; };
-struct X86YmmReg { Operand::VRegOp data; };
-struct X86ZmmReg { Operand::VRegOp data; };
-
-#else
-
-// ============================================================================
-// [asmjit::X86RegCount]
-// ============================================================================
-
-//! \internal
-//!
-//! X86/X64 registers count (Gp, Mm, K, Xmm/Ymm/Zmm).
-//!
-//! Since the number of registers changed across CPU generations `X86RegCount`
-//! class is used by `X86Assembler` and `X86Compiler` to provide a way to get
-//! number of available registers dynamically. 32-bit mode offers always only
-//! 8 registers of all classes, however, 64-bit mode offers 16 Gp registers and
-//! 16 Xmm/Ymm/Zmm registers. AVX512 instruction set doubles the number of SIMD
-//! registers (Xmm/Ymm/Zmm) to 32, this mode has to be explicitly enabled to
-//! take effect as it changes some assumptions.
-//! 
-//! `X86RegCount` is also used extensively by `X86Compiler`'s register allocator
-//! and data structures. Fp registers were omitted as they are never mapped to
-//! variables, thus, not needed to be managed.
-//!
-//! \note At the moment `X86RegCount` can fit into 32-bits, having 8-bits for
-//! all register classes (except Fp). This can change in the future after a
-//! new instruction set is announced.
-struct X86RegCount {
-  // --------------------------------------------------------------------------
-  // [Zero]
-  // --------------------------------------------------------------------------
-
-  //! Reset all counters to zero.
-  ASMJIT_INLINE void reset() {
-    _packed = 0;
-  }
-
-  // --------------------------------------------------------------------------
-  // [Get]
-  // --------------------------------------------------------------------------
-
-  //! Get register count by `classId`.
-  ASMJIT_INLINE uint32_t get(uint32_t classId) const {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-    return _regs[classId];
-  }
-
-  //! Get Gp register count.
-  ASMJIT_INLINE uint32_t getGp() const { return _regs[kX86RegClassGp]; }
-  //! Get Mm register count.
-  ASMJIT_INLINE uint32_t getMm() const { return _regs[kX86RegClassMm]; }
-  //! Get K register count.
-  ASMJIT_INLINE uint32_t getK() const { return _regs[kX86RegClassK]; }
-  //! Get Xmm/Ymm/Zmm register count.
-  ASMJIT_INLINE uint32_t getXyz() const { return _regs[kX86RegClassXyz]; }
-
-  // --------------------------------------------------------------------------
-  // [Set]
-  // --------------------------------------------------------------------------
-
-  //! Set register count by `classId`.
-  ASMJIT_INLINE void set(uint32_t classId, uint32_t n) {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-    ASMJIT_ASSERT(n <= 0xFF);
-
-    _regs[classId] = static_cast<uint8_t>(n);
-  }
-
-  //! Set Gp register count.
-  ASMJIT_INLINE void setGp(uint32_t n) { set(kX86RegClassGp, n); }
-  //! Set Mm register count.
-  ASMJIT_INLINE void setMm(uint32_t n) { set(kX86RegClassMm, n); }
-  //! Set K register count.
-  ASMJIT_INLINE void setK(uint32_t n) { set(kX86RegClassK, n); }
-  //! Set Xmm/Ymm/Zmm register count.
-  ASMJIT_INLINE void setXyz(uint32_t n) { set(kX86RegClassXyz, n); }
-
-  // --------------------------------------------------------------------------
-  // [Add]
-  // --------------------------------------------------------------------------
-
-  //! Add register count by `classId`.
-  ASMJIT_INLINE void add(uint32_t classId, uint32_t n = 1) {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-    ASMJIT_ASSERT(0xFF - static_cast<uint32_t>(_regs[classId]) >= n);
-
-    _regs[classId] += static_cast<uint8_t>(n);
-  }
-
-  //! Add Gp register count.
-  ASMJIT_INLINE void addGp(uint32_t n) { add(kX86RegClassGp, n); }
-  //! Add Mm register count.
-  ASMJIT_INLINE void addMm(uint32_t n) { add(kX86RegClassMm, n); }
-  //! Add K register count.
-  ASMJIT_INLINE void addK(uint32_t n) { add(kX86RegClassK, n); }
-  //! Add Xmm/Ymm/Zmm register count.
-  ASMJIT_INLINE void addXyz(uint32_t n) { add(kX86RegClassXyz, n); }
-
-  // --------------------------------------------------------------------------
-  // [Misc]
-  // --------------------------------------------------------------------------
-
-  //! Build a register indexes, based on register's `count`.
-  //!
-  //! Register index is used by \ref `X86Compiler` in per-instruction register
-  //! data. Indexes are sorted by register class in Gp, Mm, K, and Xmm/Ymm/Zmm
-  //! order.
-  ASMJIT_INLINE void indexFromRegCount(const X86RegCount& count) {
-    uint32_t x = count._regs[0];
-    uint32_t y;
-
-    _regs[0] = static_cast<uint8_t>(0);
-    _regs[1] = static_cast<uint8_t>(x);
-
-    x = x + count._regs[1];
-    y = x + count._regs[2];
-
-    ASMJIT_ASSERT(x <= 0xFF);
-    ASMJIT_ASSERT(y <= 0xFF);
-
-    _regs[2] = static_cast<uint8_t>(x);
-    _regs[3] = static_cast<uint8_t>(y);
-  }
-
-  // --------------------------------------------------------------------------
-  // [Members]
-  // --------------------------------------------------------------------------
-
-  union {
-    struct {
-      //! Count of Gp registers.
-      uint8_t _gp;
-      //! Count of Mm registers.
-      uint8_t _mm;
-      //! Count of K registers.
-      uint8_t _k;
-      //! Count of Xmm/Ymm/Zmm registers.
-      uint8_t _xyz;
-      //! \internal
-      uint8_t _reserved[3];
-    };
-
-    uint8_t _regs[4];
-    uint32_t _packed;
-  };
-};
-
-// ============================================================================
-// [asmjit::X86RegMask]
-// ============================================================================
-
-//! \internal
-//!
-//! X86/X64 registers mask (Gp, Mm, K, Xmm/Ymm/Zmm).
-struct X86RegMask {
-  // --------------------------------------------------------------------------
-  // [Reset]
-  // --------------------------------------------------------------------------
-
-  //! Reset all register masks to zero.
-  ASMJIT_INLINE void reset() {
-    _packed.reset();
-  }
-
-  // --------------------------------------------------------------------------
-  // [IsEmpty / Has]
-  // --------------------------------------------------------------------------
-
-  //! Get whether all register masks are zero (empty).
-  ASMJIT_INLINE bool isEmpty() const {
-    return _packed.isZero();
-  }
-
-  ASMJIT_INLINE bool has(uint32_t classId, uint32_t mask = 0xFFFFFFFF) const {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-
-    switch (classId) {
-      case kX86RegClassGp : return (static_cast<uint32_t>(_gp ) & mask) != 0;
-      case kX86RegClassMm : return (static_cast<uint32_t>(_mm ) & mask) != 0;
-      case kX86RegClassK  : return (static_cast<uint32_t>(_k  ) & mask) != 0;
-      case kX86RegClassXyz: return (static_cast<uint32_t>(_xyz) & mask) != 0;
-    }
-
-    return false;
-  }
-
-  ASMJIT_INLINE bool hasGp(uint32_t mask = 0xFFFFFFFF) const { return has(kX86RegClassGp, mask); }
-  ASMJIT_INLINE bool hasMm(uint32_t mask = 0xFFFFFFFF) const { return has(kX86RegClassMm, mask); }
-  ASMJIT_INLINE bool hasK(uint32_t mask = 0xFFFFFFFF) const { return has(kX86RegClassK, mask); }
-  ASMJIT_INLINE bool hasXyz(uint32_t mask = 0xFFFFFFFF) const { return has(kX86RegClassXyz, mask); }
-
-  // --------------------------------------------------------------------------
-  // [Get]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_INLINE uint32_t get(uint32_t classId) const {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-
-    switch (classId) {
-      case kX86RegClassGp : return _gp;
-      case kX86RegClassMm : return _mm;
-      case kX86RegClassK  : return _k;
-      case kX86RegClassXyz: return _xyz;
-    }
-
-    return 0;
-  }
-
-  ASMJIT_INLINE uint32_t getGp() const { return get(kX86RegClassGp); }
-  ASMJIT_INLINE uint32_t getMm() const { return get(kX86RegClassMm); }
-  ASMJIT_INLINE uint32_t getK() const { return get(kX86RegClassK); }
-  ASMJIT_INLINE uint32_t getXyz() const { return get(kX86RegClassXyz); }
-
-  // --------------------------------------------------------------------------
-  // [Zero]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_INLINE void zero(uint32_t classId) {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-
-    switch (classId) {
-      case kX86RegClassGp : _gp  = 0; break;
-      case kX86RegClassMm : _mm  = 0; break;
-      case kX86RegClassK  : _k   = 0; break;
-      case kX86RegClassXyz: _xyz = 0; break;
-    }
-  }
-
-  ASMJIT_INLINE void zeroGp() { zero(kX86RegClassGp); }
-  ASMJIT_INLINE void zeroMm() { zero(kX86RegClassMm); }
-  ASMJIT_INLINE void zeroK() { zero(kX86RegClassK); }
-  ASMJIT_INLINE void zeroXyz() { zero(kX86RegClassXyz); }
-
-  // --------------------------------------------------------------------------
-  // [Set]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_INLINE void set(const X86RegMask& other) {
-    _packed = other._packed;
-  }
-
-  ASMJIT_INLINE void set(uint32_t classId, uint32_t mask) {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-
-    switch (classId) {
-      case kX86RegClassGp : _gp  = static_cast<uint16_t>(mask); break;
-      case kX86RegClassMm : _mm  = static_cast<uint8_t >(mask); break;
-      case kX86RegClassK  : _k   = static_cast<uint8_t >(mask); break;
-      case kX86RegClassXyz: _xyz = static_cast<uint32_t>(mask); break;
-    }
-  }
-
-  ASMJIT_INLINE void setGp(uint32_t mask) { return set(kX86RegClassGp, mask); }
-  ASMJIT_INLINE void setMm(uint32_t mask) { return set(kX86RegClassMm, mask); }
-  ASMJIT_INLINE void setK(uint32_t mask) { return set(kX86RegClassK, mask); }
-  ASMJIT_INLINE void setXyz(uint32_t mask) { return set(kX86RegClassXyz, mask); }
-
-  // --------------------------------------------------------------------------
-  // [And]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_INLINE void and_(const X86RegMask& other) {
-    _packed.and_(other._packed);
-  }
-
-  ASMJIT_INLINE void and_(uint32_t classId, uint32_t mask) {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-
-    switch (classId) {
-      case kX86RegClassGp : _gp  &= static_cast<uint16_t>(mask); break;
-      case kX86RegClassMm : _mm  &= static_cast<uint8_t >(mask); break;
-      case kX86RegClassK  : _k   &= static_cast<uint8_t >(mask); break;
-      case kX86RegClassXyz: _xyz &= static_cast<uint32_t>(mask); break;
-    }
-  }
-
-  ASMJIT_INLINE void andGp(uint32_t mask) { and_(kX86RegClassGp, mask); }
-  ASMJIT_INLINE void andMm(uint32_t mask) { and_(kX86RegClassMm, mask); }
-  ASMJIT_INLINE void andK(uint32_t mask) { and_(kX86RegClassK, mask); }
-  ASMJIT_INLINE void andXyz(uint32_t mask) { and_(kX86RegClassXyz, mask); }
-
-  // --------------------------------------------------------------------------
-  // [AndNot]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_INLINE void andNot(const X86RegMask& other) {
-    _packed.andNot(other._packed);
-  }
-
-  ASMJIT_INLINE void andNot(uint32_t classId, uint32_t mask) {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-
-    switch (classId) {
-      case kX86RegClassGp : _gp  &= ~static_cast<uint16_t>(mask); break;
-      case kX86RegClassMm : _mm  &= ~static_cast<uint8_t >(mask); break;
-      case kX86RegClassK  : _k   &= ~static_cast<uint8_t >(mask); break;
-      case kX86RegClassXyz: _xyz &= ~static_cast<uint32_t>(mask); break;
-    }
-  }
-
-  ASMJIT_INLINE void andNotGp(uint32_t mask) { andNot(kX86RegClassGp, mask); }
-  ASMJIT_INLINE void andNotMm(uint32_t mask) { andNot(kX86RegClassMm, mask); }
-  ASMJIT_INLINE void andNotK(uint32_t mask) { andNot(kX86RegClassK, mask); }
-  ASMJIT_INLINE void andNotXyz(uint32_t mask) { andNot(kX86RegClassXyz, mask); }
-
-  // --------------------------------------------------------------------------
-  // [Or]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_INLINE void or_(const X86RegMask& other) {
-    _packed.or_(other._packed);
-  }
-
-  ASMJIT_INLINE void or_(uint32_t classId, uint32_t mask) {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-    switch (classId) {
-      case kX86RegClassGp : _gp  |= static_cast<uint16_t>(mask); break;
-      case kX86RegClassMm : _mm  |= static_cast<uint8_t >(mask); break;
-      case kX86RegClassK  : _k   |= static_cast<uint8_t >(mask); break;
-      case kX86RegClassXyz: _xyz |= static_cast<uint32_t>(mask); break;
-    }
-  }
-
-  ASMJIT_INLINE void orGp(uint32_t mask) { return or_(kX86RegClassGp, mask); }
-  ASMJIT_INLINE void orMm(uint32_t mask) { return or_(kX86RegClassMm, mask); }
-  ASMJIT_INLINE void orK(uint32_t mask) { return or_(kX86RegClassK, mask); }
-  ASMJIT_INLINE void orXyz(uint32_t mask) { return or_(kX86RegClassXyz, mask); }
-
-  // --------------------------------------------------------------------------
-  // [Xor]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_INLINE void xor_(const X86RegMask& other) {
-    _packed.xor_(other._packed);
-  }
-
-  ASMJIT_INLINE void xor_(uint32_t classId, uint32_t mask) {
-    ASMJIT_ASSERT(classId < _kX86RegClassManagedCount);
-
-    switch (classId) {
-      case kX86RegClassGp : _gp  ^= static_cast<uint16_t>(mask); break;
-      case kX86RegClassMm : _mm  ^= static_cast<uint8_t >(mask); break;
-      case kX86RegClassK  : _k   ^= static_cast<uint8_t >(mask); break;
-      case kX86RegClassXyz: _xyz ^= static_cast<uint32_t>(mask); break;
-    }
-  }
-
-  ASMJIT_INLINE void xorGp(uint32_t mask) { xor_(kX86RegClassGp, mask); }
-  ASMJIT_INLINE void xorMm(uint32_t mask) { xor_(kX86RegClassMm, mask); }
-  ASMJIT_INLINE void xorK(uint32_t mask) { xor_(kX86RegClassK, mask); }
-  ASMJIT_INLINE void xorXyz(uint32_t mask) { xor_(kX86RegClassXyz, mask); }
-
-  // --------------------------------------------------------------------------
-  // [Members]
-  // --------------------------------------------------------------------------
-
-  union {
-    struct {
-      //! Gp registers mask (16 bits).
-      uint16_t _gp;
-      //! Mm registers mask (8 bits).
-      uint8_t _mm;
-      //! K registers mask (8 bits).
-      uint8_t _k;
-      //! Xmm/Ymm/Zmm registers mask (32 bits).
-      uint32_t _xyz;
-    };
-
-    //! Packed masks.
-    UInt64 _packed;
-  };
-};
-
-// ============================================================================
-// [asmjit::X86Reg]
-// ============================================================================
-
-//! X86/X86 register base class.
-struct X86Reg : public Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy X86 register.
-  ASMJIT_INLINE X86Reg() : Reg() {}
-  //! Create a reference to `other` X86 register.
-  ASMJIT_INLINE X86Reg(const X86Reg& other) : Reg(other) {}
-  //! Create a reference to `other` X86 register and change the index to `index`.
-  ASMJIT_INLINE X86Reg(const X86Reg& other, uint32_t index) : Reg(other, index) {}
-  //! Create a custom X86 register.
-  ASMJIT_INLINE X86Reg(uint32_t type, uint32_t index, uint32_t size) : Reg(type, index, size) {}
-  //! Create non-initialized X86 register.
-  explicit ASMJIT_INLINE X86Reg(const _NoInit&) : Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86Reg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86Reg)
-
-  //! Get whether the register is Gp register.
-  ASMJIT_INLINE bool isGp() const { return _vreg.type <= kX86RegTypeGpq; }
-  //! Get whether the register is Gp byte (8-bit) register.
-  ASMJIT_INLINE bool isGpb() const { return _vreg.type <= _kX86RegTypePatchedGpbHi; }
-  //! Get whether the register is Gp lo-byte (8-bit) register.
-  ASMJIT_INLINE bool isGpbLo() const { return _vreg.type == kX86RegTypeGpbLo; }
-  //! Get whether the register is Gp hi-byte (8-bit) register.
-  ASMJIT_INLINE bool isGpbHi() const { return _vreg.type == kX86RegTypeGpbHi; }
-  //! Get whether the register is Gp word (16-bit) register.
-  ASMJIT_INLINE bool isGpw() const { return _vreg.type == kX86RegTypeGpw; }
-  //! Get whether the register is Gp dword (32-bit) register.
-  ASMJIT_INLINE bool isGpd() const { return _vreg.type == kX86RegTypeGpd; }
-  //! Get whether the register is Gp qword (64-bit) register.
-  ASMJIT_INLINE bool isGpq() const { return _vreg.type == kX86RegTypeGpq; }
-
-  //! Get whether the register is Fp register.
-  ASMJIT_INLINE bool isFp() const { return _vreg.type == kX86RegTypeFp; }
-  //! Get whether the register is Mm (64-bit) register.
-  ASMJIT_INLINE bool isMm() const { return _vreg.type == kX86RegTypeMm; }
-
-  //! Get whether the register is K (64-bit) register.
-  ASMJIT_INLINE bool isK() const { return _vreg.type == kX86RegTypeK; }
-
-  //! Get whether the register is Xmm (128-bit) register.
-  ASMJIT_INLINE bool isXmm() const { return _vreg.type == kX86RegTypeXmm; }
-  //! Get whether the register is Ymm (256-bit) register.
-  ASMJIT_INLINE bool isYmm() const { return _vreg.type == kX86RegTypeYmm; }
-  //! Get whether the register is Zmm (512-bit) register.
-  ASMJIT_INLINE bool isZmm() const { return _vreg.type == kX86RegTypeZmm; }
-
-  //! Get whether the register is RIP.
-  ASMJIT_INLINE bool isRip() const { return _vreg.type == kX86RegTypeRip; }
-  //! Get whether the register is Segment.
-  ASMJIT_INLINE bool isSeg() const { return _vreg.type == kX86RegTypeSeg; }
-
-  // --------------------------------------------------------------------------
-  // [Statics]
-  // --------------------------------------------------------------------------
-
-  //! Get whether the `op` operand is Gpb-Lo or Gpb-Hi register.
-  static ASMJIT_INLINE bool isGpbReg(const Operand& op) {
-    const uint32_t mask = IntUtil::pack32_2x8_1x16(
-      0xFF, 0xFF, ~(_kX86RegTypePatchedGpbHi << 8) & 0xFF00);
-
-    return (op._packed[0].u32[0] & mask) == IntUtil::pack32_2x8_1x16(kOperandTypeReg, 1, 0x0000);
-  }
-};
-
-// ============================================================================
-// [asmjit::X86RipReg]
-// ============================================================================
-
-//! X86/X64 RIP register.
-struct X86RipReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a RIP register.
-  ASMJIT_INLINE X86RipReg() : X86Reg(kX86RegTypeRip, 0, 0) {}
-  //! Create a reference to `other` RIP register.
-  ASMJIT_INLINE X86RipReg(const X86RipReg& other) : X86Reg(other) {}
-  //! Create non-initialized RIP register.
-  explicit ASMJIT_INLINE X86RipReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86RipReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86RipReg)
-};
-
-// ============================================================================
-// [asmjit::X86SegReg]
-// ============================================================================
-
-//! X86/X64 segment register.
-struct X86SegReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy segment register.
-  ASMJIT_INLINE X86SegReg() : X86Reg() {}
-  //! Create a reference to `other` segment register.
-  ASMJIT_INLINE X86SegReg(const X86SegReg& other) : X86Reg(other) {}
-  //! Create a reference to `other` segment register and change the index to `index`.
-  ASMJIT_INLINE X86SegReg(const X86SegReg& other, uint32_t index) : X86Reg(other, index) {}
-  //! Create a custom segment register.
-  ASMJIT_INLINE X86SegReg(uint32_t type, uint32_t index, uint32_t size) : X86Reg(type, index, size) {}
-  //! Create non-initialized segment register.
-  explicit ASMJIT_INLINE X86SegReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86SegReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86SegReg)
-};
-
-// ============================================================================
-// [asmjit::X86GpReg]
-// ============================================================================
-
-//! X86/X64 Gpb/Gpw/Gpd/Gpq register.
-struct X86GpReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy Gp register.
-  ASMJIT_INLINE X86GpReg() : X86Reg() {}
-  //! Create a reference to `other` Gp register.
-  ASMJIT_INLINE X86GpReg(const X86GpReg& other) : X86Reg(other) {}
-  //! Create a reference to `other` Gp register and change the index to `index`.
-  ASMJIT_INLINE X86GpReg(const X86GpReg& other, uint32_t index) : X86Reg(other, index) {}
-  //! Create a custom Gp register.
-  ASMJIT_INLINE X86GpReg(uint32_t type, uint32_t index, uint32_t size) : X86Reg(type, index, size) {}
-  //! Create non-initialized Gp register.
-  explicit ASMJIT_INLINE X86GpReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86GpReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86GpReg)
-
-  // --------------------------------------------------------------------------
-  // [X86GpReg Cast]
-  // --------------------------------------------------------------------------
-
-  //! Cast this register to 8-bit (LO) part.
-  ASMJIT_INLINE X86GpReg r8() const { return X86GpReg(kX86RegTypeGpbLo, getRegIndex(), 1); }
-  //! Cast this register to 8-bit (LO) part.
-  ASMJIT_INLINE X86GpReg r8Lo() const { return X86GpReg(kX86RegTypeGpbLo, getRegIndex(), 1); }
-  //! Cast this register to 8-bit (HI) part.
-  ASMJIT_INLINE X86GpReg r8Hi() const { return X86GpReg(kX86RegTypeGpbHi, getRegIndex(), 1); }
-
-  //! Cast this register to 16-bit.
-  ASMJIT_INLINE X86GpReg r16() const { return X86GpReg(kX86RegTypeGpw, getRegIndex(), 2); }
-  //! Cast this register to 32-bit.
-  ASMJIT_INLINE X86GpReg r32() const { return X86GpReg(kX86RegTypeGpd, getRegIndex(), 4); }
-  //! Cast this register to 64-bit.
-  ASMJIT_INLINE X86GpReg r64() const { return X86GpReg(kX86RegTypeGpq, getRegIndex(), 8); }
-};
-
-// ============================================================================
-// [asmjit::X86FpReg]
-// ============================================================================
-
-//! X86/X64 80-bit Fp register.
-struct X86FpReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy Fp register.
-  ASMJIT_INLINE X86FpReg() : X86Reg() {}
-  //! Create a reference to `other` Fp register.
-  ASMJIT_INLINE X86FpReg(const X86FpReg& other) : X86Reg(other) {}
-  //! Create a reference to `other` Fp register and change the index to `index`.
-  ASMJIT_INLINE X86FpReg(const X86FpReg& other, uint32_t index) : X86Reg(other, index) {}
-  //! Create a custom Fp register.
-  ASMJIT_INLINE X86FpReg(uint32_t type, uint32_t index, uint32_t size) : X86Reg(type, index, size) {}
-  //! Create non-initialized Fp register.
-  explicit ASMJIT_INLINE X86FpReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86FpReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86FpReg)
-};
-
-// ============================================================================
-// [asmjit::X86MmReg]
-// ============================================================================
-
-//! X86/X64 64-bit Mm register (MMX+).
-//!
-//! Structure of MMX register and it's memory mapping:
-//!
-//! ~~~
-//!       Memory Bytes
-//! +--+--+--+--+--+--+--+--+
-//! |00|01|02|03|04|05|06|07|
-//! +--+--+--+--+--+--+--+--+
-//!
-//!       MMX Register
-//! +-----------------------+
-//! |         QWord         |
-//! +-----------+-----------+
-//! |  HI-DWord |  LO-DWord |
-//! +-----------+-----------+
-//! |  W3 |  W2 |  W1 |  W0 |
-//! +--+--+--+--+--+--+--+--+
-//! |07|06|05|04|03|02|01|00|
-//! +--+--+--+--+--+--+--+--+
-//! ~~~
-//!
-//! Move instruction semantics:
-//!
-//!   - `movd` - writes 4-bytes in `LO-DWord` and zeroes `HI-DWord`.
-//!   - `movq` - writes 8-bytes in `QWord`.
-struct X86MmReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy Mm register.
-  ASMJIT_INLINE X86MmReg() : X86Reg() {}
-  //! Create a reference to `other` Mm register.
-  ASMJIT_INLINE X86MmReg(const X86MmReg& other) : X86Reg(other) {}
-  //! Create a reference to `other` Mm register and change the index to `index`.
-  ASMJIT_INLINE X86MmReg(const X86MmReg& other, uint32_t index) : X86Reg(other, index) {}
-  //! Create a custom Mm register.
-  ASMJIT_INLINE X86MmReg(uint32_t type, uint32_t index, uint32_t size) : X86Reg(type, index, size) {}
-  //! Create non-initialized Mm register.
-  explicit ASMJIT_INLINE X86MmReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86MmReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86MmReg)
-};
-
-// ============================================================================
-// [asmjit::X86KReg]
-// ============================================================================
-
-//! X86/X64 64-bit K register (AVX512+).
-struct X86KReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy K register.
-  ASMJIT_INLINE X86KReg() : X86Reg() {}
-  //! Create a reference to `other` K register.
-  ASMJIT_INLINE X86KReg(const X86KReg& other) : X86Reg(other) {}
-  //! Create a reference to `other` K register and change the index to `index`.
-  ASMJIT_INLINE X86KReg(const X86KReg& other, uint32_t index) : X86Reg(other, index) {}
-  //! Create a custom K register.
-  ASMJIT_INLINE X86KReg(uint32_t type, uint32_t index, uint32_t size) : X86Reg(type, index, size) {}
-  //! Create non-initialized K register.
-  explicit ASMJIT_INLINE X86KReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86KReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86KReg)
-};
-
-// ============================================================================
-// [asmjit::X86XmmReg]
-// ============================================================================
-
-//! X86/X64 128-bit Xmm register (SSE+).
-//!
-//! Structure of XMM register and it's memory mapping:
-//!
-//! ~~~
-//!                   Memory Bytes
-//! +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-//! |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|
-//! +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-//!
-//!                   XMM Register
-//! +-----------------------------------------------+
-//! |                     OWord                     |
-//! +-----------------------+-----------------------+
-//! |      HI-QWord/PD      |      LO-QWord/SD      |
-//! +-----------+-----------+-----------+-----------+
-//! |   D3/PS   |   D2/PS   |   D1/PS   |   D0/SS   |
-//! +-----------+-----------+-----------+-----------+
-//! |  W7 |  W6 |  W5 |  W4 |  W3 |  W2 |  W1 |  W0 |
-//! +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-//! |15|14|13|12|11|10|09|08|07|06|05|04|03|02|01|00|
-//! +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-//! ~~~
-//!
-//! Move instruction semantics:
-//!
-//!   - `movd` - writes 4-bytes in `D0` and zeroes the rest.
-//!   - `movq` - writes 8-bytes in `Lo-QWord` and zeroes the rest.
-//!   - `movq2dq` - writes 8 bytes in `Lo-QWord` and zeroes the rest.
-//!
-//!   - `movss` - writes 4-bytes in `D0`
-//!       (the rest is zeroed only if the source operand is a memory location).
-//!   - `movsd` - writes 8-bytes in `Lo-QWord`
-//!       (the rest is zeroed only if the source operand is a memory location).
-//!
-//!   - `movaps`,
-//!     `movups`,
-//!     `movapd`,
-//!     `movupd`,
-//!     `movdqu`,
-//!     `movdqa`,
-//!     `lddqu` - writes 16-bytes in `OWord`.
-//!
-//!   - `movlps`,
-//!     `movlpd`,
-//!     `movhlps` - writes 8-bytes in `Lo-QWord` and keeps the rest untouched.
-//!
-//!   - `movhps`,
-//!     `movhpd`,
-//!     `movlhps` - writes 8-bytes in `Hi-QWord` and keeps the rest untouched.
-//!
-//!   - `movddup`,
-//!   - `movsldup`,
-//!   - `movshdup` - writes 16 bytes in `OWord`.
-struct X86XmmReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy Xmm register.
-  ASMJIT_INLINE X86XmmReg() : X86Reg() {}
-  //! Create a reference to `other` Xmm register.
-  ASMJIT_INLINE X86XmmReg(const X86XmmReg& other) : X86Reg(other) {}
-  //! Create a reference to `other` Xmm register and change the index to `index`.
-  ASMJIT_INLINE X86XmmReg(const X86XmmReg& other, uint32_t index) : X86Reg(other, index) {}
-  //! Create a custom Xmm register.
-  ASMJIT_INLINE X86XmmReg(uint32_t type, uint32_t index, uint32_t size) : X86Reg(type, index, size) {}
-  //! Create non-initialized Xmm register.
-  explicit ASMJIT_INLINE X86XmmReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86XmmReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86XmmReg)
-
-  // --------------------------------------------------------------------------
-  // [X86XmmReg Cast]
-  // --------------------------------------------------------------------------
-
-  //! Cast this register to Xmm (clone).
-  ASMJIT_INLINE X86XmmReg xmm() const { return X86XmmReg(kX86RegTypeXmm, getRegIndex(), 16); }
-  //! Cast this register to Ymm.
-  ASMJIT_INLINE X86YmmReg ymm() const;
-  //! Cast this register to Zmm.
-  ASMJIT_INLINE X86ZmmReg zmm() const;
-};
-
-// ============================================================================
-// [asmjit::X86YmmReg]
-// ============================================================================
-
-//! X86/X64 256-bit Ymm register (AVX+).
-//!
-//! Structure of YMM register and it's memory mapping:
-//!
-//! ~~~
-//!                                           Memory Bytes
-//! +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-//! |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|
-//! +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-//!
-//!                                           YMM Register
-//! +-----------------------------------------------+-----------------------------------------------+
-//! |                  HI-DQWord                    |                  LO-DQWord                    |
-//! +-----------------------+-----------------------+-----------------------+-----------------------+
-//! |         Q3/PD         |         Q2/PD         |         Q1/PD         |         Q0/SD         |
-//! +-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+
-//! |   D7/PS   |   D6/PS   |   D5/PS   |   D4/PS   |   D3/PS   |   D2/PS   |   D1/PS   |   D0/SS   |
-//! +-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+
-//! | W15 | W14 | W13 | W12 | W11 | W10 |  W9 |  W8 |  W7 |  W6 |  W5 |  W4 |  W3 |  W2 |  W1 |  W0 |
-//! +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-//! |31|30|29|28|27|26|25|24|23|22|21|20|19|18|17|16|15|14|13|12|11|10|09|08|07|06|05|04|03|02|01|00|
-//! +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
-//! ~~~
-struct X86YmmReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy Ymm register.
-  ASMJIT_INLINE X86YmmReg() : X86Reg() {}
-  //! Create a reference to `other` Ymm register.
-  ASMJIT_INLINE X86YmmReg(const X86YmmReg& other) : X86Reg(other) {}
-  //! Create a reference to `other` Ymm register and change the index to `index`.
-  ASMJIT_INLINE X86YmmReg(const X86YmmReg& other, uint32_t index) : X86Reg(other, index) {}
-  //! Create a custom Ymm register.
-  ASMJIT_INLINE X86YmmReg(uint32_t type, uint32_t index, uint32_t size) : X86Reg(type, index, size) {}
-  //! Create non-initialized Ymm register.
-  explicit ASMJIT_INLINE X86YmmReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86YmmReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86YmmReg)
-
-  // --------------------------------------------------------------------------
-  // [X86YmmReg Cast]
-  // --------------------------------------------------------------------------
-
-  //! Cast this register to Xmm.
-  ASMJIT_INLINE X86XmmReg xmm() const { return X86XmmReg(kX86RegTypeXmm, getRegIndex(), 16); }
-  //! Cast this register to Ymm (clone).
-  ASMJIT_INLINE X86YmmReg ymm() const { return X86YmmReg(kX86RegTypeYmm, getRegIndex(), 32); }
-  //! Cast this register to Zmm.
-  ASMJIT_INLINE X86ZmmReg zmm() const;
-};
-
-ASMJIT_INLINE X86YmmReg X86XmmReg::ymm() const { return X86YmmReg(kX86RegTypeYmm, getRegIndex(), 32); }
+namespace asmjit {
 
 // ============================================================================
-// [asmjit::X86ZmmReg]
+// [Forward Declarations]
 // ============================================================================
 
-//! X86/X64 512-bit Zmm register (AVX512+).
-struct X86ZmmReg : public X86Reg {
-  // --------------------------------------------------------------------------
-  // [Construction / Destruction]
-  // --------------------------------------------------------------------------
-
-  //! Create a dummy Zmm register.
-  ASMJIT_INLINE X86ZmmReg() : X86Reg() {}
-  //! Create a reference to `other` Zmm register.
-  ASMJIT_INLINE X86ZmmReg(const X86ZmmReg& other) : X86Reg(other) {}
-  //! Create a reference to `other` Zmm register and change the index to `index`.
-  ASMJIT_INLINE X86ZmmReg(const X86ZmmReg& other, uint32_t index) : X86Reg(other, index) {}
-  //! Create a custom Zmm register.
-  ASMJIT_INLINE X86ZmmReg(uint32_t type, uint32_t index, uint32_t size) : X86Reg(type, index, size) {}
-  //! Create non-initialized Zmm register.
-  explicit ASMJIT_INLINE X86ZmmReg(const _NoInit&) : X86Reg(NoInit) {}
-
-  // --------------------------------------------------------------------------
-  // [X86ZmmReg Specific]
-  // --------------------------------------------------------------------------
-
-  ASMJIT_REG_OP(X86ZmmReg)
-
-  // --------------------------------------------------------------------------
-  // [X86ZmmReg Cast]
-  // --------------------------------------------------------------------------
-
-  //! Cast this register to Xmm.
-  ASMJIT_INLINE X86XmmReg xmm() const { return X86XmmReg(kX86RegTypeXmm, getRegIndex(), 16); }
-  //! Cast this register to Ymm.
-  ASMJIT_INLINE X86YmmReg ymm() const { return X86YmmReg(kX86RegTypeYmm, getRegIndex(), 32); }
-  //! Cast this register to Zmm (clone).
-  ASMJIT_INLINE X86ZmmReg zmm() const { return X86ZmmReg(kX86RegTypeZmm, getRegIndex(), 64); }
-};
-
-ASMJIT_INLINE X86ZmmReg X86XmmReg::zmm() const { return X86ZmmReg(kX86RegTypeZmm, getRegIndex(), 64); }
-ASMJIT_INLINE X86ZmmReg X86YmmReg::zmm() const { return X86ZmmReg(kX86RegTypeZmm, getRegIndex(), 64); }
+class X86Mem;
+class X86Reg;
+class X86Vec;
+
+class X86Gp;
+class X86Gpb;
+class X86GpbLo;
+class X86GpbHi;
+class X86Gpw;
+class X86Gpd;
+class X86Gpq;
+class X86Fp;
+class X86Mm;
+class X86KReg;
+class X86Xmm;
+class X86Ymm;
+class X86Zmm;
+class X86Bnd;
+class X86Seg;
+class X86Rip;
+class X86CReg;
+class X86DReg;
+
+//! \addtogroup asmjit_x86
+//! \{
 
 // ============================================================================
 // [asmjit::X86Mem]
 // ============================================================================
 
-//! X86 memory operand.
-struct X86Mem : public BaseMem {
+//! Memory operand (X86).
+class X86Mem : public Mem {
+public:
+  //! Additional bits of operand's signature used by `X86Mem`.
+  ASMJIT_ENUM(AdditionalBits) {
+    kSignatureMemShiftShift   = 19,
+    kSignatureMemShiftBits    = 0x03U,
+    kSignatureMemShiftMask    = kSignatureMemShiftBits << kSignatureMemShiftShift,
+
+    kSignatureMemSegmentShift = 21,
+    kSignatureMemSegmentBits  = 0x07U,
+    kSignatureMemSegmentMask  = kSignatureMemSegmentBits << kSignatureMemSegmentShift
+  };
+
   // --------------------------------------------------------------------------
   // [Construction / Destruction]
   // --------------------------------------------------------------------------
 
-  ASMJIT_INLINE X86Mem() : BaseMem(NoInit) {
-    reset();
-  }
-
-  ASMJIT_INLINE X86Mem(const Label& label, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeLabel, 0, label._base.id);
-    _init_packed_d2_d3(kInvalidValue, disp);
-  }
-
-  ASMJIT_INLINE X86Mem(const Label& label, const X86GpReg& index, uint32_t shift, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
-
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeLabel,
-      (kX86MemVSibGpz << kX86MemVSibIndex)
-        + (shift << kX86MemShiftIndex),
-      label.getId());
-    _vmem.index = index.getRegIndex();
-    _vmem.displacement = disp;
-  }
-
-  ASMJIT_INLINE X86Mem(const X86GpReg& base, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeBaseIndex,
-      _getGpdFlags(base)
-        + (kX86MemVSibGpz << kX86MemVSibIndex),
-      base.getRegIndex());
-    _init_packed_d2_d3(kInvalidValue, disp);
-  }
-
-  ASMJIT_INLINE X86Mem(const X86GpReg& base, const X86GpReg& index, uint32_t shift, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
-
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeBaseIndex,
-      _getGpdFlags(base) + (shift << kX86MemShiftIndex),
-      base.getRegIndex());
-    _vmem.index = index.getRegIndex();
-    _vmem.displacement = disp;
-  }
+  //! Construct a default `X86Mem` operand, that points to [0].
+  ASMJIT_INLINE X86Mem() noexcept : Mem(NoInit) { reset(); }
+  ASMJIT_INLINE X86Mem(const X86Mem& other) noexcept : Mem(other) {}
 
-  ASMJIT_INLINE X86Mem(const X86GpReg& base, const X86XmmReg& index, uint32_t shift, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
+  ASMJIT_INLINE X86Mem(const Label& base, int32_t off, uint32_t size = 0, uint32_t flags = 0) noexcept : Mem(NoInit) {
+    uint32_t signature = (Label::kLabelTag << kSignatureMemBaseTypeShift ) |
+                         (size             << kSignatureSizeShift        ) ;
 
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeBaseIndex,
-      _getGpdFlags(base)
-        + (kX86MemVSibXmm << kX86MemVSibIndex)
-        + (shift << kX86MemShiftIndex),
-      base.getRegIndex());
-    _vmem.index = index.getRegIndex();
-    _vmem.displacement = disp;
+    _init_packed_d0_d1(kOpMem | signature | flags, 0);
+    _mem.base = base.getId();
+    _mem.offsetLo32 = static_cast<uint32_t>(off);
   }
 
-  ASMJIT_INLINE X86Mem(const X86GpReg& base, const X86YmmReg& index, uint32_t shift, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
+  ASMJIT_INLINE X86Mem(const Label& base, const Reg& index, uint32_t shift, int32_t off, uint32_t size = 0, uint32_t flags = 0) noexcept : Mem(NoInit) {
+    ASMJIT_ASSERT(shift <= kSignatureMemShiftMask);
+    uint32_t signature = (Label::kLabelTag << kSignatureMemBaseTypeShift ) |
+                         (index.getType()  << kSignatureMemIndexTypeShift) |
+                         (shift            << kSignatureMemShiftShift    ) |
+                         (size             << kSignatureSizeShift        ) ;
 
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeBaseIndex,
-      _getGpdFlags(base)
-        + (kX86MemVSibYmm << kX86MemVSibIndex)
-        + (shift << kX86MemShiftIndex),
-      base.getRegIndex());
-    _vmem.index = index.getRegIndex();
-    _vmem.displacement = disp;
+    _init_packed_d0_d1(kOpMem | signature | flags, index.getId());
+    _mem.base = base.getId();
+    _mem.offsetLo32 = static_cast<uint32_t>(off);
   }
 
-#if !defined(ASMJIT_DISABLE_COMPILER)
-  ASMJIT_INLINE X86Mem(const Label& label, const X86GpVar& index, uint32_t shift, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
+  ASMJIT_INLINE X86Mem(const Reg& base, int32_t off, uint32_t size = 0, uint32_t flags = 0) noexcept : Mem(NoInit) {
+    uint32_t signature = (base.getType()   << kSignatureMemBaseTypeShift ) |
+                         (size             << kSignatureSizeShift        ) ;
 
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeLabel,
-      (kX86MemVSibGpz << kX86MemVSibIndex)
-        + (shift << kX86MemShiftIndex),
-      label.getId());
-    _vmem.index = ASMJIT_OP_ID(index);
-    _vmem.displacement = disp;
+    _init_packed_d0_d1(kOpMem | signature | flags, 0);
+    _mem.base = base.getId();
+    _mem.offsetLo32 = static_cast<uint32_t>(off);
   }
 
-  ASMJIT_INLINE X86Mem(const X86GpVar& base, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeBaseIndex,
-      _getGpdFlags(reinterpret_cast<const Var&>(base))
-        + (kX86MemVSibGpz << kX86MemVSibIndex),
-      ASMJIT_OP_ID(base));
-    _init_packed_d2_d3(kInvalidValue, disp);
-  }
-
-  ASMJIT_INLINE X86Mem(const X86GpVar& base, const X86GpVar& index, uint32_t shift, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
+  ASMJIT_INLINE X86Mem(const Reg& base, const Reg& index, uint32_t shift, int32_t off, uint32_t size = 0, uint32_t flags = 0) noexcept : Mem(NoInit) {
+    ASMJIT_ASSERT(shift <= kSignatureMemShiftMask);
+    uint32_t signature = (base.getType()   << kSignatureMemBaseTypeShift ) |
+                         (index.getType()  << kSignatureMemIndexTypeShift) |
+                         (shift            << kSignatureMemShiftShift    ) |
+                         (size             << kSignatureSizeShift        ) ;
 
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeBaseIndex,
-      _getGpdFlags(reinterpret_cast<const Var&>(base))
-        + (shift << kX86MemShiftIndex),
-      ASMJIT_OP_ID(base));
-    _vmem.index = ASMJIT_OP_ID(index);
-    _vmem.displacement = disp;
+    _init_packed_d0_d1(kOpMem | signature | flags, index.getId());
+    _mem.base = base.getId();
+    _mem.offsetLo32 = static_cast<uint32_t>(off);
   }
 
-  ASMJIT_INLINE X86Mem(const X86GpVar& base, const X86XmmVar& index, uint32_t shift, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
-
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeBaseIndex,
-      _getGpdFlags(reinterpret_cast<const Var&>(base))
-        + (kX86MemVSibXmm << kX86MemVSibIndex)
-        + (shift << kX86MemShiftIndex),
-      ASMJIT_OP_ID(base));
-    _vmem.index = ASMJIT_OP_ID(index);
-    _vmem.displacement = disp;
+  ASMJIT_INLINE X86Mem(uint64_t base, uint32_t size = 0, uint32_t flags = 0) noexcept : Mem(NoInit) {
+    _init_packed_d0_d1(kOpMem | flags | (size << kSignatureSizeShift), 0);
+    _mem.offset64 = base;
   }
 
-  ASMJIT_INLINE X86Mem(const X86GpVar& base, const X86YmmVar& index, uint32_t shift, int32_t disp, uint32_t size = 0) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
-
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, kMemTypeBaseIndex,
-      _getGpdFlags(reinterpret_cast<const Var&>(base))
-        + (kX86MemVSibYmm << kX86MemVSibIndex)
-        + (shift << kX86MemShiftIndex),
-      ASMJIT_OP_ID(base));
-    _vmem.index = ASMJIT_OP_ID(index);
-    _vmem.displacement = disp;
-  }
+  ASMJIT_INLINE X86Mem(uint64_t base, const Reg& index, uint32_t shift = 0, uint32_t size = 0, uint32_t flags = 0) noexcept : Mem(NoInit) {
+    ASMJIT_ASSERT(shift <= kSignatureMemShiftMask);
+    uint32_t signature = (index.getType()  << kSignatureMemIndexTypeShift) |
+                         (shift            << kSignatureMemShiftShift    ) |
+                         (size             << kSignatureSizeShift        ) ;
 
-  ASMJIT_INLINE X86Mem(const _Init&, uint32_t memType, const X86Var& base, int32_t disp, uint32_t size) : BaseMem(NoInit) {
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, memType, 0, ASMJIT_OP_ID(base));
-    _vmem.index = kInvalidValue;
-    _vmem.displacement = disp;
+    _init_packed_d0_d1(kOpMem | signature | flags, index.getId());
+    _mem.offset64 = base;
   }
 
-  ASMJIT_INLINE X86Mem(const _Init&, uint32_t memType, const X86Var& base, const X86GpVar& index, uint32_t shift, int32_t disp, uint32_t size) : BaseMem(NoInit) {
-    ASMJIT_ASSERT(shift <= 3);
-
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, memType, shift << kX86MemShiftIndex, ASMJIT_OP_ID(base));
-    _vmem.index = ASMJIT_OP_ID(index);
-    _vmem.displacement = disp;
-  }
-#endif // !ASMJIT_DISABLE_COMPILER
+  ASMJIT_INLINE X86Mem(const _Init& init,
+    uint32_t baseType, uint32_t baseId,
+    uint32_t indexType, uint32_t indexId,
+    int32_t off, uint32_t size, uint32_t flags) noexcept : Mem(init, baseType, baseId, indexType, indexId, off, size, flags) {}
 
-  ASMJIT_INLINE X86Mem(const X86Mem& other) : BaseMem(other) {}
-  explicit ASMJIT_INLINE X86Mem(const _NoInit&) : BaseMem(NoInit) {}
+  explicit ASMJIT_INLINE X86Mem(const _NoInit&) noexcept : Mem(NoInit) {}
 
   // --------------------------------------------------------------------------
-  // [X86Mem Specific]
+  // [X86Mem]
   // --------------------------------------------------------------------------
 
-  //! Clone X86Mem operand.
-  ASMJIT_INLINE X86Mem clone() const {
-    return X86Mem(*this);
-  }
+  //! Clone the memory operand.
+  ASMJIT_INLINE X86Mem clone() const noexcept { return X86Mem(*this); }
 
-  //! Reset X86Mem operand.
-  ASMJIT_INLINE void reset() {
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, 0, kMemTypeBaseIndex, 0, kInvalidValue);
-    _init_packed_d2_d3(kInvalidValue, 0);
-  }
+  using Mem::setIndex;
 
-  //! \internal
-  ASMJIT_INLINE void _init(uint32_t memType, uint32_t base, int32_t disp, uint32_t size) {
-    _init_packed_op_sz_b0_b1_id(kOperandTypeMem, size, memType, 0, base);
-    _vmem.index = kInvalidValue;
-    _vmem.displacement = disp;
+  ASMJIT_INLINE void setIndex(const Reg& index, uint32_t shift) noexcept {
+    setIndex(index);
+    setShift(shift);
   }
 
-  // --------------------------------------------------------------------------
-  // [Segment]
-  // --------------------------------------------------------------------------
-
-  //! Get whether the memory operand has segment override prefix.
-  ASMJIT_INLINE bool hasSegment() const {
-    return (_vmem.flags & kX86MemSegMask) != (kX86SegDefault << kX86MemSegIndex);
-  }
+  //! Get if the memory operand has shift (aka scale) constant.
+  ASMJIT_INLINE bool hasShift() const noexcept { return _hasSignatureData(kSignatureMemShiftMask); }
+  //! Get the memory operand's shift (aka scale) constant.
+  ASMJIT_INLINE uint32_t getShift() const noexcept { return _getSignatureData(kSignatureMemShiftBits, kSignatureMemShiftShift); }
+  //! Set the memory operand's shift (aka scale) constant.
+  ASMJIT_INLINE void setShift(uint32_t shift) noexcept { _setSignatureData(shift, kSignatureMemShiftBits, kSignatureMemShiftShift); }
+  //! Reset the memory operand's shift (aka scale) constant to zero.
+  ASMJIT_INLINE void resetShift() noexcept { _signature &= ~kSignatureMemShiftMask; }
 
-  //! Get memory operand segment, see `X86Seg`.
-  ASMJIT_INLINE uint32_t getSegment() const {
-    return (static_cast<uint32_t>(_vmem.flags) >> kX86MemSegIndex) & kX86MemSegBits;
-  }
+  //! Get if the memory operand has a segment override.
+  ASMJIT_INLINE bool hasSegment() const noexcept { return _hasSignatureData(kSignatureMemSegmentMask); }
+  //! Get associated segment override as `X86Seg` operand.
+  ASMJIT_INLINE X86Seg getSegment() const noexcept;
+  //! Set the segment override to `seg`.
+  ASMJIT_INLINE void setSegment(const X86Seg& seg) noexcept;
 
-  //! Set memory operand segment, see `X86Seg`.
-  ASMJIT_INLINE X86Mem& setSegment(uint32_t segIndex) {
-    _vmem.flags = static_cast<uint8_t>(
-      (static_cast<uint32_t>(_vmem.flags) & kX86MemSegMask) + (segIndex << kX86MemSegIndex));
-    return *this;
-  }
+  //! Get segment override as id, see \ref X86Seg::Id.
+  ASMJIT_INLINE uint32_t getSegmentId() const noexcept { return _getSignatureData(kSignatureMemSegmentBits, kSignatureMemSegmentShift); }
+  //! Set the segment override to `id`.
+  ASMJIT_INLINE void setSegmentId(uint32_t sId) noexcept { _setSignatureData(sId, kSignatureMemSegmentBits, kSignatureMemSegmentShift); }
+  //! Reset the segment override.
+  ASMJIT_INLINE void resetSegment() noexcept { _signature &= ~kSignatureMemSegmentMask; }
 
-  //! Set memory operand segment, see `X86Seg`.
-  ASMJIT_INLINE X86Mem& setSegment(const X86SegReg& seg) {
-    return setSegment(seg.getRegIndex());
+  //! Get new memory operand adjusted by `off`.
+  ASMJIT_INLINE X86Mem adjusted(int64_t off) const noexcept {
+    X86Mem result(*this);
+    result.addOffset(off);
+    return result;
   }
 
   // --------------------------------------------------------------------------
-  // [Gpd]
+  // [Operator Overload]
   // --------------------------------------------------------------------------
 
-  //! Get whether the memory operand has 32-bit GP base.
-  ASMJIT_INLINE bool hasGpdBase() const {
-    return (_packed[0].u32[0] & IntUtil::pack32_4x8(0x00, 0x00, 0x00, kX86MemGpdMask)) != 0;
-  }
+  ASMJIT_INLINE X86Mem& operator=(const X86Mem& other) noexcept { copyFrom(other); return *this; }
+};
 
-  //! Set whether the memory operand has 32-bit GP base.
-  ASMJIT_INLINE X86Mem& setGpdBase() {
-    _packed[0].u32[0] |= IntUtil::pack32_4x8(0x00, 0x00, 0x00, kX86MemGpdMask);
-    return *this;
-  }
+// ============================================================================
+// [asmjit::X86Reg]
+// ============================================================================
 
-  //! Set whether the memory operand has 32-bit GP base to `b`.
-  ASMJIT_INLINE X86Mem& setGpdBase(uint32_t b) {
-    _packed[0].u32[0] &=~IntUtil::pack32_4x8(0x00, 0x00, 0x00, kX86MemGpdMask);
-    _packed[0].u32[0] |= IntUtil::pack32_4x8(0x00, 0x00, 0x00, b << kX86MemGpdIndex);
-    return *this;
-  }
+//! Register traits (X86/X64).
+//!
+//! Register traits contains information about a particular register type. It's
+//! used by asmjit to setup register information on-the-fly and to populate
+//! tables that contain register information (this way it's possible to change
+//! register types and kinds without having to reorder these tables).
+template<uint32_t RegType>
+struct X86RegTraits {
+  enum {
+    kValid     = 0,                      //!< RegType is not valid by default.
+    kCount     = 0,                      //!< Count of registers (0 if none).
+    kTypeId    = TypeId::kVoid,          //!< Everything is void by default.
+
+    kType      = 0,                      //!< Zero type by default.
+    kKind      = 0,                      //!< Zero kind by default.
+    kSize      = 0,                      //!< No size by default.
+    kSignature = 0                       //!< No signature by default.
+  };
+};
 
-  // --------------------------------------------------------------------------
-  // [VSib]
-  // --------------------------------------------------------------------------
+//! Register (X86/X64).
+class X86Reg : public Reg {
+public:
+  //! Register type.
+  //!
+  //! Don't change these constants; they are essential to some built-in tables.
+  ASMJIT_ENUM(RegType) {
+    kRegNone      = Reg::kRegNone,       //!< No register type or invalid register.
+
+    kRegGpbLo     = Reg::kRegGp8Lo,      //!< Low GPB register (AL, BL, CL, DL, ...).
+    kRegGpbHi     = Reg::kRegGp8Hi,      //!< High GPB register (AH, BH, CH, DH only).
+    kRegGpw       = Reg::kRegGp16,       //!< GPW register.
+    kRegGpd       = Reg::kRegGp32,       //!< GPD register.
+    kRegGpq       = Reg::kRegGp64,       //!< GPQ register (X64).
+    kRegXmm       = Reg::kRegVec128,     //!< XMM register (SSE+).
+    kRegYmm       = Reg::kRegVec256,     //!< YMM register (AVX+).
+    kRegZmm       = Reg::kRegVec512,     //!< ZMM register (AVX512+).
+    kRegRip       = Reg::kRegIP,         //!< Instruction pointer (EIP, RIP).
+    kRegSeg       = Reg::kRegCustom,     //!< Segment register (None, ES, CS, SS, DS, FS, GS).
+    kRegFp        = Reg::kRegCustom + 1, //!< FPU (x87) register.
+    kRegMm        = Reg::kRegCustom + 2, //!< MMX register.
+    kRegK         = Reg::kRegCustom + 3, //!< K register (AVX512+).
+    kRegBnd       = Reg::kRegCustom + 4, //!< Bound register (BND).
+    kRegCr        = Reg::kRegCustom + 5, //!< Control register (CR).
+    kRegDr        = Reg::kRegCustom + 6, //!< Debug register (DR).
+    kRegCount                            //!< Count of register types.
+  };
 
-  //! Get V-SIB type.
-  ASMJIT_INLINE uint32_t getVSib() const {
-    return (static_cast<uint32_t>(_vmem.flags) >> kX86MemVSibIndex) & kX86MemVSibBits;
-  }
+  //! Register kind.
+  ASMJIT_ENUM(Kind) {
+    kKindGp       = Reg::kKindGp,        //!< GP register kind or none (universal).
+    kKindVec      = Reg::kKindVec,       //!< XMM|YMM|ZMM register kind (universal).
+    kKindMm       = 2,                   //!< MMX register kind (legacy).
+    kKindK        = 3,                   //!< K register kind.
+
+    // These are not managed by CodeCompiler nor used by Func-API:
+    kKindFp       = 4,                   //!< FPU (x87) register kind.
+    kKindCr       = 5,                   //!< Control register kind.
+    kKindDr       = 6,                   //!< Debug register kind.
+    kKindBnd      = 7,                   //!< Bound register kind.
+    kKindSeg      = 8,                   //!< Segment register kind.
+    kKindRip      = 9,                   //!< Instrucion pointer (IP).
+    kKindCount                           //!< Count of all register kinds.
+  };
 
-  //! Set V-SIB type.
-  ASMJIT_INLINE X86Mem& _setVSib(uint32_t vsib) {
-    _packed[0].u32[0] &=~IntUtil::pack32_4x8(0x00, 0x00, 0x00, kX86MemVSibMask);
-    _packed[0].u32[0] |= IntUtil::pack32_4x8(0x00, 0x00, 0x00, vsib << kX86MemVSibIndex);
-    return *this;
+  ASMJIT_DEFINE_ABSTRACT_REG(X86Reg, Reg)
+
+  //! Get if the register is a GPB register (8-bit).
+  ASMJIT_INLINE bool isGpb() const noexcept { return getSize() == 1; }
+  //! Get if the register is RIP.
+  ASMJIT_INLINE bool isRip() const noexcept { return hasSignature(signatureOf(kRegRip)); }
+  //! Get if the register is a segment register.
+  ASMJIT_INLINE bool isSeg() const noexcept { return hasSignature(signatureOf(kRegSeg)); }
+  //! Get if the register is a low GPB register (8-bit).
+  ASMJIT_INLINE bool isGpbLo() const noexcept { return hasSignature(signatureOf(kRegGpbLo)); }
+  //! Get if the register is a high GPB register (8-bit).
+  ASMJIT_INLINE bool isGpbHi() const noexcept { return hasSignature(signatureOf(kRegGpbHi)); }
+  //! Get if the register is a GPW register (16-bit).
+  ASMJIT_INLINE bool isGpw() const noexcept { return hasSignature(signatureOf(kRegGpw)); }
+  //! Get if the register is a GPD register (32-bit).
+  ASMJIT_INLINE bool isGpd() const noexcept { return hasSignature(signatureOf(kRegGpd)); }
+  //! Get if the register is a GPQ register (64-bit).
+  ASMJIT_INLINE bool isGpq() const noexcept { return hasSignature(signatureOf(kRegGpq)); }
+  //! Get if the register is an FPU register (80-bit).
+  ASMJIT_INLINE bool isFp() const noexcept { return hasSignature(signatureOf(kRegFp)); }
+  //! Get if the register is an MMX register (64-bit).
+  ASMJIT_INLINE bool isMm() const noexcept { return hasSignature(signatureOf(kRegMm)); }
+  //! Get if the register is a K register (64-bit).
+  ASMJIT_INLINE bool isK() const noexcept { return hasSignature(signatureOf(kRegK)); }
+  //! Get if the register is an XMM register (128-bit).
+  ASMJIT_INLINE bool isXmm() const noexcept { return hasSignature(signatureOf(kRegXmm)); }
+  //! Get if the register is a YMM register (256-bit).
+  ASMJIT_INLINE bool isYmm() const noexcept { return hasSignature(signatureOf(kRegYmm)); }
+  //! Get if the register is a ZMM register (512-bit).
+  ASMJIT_INLINE bool isZmm() const noexcept { return hasSignature(signatureOf(kRegZmm)); }
+  //! Get if the register is a bound register.
+  ASMJIT_INLINE bool isBnd() const noexcept { return hasSignature(signatureOf(kRegBnd)); }
+  //! Get if the register is a control register.
+  ASMJIT_INLINE bool isCr() const noexcept { return hasSignature(signatureOf(kRegCr)); }
+  //! Get if the register is a debug register.
+  ASMJIT_INLINE bool isDr() const noexcept { return hasSignature(signatureOf(kRegDr)); }
+
+  template<uint32_t Type>
+  ASMJIT_INLINE void setX86RegT(uint32_t rId) noexcept {
+    setSignature(X86RegTraits<Type>::kSignature);
+    setId(rId);
+  }
+
+  ASMJIT_INLINE void setTypeAndId(uint32_t rType, uint32_t rId) noexcept {
+    ASMJIT_ASSERT(rType < kRegCount);
+    setSignature(signatureOf(rType));
+    setId(rId);
+  }
+
+  static ASMJIT_INLINE uint32_t kindOf(uint32_t rType) noexcept;
+  template<uint32_t Type>
+  static ASMJIT_INLINE uint32_t kindOfT() noexcept { return X86RegTraits<Type>::kKind; }
+
+  static ASMJIT_INLINE uint32_t signatureOf(uint32_t rType) noexcept;
+  template<uint32_t Type>
+  static ASMJIT_INLINE uint32_t signatureOfT() noexcept { return X86RegTraits<Type>::kSignature; }
+
+  static ASMJIT_INLINE uint32_t signatureOfVecByType(uint32_t typeId) noexcept {
+    return typeId <= TypeId::_kVec128End ? signatureOfT<kRegXmm>() :
+           typeId <= TypeId::_kVec256End ? signatureOfT<kRegYmm>() : signatureOfT<kRegZmm>() ;
+  }
+
+  static ASMJIT_INLINE uint32_t signatureOfVecBySize(uint32_t size) noexcept {
+    return size <= 16 ? signatureOfT<kRegXmm>() :
+           size <= 32 ? signatureOfT<kRegYmm>() : signatureOfT<kRegZmm>() ;
   }
-
-  // --------------------------------------------------------------------------
-  // [Size]
-  // --------------------------------------------------------------------------
-
-  //! Set memory operand size.
-  ASMJIT_INLINE X86Mem& setSize(uint32_t size) {
-    _vmem.size = static_cast<uint8_t>(size);
-    return *this;
+
+  //! Get if the `op` operand is either a low or high 8-bit GPB register.
+  static ASMJIT_INLINE bool isGpb(const Operand_& op) noexcept {
+    // Check operand type, register kind, and size. Not interested in register type.
+    const uint32_t kSgn = (Operand::kOpReg << kSignatureOpShift  ) |
+                          (1               << kSignatureSizeShift) ;
+    return (op.getSignature() & (kSignatureOpMask | kSignatureSizeMask)) == kSgn;
   }
 
-  // --------------------------------------------------------------------------
-  // [Base]
-  // --------------------------------------------------------------------------
-
-  //! Get whether the memory operand has base register.
-  ASMJIT_INLINE bool hasBase() const {
-    return _vmem.base != kInvalidValue;
-  }
+  static ASMJIT_INLINE bool isRip(const Operand_& op) noexcept { return op.as<X86Reg>().isRip(); }
+  static ASMJIT_INLINE bool isSeg(const Operand_& op) noexcept { return op.as<X86Reg>().isSeg(); }
+  static ASMJIT_INLINE bool isGpbLo(const Operand_& op) noexcept { return op.as<X86Reg>().isGpbLo(); }
+  static ASMJIT_INLINE bool isGpbHi(const Operand_& op) noexcept { return op.as<X86Reg>().isGpbHi(); }
+  static ASMJIT_INLINE bool isGpw(const Operand_& op) noexcept { return op.as<X86Reg>().isGpw(); }
+  static ASMJIT_INLINE bool isGpd(const Operand_& op) noexcept { return op.as<X86Reg>().isGpd(); }
+  static ASMJIT_INLINE bool isGpq(const Operand_& op) noexcept { return op.as<X86Reg>().isGpq(); }
+  static ASMJIT_INLINE bool isFp(const Operand_& op) noexcept { return op.as<X86Reg>().isFp(); }
+  static ASMJIT_INLINE bool isMm(const Operand_& op) noexcept { return op.as<X86Reg>().isMm(); }
+  static ASMJIT_INLINE bool isK(const Operand_& op) noexcept { return op.as<X86Reg>().isK(); }
+  static ASMJIT_INLINE bool isXmm(const Operand_& op) noexcept { return op.as<X86Reg>().isXmm(); }
+  static ASMJIT_INLINE bool isYmm(const Operand_& op) noexcept { return op.as<X86Reg>().isYmm(); }
+  static ASMJIT_INLINE bool isZmm(const Operand_& op) noexcept { return op.as<X86Reg>().isZmm(); }
+  static ASMJIT_INLINE bool isBnd(const Operand_& op) noexcept { return op.as<X86Reg>().isBnd(); }
+  static ASMJIT_INLINE bool isCr(const Operand_& op) noexcept { return op.as<X86Reg>().isCr(); }
+  static ASMJIT_INLINE bool isDr(const Operand_& op) noexcept { return op.as<X86Reg>().isDr(); }
 
-  //! Get memory operand base register code, variable id, or `kInvalidValue`.
-  ASMJIT_INLINE uint32_t getBase() const {
-    return _vmem.base;
-  }
+  static ASMJIT_INLINE bool isGpb(const Operand_& op, uint32_t rId) noexcept { return isGpb(op) & (op.getId() == rId); }
 
-  //! Set memory operand base register code, variable id, or `kInvalidValue`.
-  ASMJIT_INLINE X86Mem& setBase(uint32_t base) {
-    _vmem.base = base;
-    return *this;
-  }
+  static ASMJIT_INLINE bool isRip(const Operand_& op, uint32_t rId) noexcept { return isRip(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isSeg(const Operand_& op, uint32_t rId) noexcept { return isSeg(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isGpbLo(const Operand_& op, uint32_t rId) noexcept { return isGpbLo(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isGpbHi(const Operand_& op, uint32_t rId) noexcept { return isGpbHi(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isGpw(const Operand_& op, uint32_t rId) noexcept { return isGpw(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isGpd(const Operand_& op, uint32_t rId) noexcept { return isGpd(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isGpq(const Operand_& op, uint32_t rId) noexcept { return isGpq(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isFp(const Operand_& op, uint32_t rId) noexcept { return isFp(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isMm(const Operand_& op, uint32_t rId) noexcept { return isMm(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isK(const Operand_& op, uint32_t rId) noexcept { return isK(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isXmm(const Operand_& op, uint32_t rId) noexcept { return isXmm(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isYmm(const Operand_& op, uint32_t rId) noexcept { return isYmm(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isZmm(const Operand_& op, uint32_t rId) noexcept { return isZmm(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isBnd(const Operand_& op, uint32_t rId) noexcept { return isBnd(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isCr(const Operand_& op, uint32_t rId) noexcept { return isCr(op) & (op.getId() == rId); }
+  static ASMJIT_INLINE bool isDr(const Operand_& op, uint32_t rId) noexcept { return isDr(op) & (op.getId() == rId); }
 
   // --------------------------------------------------------------------------
-  // [Index]
+  // [Memory Cast]
   // --------------------------------------------------------------------------
 
-  //! Get whether the memory operand has index.
-  ASMJIT_INLINE bool hasIndex() const {
-    return _vmem.index != kInvalidValue;
+  // DEPRECATED in next-wip.
+  ASMJIT_INLINE X86Mem m(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, getSize(), Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Get memory operand index register code, variable id, or `kInvalidValue`.
-  ASMJIT_INLINE uint32_t getIndex() const {
-    return _vmem.index;
+  //! \overload
+  ASMJIT_INLINE X86Mem m(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, getSize(), Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory operand index register code, variable id, or `kInvalidValue`.
-  ASMJIT_INLINE X86Mem& setIndex(uint32_t index) {
-    _vmem.index = index;
-    return *this;
+  //! Cast this variable to 8-bit memory operand.
+  ASMJIT_INLINE X86Mem m8(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, 1, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86GpReg& index) {
-    _vmem.index = index.getRegIndex();
-    return _setVSib(kX86MemVSibGpz);
+  //! \overload
+  ASMJIT_INLINE X86Mem m8(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, 1, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86GpReg& index, uint32_t shift) {
-    _vmem.index = index.getRegIndex();
-    return _setVSib(kX86MemVSibGpz).setShift(shift);
+  //! Cast this variable to 16-bit memory operand.
+  ASMJIT_INLINE X86Mem m16(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, 2, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86XmmReg& index) {
-    _vmem.index = index.getRegIndex();
-    return _setVSib(kX86MemVSibXmm);
+  //! \overload
+  ASMJIT_INLINE X86Mem m16(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, 2, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86XmmReg& index, uint32_t shift) {
-    _vmem.index = index.getRegIndex();
-    return _setVSib(kX86MemVSibXmm).setShift(shift);
+  //! Cast this variable to 32-bit memory operand.
+  ASMJIT_INLINE X86Mem m32(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, 4, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86YmmReg& index) {
-    _vmem.index = index.getRegIndex();
-    return _setVSib(kX86MemVSibYmm);
+  //! \overload
+  ASMJIT_INLINE X86Mem m32(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, 4, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86YmmReg& index, uint32_t shift) {
-    _vmem.index = index.getRegIndex();
-    return _setVSib(kX86MemVSibYmm).setShift(shift);
+  //! Cast this variable to 64-bit memory operand.
+  ASMJIT_INLINE X86Mem m64(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, 8, Mem::kSignatureMemRegHomeFlag);
   }
 
-#if !defined(ASMJIT_DISABLE_COMPILER)
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86GpVar& index) {
-    _vmem.index = ASMJIT_OP_ID(index);
-    return _setVSib(kX86MemVSibGpz);
+  //! \overload
+  ASMJIT_INLINE X86Mem m64(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, 8, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86GpVar& index, uint32_t shift) {
-    _vmem.index = ASMJIT_OP_ID(index);
-    return _setVSib(kX86MemVSibGpz).setShift(shift);
+  //! Cast this variable to 80-bit memory operand (long double).
+  ASMJIT_INLINE X86Mem m80(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, 10, Mem::kSignatureMemRegHomeFlag);
   }
 
-
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86XmmVar& index) {
-    _vmem.index = ASMJIT_OP_ID(index);
-    return _setVSib(kX86MemVSibXmm);
-  }
-
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86XmmVar& index, uint32_t shift) {
-    _vmem.index = ASMJIT_OP_ID(index);
-    return _setVSib(kX86MemVSibXmm).setShift(shift);
+  //! \overload
+  ASMJIT_INLINE X86Mem m80(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, 10, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86YmmVar& index) {
-    _vmem.index = ASMJIT_OP_ID(index);
-    return _setVSib(kX86MemVSibYmm);
+  //! Cast this variable to 128-bit memory operand.
+  ASMJIT_INLINE X86Mem m128(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, 16, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Set memory index.
-  ASMJIT_INLINE X86Mem& setIndex(const X86YmmVar& index, uint32_t shift) {
-    _vmem.index = ASMJIT_OP_ID(index);
-    return _setVSib(kX86MemVSibYmm).setShift(shift);
+  //! \overload
+  ASMJIT_INLINE X86Mem m128(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, 16, Mem::kSignatureMemRegHomeFlag);
   }
-#endif // !ASMJIT_DISABLE_COMPILER
 
-  //! Reset memory index.
-  ASMJIT_INLINE X86Mem& resetIndex() {
-    _vmem.index = kInvalidValue;
-    return _setVSib(kX86MemVSibGpz);
+  //! Cast this variable to 256-bit memory operand.
+  ASMJIT_INLINE X86Mem m256(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, 32, Mem::kSignatureMemRegHomeFlag);
   }
 
-  // --------------------------------------------------------------------------
-  // [Misc]
-  // --------------------------------------------------------------------------
-
-  //! Get whether the memory operand has base and index register.
-  ASMJIT_INLINE bool hasBaseOrIndex() const {
-    return _vmem.base != kInvalidValue || _vmem.index != kInvalidValue;
+  //! \overload
+  ASMJIT_INLINE X86Mem m256(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, 32, Mem::kSignatureMemRegHomeFlag);
   }
 
-  //! Get whether the memory operand has base and index register.
-  ASMJIT_INLINE bool hasBaseAndIndex() const {
-    return _vmem.base != kInvalidValue && _vmem.index != kInvalidValue;
+  //! Cast this variable to 256-bit memory operand.
+  ASMJIT_INLINE X86Mem m512(int32_t disp = 0) const noexcept {
+    return X86Mem(*this, disp, 64, Mem::kSignatureMemRegHomeFlag);
   }
 
-  // --------------------------------------------------------------------------
-  // [Shift]
-  // --------------------------------------------------------------------------
+  //! \overload
+  ASMJIT_INLINE X86Mem m512(const X86Reg& index, uint32_t shift = 0, int32_t disp = 0) const noexcept {
+    return X86Mem(*this, index, shift, disp, 64, Mem::kSignatureMemRegHomeFlag);
+  };
+};
 
-  //! Get whether the memory operand has shift used.
-  ASMJIT_INLINE bool hasShift() const {
-    return (_vmem.flags & kX86MemShiftMask) != 0;
-  }
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Gp  , X86Reg::kRegGpbLo, X86Reg::kKindGp , 1 , 16, TypeId::kI8    );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Gp  , X86Reg::kRegGpbHi, X86Reg::kKindGp , 1 , 4 , TypeId::kI8    );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Gp  , X86Reg::kRegGpw  , X86Reg::kKindGp , 2 , 16, TypeId::kI16   );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Gp  , X86Reg::kRegGpd  , X86Reg::kKindGp , 4 , 16, TypeId::kI32   );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Gp  , X86Reg::kRegGpq  , X86Reg::kKindGp , 8 , 16, TypeId::kI64   );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Xmm , X86Reg::kRegXmm  , X86Reg::kKindVec, 16, 32, TypeId::kI32x4 );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Ymm , X86Reg::kRegYmm  , X86Reg::kKindVec, 32, 32, TypeId::kI32x8 );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Zmm , X86Reg::kRegZmm  , X86Reg::kKindVec, 64, 32, TypeId::kI32x16);
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Rip , X86Reg::kRegRip  , X86Reg::kKindRip, 0 , 1 , TypeId::kVoid  );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Seg , X86Reg::kRegSeg  , X86Reg::kKindSeg, 2 , 7 , TypeId::kVoid  );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Fp  , X86Reg::kRegFp   , X86Reg::kKindFp , 10, 8 , TypeId::kF80   );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Mm  , X86Reg::kRegMm   , X86Reg::kKindMm , 8 , 8 , TypeId::kMmx64 );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86KReg, X86Reg::kRegK    , X86Reg::kKindK  , 0 , 8 , TypeId::kVoid  );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86Bnd , X86Reg::kRegBnd  , X86Reg::kKindBnd, 16, 4 , TypeId::kVoid  );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86CReg, X86Reg::kRegCr   , X86Reg::kKindCr , 0 , 16, TypeId::kVoid  );
+ASMJIT_DEFINE_REG_TRAITS(X86RegTraits, X86DReg, X86Reg::kRegDr   , X86Reg::kKindDr , 0 , 16, TypeId::kVoid  );
+
+//! General purpose register (X86/X64).
+class X86Gp : public X86Reg {
+public:
+  ASMJIT_DEFINE_ABSTRACT_REG(X86Gp, X86Reg)
+
+  //! X86/X64 physical id.
+  //!
+  //! NOTE: Register indexes have been reduced to only support general purpose
+  //! registers. There is no need to have enumerations with number suffix that
+  //! expands to the exactly same value as the suffix value itself.
+  ASMJIT_ENUM(Id) {
+    kIdAx  = 0,  //!< Physical id of AL|AH|AX|EAX|RAX registers.
+    kIdCx  = 1,  //!< Physical id of CL|CH|CX|ECX|RCX registers.
+    kIdDx  = 2,  //!< Physical id of DL|DH|DX|EDX|RDX registers.
+    kIdBx  = 3,  //!< Physical id of BL|BH|BX|EBX|RBX registers.
+    kIdSp  = 4,  //!< Physical id of SPL|SP|ESP|RSP registers.
+    kIdBp  = 5,  //!< Physical id of BPL|BP|EBP|RBP registers.
+    kIdSi  = 6,  //!< Physical id of SIL|SI|ESI|RSI registers.
+    kIdDi  = 7,  //!< Physical id of DIL|DI|EDI|RDI registers.
+    kIdR8  = 8,  //!< Physical id of R8B|R8W|R8D|R8 registers (64-bit only).
+    kIdR9  = 9,  //!< Physical id of R9B|R9W|R9D|R9 registers (64-bit only).
+    kIdR10 = 10, //!< Physical id of R10B|R10W|R10D|R10 registers (64-bit only).
+    kIdR11 = 11, //!< Physical id of R11B|R11W|R11D|R11 registers (64-bit only).
+    kIdR12 = 12, //!< Physical id of R12B|R12W|R12D|R12 registers (64-bit only).
+    kIdR13 = 13, //!< Physical id of R13B|R13W|R13D|R13 registers (64-bit only).
+    kIdR14 = 14, //!< Physical id of R14B|R14W|R14D|R14 registers (64-bit only).
+    kIdR15 = 15  //!< Physical id of R15B|R15W|R15D|R15 registers (64-bit only).
+  };
 
-  //! Get memory operand index scale (0, 1, 2 or 3).
-  ASMJIT_INLINE uint32_t getShift() const {
-    return _vmem.flags >> kX86MemShiftIndex;
-  }
+  //! Cast this register to 8-bit (LO) part.
+  ASMJIT_INLINE X86GpbLo r8() const noexcept;
+  //! Cast this register to 8-bit (LO) part.
+  ASMJIT_INLINE X86GpbLo r8Lo() const noexcept;
+  //! Cast this register to 8-bit (HI) part.
+  ASMJIT_INLINE X86GpbHi r8Hi() const noexcept;
+  //! Cast this register to 16-bit.
+  ASMJIT_INLINE X86Gpw r16() const noexcept;
+  //! Cast this register to 32-bit.
+  ASMJIT_INLINE X86Gpd r32() const noexcept;
+  //! Cast this register to 64-bit.
+  ASMJIT_INLINE X86Gpq r64() const noexcept;
+};
 
-  //! Set memory operand index scale (0, 1, 2 or 3).
-  ASMJIT_INLINE X86Mem& setShift(uint32_t shift) {
-    _packed[0].u32[0] &=~IntUtil::pack32_4x8(0x00, 0x00, 0x00, kX86MemShiftMask);
-    _packed[0].u32[0] |= IntUtil::pack32_4x8(0x00, 0x00, 0x00, shift << kX86MemShiftIndex);
-    return *this;
-  }
+//! XMM|YMM|ZMM register (X86/X64).
+class X86Vec : public X86Reg {
+  ASMJIT_DEFINE_ABSTRACT_REG(X86Vec, X86Reg)
 
-  // --------------------------------------------------------------------------
-  // [Displacement]
-  // --------------------------------------------------------------------------
+  //! Cast this register to XMM (clone).
+  ASMJIT_INLINE X86Xmm xmm() const noexcept;
+  //! Cast this register to YMM.
+  ASMJIT_INLINE X86Ymm ymm() const noexcept;
+  //! Cast this register to ZMM.
+  ASMJIT_INLINE X86Zmm zmm() const noexcept;
+};
 
-  //! Get memory operand relative displacement.
-  ASMJIT_INLINE int32_t getDisplacement() const {
-    return _vmem.displacement;
-  }
+//! Segment register (X86/X64).
+class X86Seg : public X86Reg {
+  ASMJIT_DEFINE_FINAL_REG(X86Seg, X86Reg, X86RegTraits<kRegSeg>)
 
-  //! Set memory operand relative displacement.
-  ASMJIT_INLINE X86Mem& setDisplacement(int32_t disp) {
-    _vmem.displacement = disp;
-    return *this;
-  }
+  //! X86/X64 segment id.
+  ASMJIT_ENUM(Id) {
+    kIdNone = 0, //!< No segment (default).
+    kIdEs   = 1, //!< ES segment.
+    kIdCs   = 2, //!< CS segment.
+    kIdSs   = 3, //!< SS segment.
+    kIdDs   = 4, //!< DS segment.
+    kIdFs   = 5, //!< FS segment.
+    kIdGs   = 6, //!< GS segment.
 
-  //! Reset memory operand relative displacement.
-  ASMJIT_INLINE X86Mem& resetDisplacement(int32_t disp) {
-    _vmem.displacement = 0;
-    return *this;
-  }
+    //! Count of X86 segment registers supported by AsmJit.
+    //!
+    //! NOTE: X86 architecture has 6 segment registers - ES, CS, SS, DS, FS, GS.
+    //! X64 architecture lowers them down to just FS and GS. AsmJit supports 7
+    //! segment registers - all addressable in both X86 and X64 modes and one
+    //! extra called `X86Seg::kIdNone`, which is AsmJit specific and means that
+    //! there is no segment register specified.
+    kIdCount = 7
+  };
+};
 
-  //! Adjust memory operand relative displacement by `disp`.
-  ASMJIT_INLINE X86Mem& adjust(int32_t disp) {
-    _vmem.displacement += disp;
-    return *this;
-  }
+//! GPB (low or high) register (X86/X64).
+class X86Gpb : public X86Gp { ASMJIT_DEFINE_ABSTRACT_REG(X86Gpb, X86Gp) };
+//! GPB low register (X86/X64).
+class X86GpbLo : public X86Gpb { ASMJIT_DEFINE_FINAL_REG(X86GpbLo, X86Gpb, X86RegTraits<kRegGpbLo>) };
+//! GPB high register (X86/X64).
+class X86GpbHi : public X86Gpb { ASMJIT_DEFINE_FINAL_REG(X86GpbHi, X86Gpb, X86RegTraits<kRegGpbHi>) };
+//! GPW register (X86/X64).
+class X86Gpw : public X86Gp { ASMJIT_DEFINE_FINAL_REG(X86Gpw, X86Gp, X86RegTraits<kRegGpw>) };
+//! GPD register (X86/X64).
+class X86Gpd : public X86Gp { ASMJIT_DEFINE_FINAL_REG(X86Gpd, X86Gp, X86RegTraits<kRegGpd>) };
+//! GPQ register (X64).
+class X86Gpq : public X86Gp { ASMJIT_DEFINE_FINAL_REG(X86Gpq, X86Gp, X86RegTraits<kRegGpq>) };
+
+//! RIP register (X86/X64).
+class X86Rip : public X86Reg { ASMJIT_DEFINE_FINAL_REG(X86Rip, X86Reg, X86RegTraits<kRegRip>) };
+//! 80-bit FPU register (X86/X64).
+class X86Fp : public X86Reg { ASMJIT_DEFINE_FINAL_REG(X86Fp, X86Reg, X86RegTraits<kRegFp>) };
+//! 64-bit MMX register (MMX+).
+class X86Mm : public X86Reg { ASMJIT_DEFINE_FINAL_REG(X86Mm, X86Reg, X86RegTraits<kRegMm>) };
+//! 64-bit K register (AVX512+).
+class X86KReg : public X86Reg { ASMJIT_DEFINE_FINAL_REG(X86KReg, X86Reg, X86RegTraits<kRegK>) };
+//! 128-bit XMM register (SSE+).
+class X86Xmm : public X86Vec { ASMJIT_DEFINE_FINAL_REG(X86Xmm, X86Vec, X86RegTraits<kRegXmm>) };
+//! 256-bit YMM register (AVX+).
+class X86Ymm : public X86Vec { ASMJIT_DEFINE_FINAL_REG(X86Ymm, X86Vec, X86RegTraits<kRegYmm>) };
+//! 512-bit ZMM register (AVX512+).
+class X86Zmm : public X86Vec { ASMJIT_DEFINE_FINAL_REG(X86Zmm, X86Vec, X86RegTraits<kRegZmm>) };
+//! 128-bit BND register (BND+).
+class X86Bnd : public X86Reg { ASMJIT_DEFINE_FINAL_REG(X86Bnd, X86Reg, X86RegTraits<kRegBnd>) };
+//! 32-bit or 64-bit control register (X86/X64).
+class X86CReg : public X86Reg { ASMJIT_DEFINE_FINAL_REG(X86CReg, X86Reg, X86RegTraits<kRegCr>) };
+//! 32-bit or 64-bit debug register (X86/X64).
+class X86DReg : public X86Reg { ASMJIT_DEFINE_FINAL_REG(X86DReg, X86Reg, X86RegTraits<kRegDr>) };
+
+ASMJIT_INLINE X86GpbLo X86Gp::r8() const noexcept { return X86GpbLo(getId()); }
+ASMJIT_INLINE X86GpbLo X86Gp::r8Lo() const noexcept { return X86GpbLo(getId()); }
+ASMJIT_INLINE X86GpbHi X86Gp::r8Hi() const noexcept { return X86GpbHi(getId()); }
+ASMJIT_INLINE X86Gpw X86Gp::r16() const noexcept { return X86Gpw(getId()); }
+ASMJIT_INLINE X86Gpd X86Gp::r32() const noexcept { return X86Gpd(getId()); }
+ASMJIT_INLINE X86Gpq X86Gp::r64() const noexcept { return X86Gpq(getId()); }
+ASMJIT_INLINE X86Xmm X86Vec::xmm() const noexcept { return X86Xmm(*this, getId()); }
+ASMJIT_INLINE X86Ymm X86Vec::ymm() const noexcept { return X86Ymm(*this, getId()); }
+ASMJIT_INLINE X86Zmm X86Vec::zmm() const noexcept { return X86Zmm(*this, getId()); }
+
+ASMJIT_INLINE X86Seg X86Mem::getSegment() const noexcept { return X86Seg(getSegmentId()); }
+ASMJIT_INLINE void X86Mem::setSegment(const X86Seg& seg) noexcept { setSegmentId(seg.getId()); }
+
+
+ASMJIT_DEFINE_TYPE_ID(X86Gpb, TypeId::kI8);
+ASMJIT_DEFINE_TYPE_ID(X86Gpw, TypeId::kI16);
+ASMJIT_DEFINE_TYPE_ID(X86Gpd, TypeId::kI32);
+ASMJIT_DEFINE_TYPE_ID(X86Gpq, TypeId::kI64);
+ASMJIT_DEFINE_TYPE_ID(X86Mm , TypeId::kMmx64);
+ASMJIT_DEFINE_TYPE_ID(X86Xmm, TypeId::kI32x4);
+ASMJIT_DEFINE_TYPE_ID(X86Ymm, TypeId::kI32x8);
+ASMJIT_DEFINE_TYPE_ID(X86Zmm, TypeId::kI32x16);
 
-  //! Get new memory operand adjusted by `disp`.
-  ASMJIT_INLINE X86Mem adjusted(int32_t disp) const {
-    X86Mem result(*this);
-    result.adjust(disp);
-    return result;
-  }
+// ============================================================================
+// [asmjit::X86OpData]
+// ============================================================================
 
+struct X86OpData {
   // --------------------------------------------------------------------------
-  // [Operator Overload]
+  // [Signatures]
   // --------------------------------------------------------------------------
 
-  ASMJIT_INLINE X86Mem& operator=(const X86Mem& other) {
-    _copy(other);
-    return *this;
-  }
-
-  ASMJIT_INLINE bool operator==(const X86Mem& other) const {
-    return (_packed[0] == other._packed[0]) & (_packed[1] == other._packed[1]) ;
-  }
-
-  ASMJIT_INLINE bool operator!=(const X86Mem& other) const {
-    return !(*this == other);
-  }
+  //! Information about all architecture registers.
+  ArchRegs archRegs;
 
   // --------------------------------------------------------------------------
-  // [Static]
+  // [Operands]
   // --------------------------------------------------------------------------
 
-  static ASMJIT_INLINE uint32_t _getGpdFlags(const Operand& base) {
-    return (base._vreg.size & 0x4) << (kX86MemGpdIndex - 2);
-  }
+  // Prevent calling constructors of these registers when exporting.
+#if defined(ASMJIT_EXPORTS_X86_OPERAND)
+# define ASMJIT_X86_REG_DATA(REG) Operand_
+#else
+# define ASMJIT_X86_REG_DATA(REG) REG
+#endif
+  ASMJIT_X86_REG_DATA(X86Rip ) rip[1];
+  ASMJIT_X86_REG_DATA(X86Seg ) seg[7];
+  ASMJIT_X86_REG_DATA(X86Gp  ) gpbLo[16];
+  ASMJIT_X86_REG_DATA(X86Gp  ) gpbHi[4];
+  ASMJIT_X86_REG_DATA(X86Gp  ) gpw[16];
+  ASMJIT_X86_REG_DATA(X86Gp  ) gpd[16];
+  ASMJIT_X86_REG_DATA(X86Gp  ) gpq[16];
+  ASMJIT_X86_REG_DATA(X86Fp  ) fp[8];
+  ASMJIT_X86_REG_DATA(X86Mm  ) mm[8];
+  ASMJIT_X86_REG_DATA(X86KReg) k[8];
+  ASMJIT_X86_REG_DATA(X86Xmm ) xmm[32];
+  ASMJIT_X86_REG_DATA(X86Ymm ) ymm[32];
+  ASMJIT_X86_REG_DATA(X86Zmm ) zmm[32];
+  ASMJIT_X86_REG_DATA(X86Bnd ) bnd[4];
+  ASMJIT_X86_REG_DATA(X86CReg) cr[16];
+  ASMJIT_X86_REG_DATA(X86DReg) dr[16];
+#undef ASMJIT_X86_REG_DATA
 };
-#endif // ASMJIT_EXPORTS_X86OPERAND_REGS
-
-// ============================================================================
-// [asmjit::X86RegData]
-// ============================================================================
-
-struct X86RegData {
-  X86RipReg rip;
-  X86GpReg noGp;
-
-  X86SegReg seg[7];
-
-  X86GpReg gpbLo[16];
-  X86GpReg gpbHi[4];
-
-  X86GpReg gpw[16];
-  X86GpReg gpd[16];
-  X86GpReg gpq[16];
-
-  X86FpReg fp[8];
-  X86MmReg mm[8];
-  X86KReg k[8];
+ASMJIT_VARAPI const X86OpData x86OpData;
 
-  X86XmmReg xmm[32];
-  X86YmmReg ymm[32];
-  X86ZmmReg zmm[32];
-};
+// ... X86Reg methods that require `x86OpData`.
+ASMJIT_INLINE uint32_t X86Reg::signatureOf(uint32_t rType) noexcept {
+  ASMJIT_ASSERT(rType <= Reg::kRegMax);
+  return x86OpData.archRegs.regInfo[rType].getSignature();
+}
 
-ASMJIT_VAR const X86RegData x86RegData;
+ASMJIT_INLINE uint32_t X86Reg::kindOf(uint32_t rType) noexcept {
+  ASMJIT_ASSERT(rType <= Reg::kRegMax);
+  return x86OpData.archRegs.regInfo[rType].getKind();
+}
 
 // ============================================================================
 // [asmjit::x86]
@@ -1611,408 +682,414 @@ namespace x86 {
 // [asmjit::x86 - Reg]
 // ============================================================================
 
-#define ASMJIT_DEF_REG(_Type_, _Name_, _Field_) \
-  static const _Type_& _Name_ = x86RegData._Field_
-
-ASMJIT_DEF_REG(X86RipReg, rip, rip);        //!< RIP register.
-ASMJIT_DEF_REG(X86GpReg , noGpReg, noGp);   //!< No GP register (for `X86Mem` operand.).
-
-ASMJIT_DEF_REG(X86SegReg, es   , seg[1]);   //!< Cs segment register.
-ASMJIT_DEF_REG(X86SegReg, cs   , seg[2]);   //!< Ss segment register.
-ASMJIT_DEF_REG(X86SegReg, ss   , seg[3]);   //!< Ds segment register.
-ASMJIT_DEF_REG(X86SegReg, ds   , seg[4]);   //!< Es segment register.
-ASMJIT_DEF_REG(X86SegReg, fs   , seg[5]);   //!< Fs segment register.
-ASMJIT_DEF_REG(X86SegReg, gs   , seg[6]);   //!< Gs segment register.
-
-ASMJIT_DEF_REG(X86GpReg , al   , gpbLo[0]); //!< 8-bit Gpb-lo register.
-ASMJIT_DEF_REG(X86GpReg , cl   , gpbLo[1]); //!< 8-bit Gpb-lo register.
-ASMJIT_DEF_REG(X86GpReg , dl   , gpbLo[2]); //!< 8-bit Gpb-lo register.
-ASMJIT_DEF_REG(X86GpReg , bl   , gpbLo[3]); //!< 8-bit Gpb-lo register.
-ASMJIT_DEF_REG(X86GpReg , spl  , gpbLo[4]); //!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , bpl  , gpbLo[5]); //!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , sil  , gpbLo[6]); //!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , dil  , gpbLo[7]); //!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , r8b  , gpbLo[8]); //!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , r9b  , gpbLo[9]); //!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , r10b , gpbLo[10]);//!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , r11b , gpbLo[11]);//!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , r12b , gpbLo[12]);//!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , r13b , gpbLo[13]);//!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , r14b , gpbLo[14]);//!< 8-bit Gpb-lo register (X64).
-ASMJIT_DEF_REG(X86GpReg , r15b , gpbLo[15]);//!< 8-bit Gpb-lo register (X64).
-
-ASMJIT_DEF_REG(X86GpReg , ah   , gpbHi[0]); //!< 8-bit Gpb-hi register.
-ASMJIT_DEF_REG(X86GpReg , ch   , gpbHi[1]); //!< 8-bit Gpb-hi register.
-ASMJIT_DEF_REG(X86GpReg , dh   , gpbHi[2]); //!< 8-bit Gpb-hi register.
-ASMJIT_DEF_REG(X86GpReg , bh   , gpbHi[3]); //!< 8-bit Gpb-hi register.
-
-ASMJIT_DEF_REG(X86GpReg , ax   , gpw[0]);   //!< 16-bit Gpw register.
-ASMJIT_DEF_REG(X86GpReg , cx   , gpw[1]);   //!< 16-bit Gpw register.
-ASMJIT_DEF_REG(X86GpReg , dx   , gpw[2]);   //!< 16-bit Gpw register.
-ASMJIT_DEF_REG(X86GpReg , bx   , gpw[3]);   //!< 16-bit Gpw register.
-ASMJIT_DEF_REG(X86GpReg , sp   , gpw[4]);   //!< 16-bit Gpw register.
-ASMJIT_DEF_REG(X86GpReg , bp   , gpw[5]);   //!< 16-bit Gpw register.
-ASMJIT_DEF_REG(X86GpReg , si   , gpw[6]);   //!< 16-bit Gpw register.
-ASMJIT_DEF_REG(X86GpReg , di   , gpw[7]);   //!< 16-bit Gpw register.
-ASMJIT_DEF_REG(X86GpReg , r8w  , gpw[8]);   //!< 16-bit Gpw register (X64).
-ASMJIT_DEF_REG(X86GpReg , r9w  , gpw[9]);   //!< 16-bit Gpw register (X64).
-ASMJIT_DEF_REG(X86GpReg , r10w , gpw[10]);  //!< 16-bit Gpw register (X64).
-ASMJIT_DEF_REG(X86GpReg , r11w , gpw[11]);  //!< 16-bit Gpw register (X64).
-ASMJIT_DEF_REG(X86GpReg , r12w , gpw[12]);  //!< 16-bit Gpw register (X64).
-ASMJIT_DEF_REG(X86GpReg , r13w , gpw[13]);  //!< 16-bit Gpw register (X64).
-ASMJIT_DEF_REG(X86GpReg , r14w , gpw[14]);  //!< 16-bit Gpw register (X64).
-ASMJIT_DEF_REG(X86GpReg , r15w , gpw[15]);  //!< 16-bit Gpw register (X64).
-
-ASMJIT_DEF_REG(X86GpReg , eax  , gpd[0]);   //!< 32-bit Gpd register.
-ASMJIT_DEF_REG(X86GpReg , ecx  , gpd[1]);   //!< 32-bit Gpd register.
-ASMJIT_DEF_REG(X86GpReg , edx  , gpd[2]);   //!< 32-bit Gpd register.
-ASMJIT_DEF_REG(X86GpReg , ebx  , gpd[3]);   //!< 32-bit Gpd register.
-ASMJIT_DEF_REG(X86GpReg , esp  , gpd[4]);   //!< 32-bit Gpd register.
-ASMJIT_DEF_REG(X86GpReg , ebp  , gpd[5]);   //!< 32-bit Gpd register.
-ASMJIT_DEF_REG(X86GpReg , esi  , gpd[6]);   //!< 32-bit Gpd register.
-ASMJIT_DEF_REG(X86GpReg , edi  , gpd[7]);   //!< 32-bit Gpd register.
-ASMJIT_DEF_REG(X86GpReg , r8d  , gpd[8]);   //!< 32-bit Gpd register (X64).
-ASMJIT_DEF_REG(X86GpReg , r9d  , gpd[9]);   //!< 32-bit Gpd register (X64).
-ASMJIT_DEF_REG(X86GpReg , r10d , gpd[10]);  //!< 32-bit Gpd register (X64).
-ASMJIT_DEF_REG(X86GpReg , r11d , gpd[11]);  //!< 32-bit Gpd register (X64).
-ASMJIT_DEF_REG(X86GpReg , r12d , gpd[12]);  //!< 32-bit Gpd register (X64).
-ASMJIT_DEF_REG(X86GpReg , r13d , gpd[13]);  //!< 32-bit Gpd register (X64).
-ASMJIT_DEF_REG(X86GpReg , r14d , gpd[14]);  //!< 32-bit Gpd register (X64).
-ASMJIT_DEF_REG(X86GpReg , r15d , gpd[15]);  //!< 32-bit Gpd register (X64).
-
-ASMJIT_DEF_REG(X86GpReg , rax  , gpq[0]);   //!< 64-bit Gpq register (X64).
-ASMJIT_DEF_REG(X86GpReg , rcx  , gpq[1]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , rdx  , gpq[2]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , rbx  , gpq[3]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , rsp  , gpq[4]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , rbp  , gpq[5]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , rsi  , gpq[6]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , rdi  , gpq[7]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , r8   , gpq[8]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , r9   , gpq[9]);   //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , r10  , gpq[10]);  //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , r11  , gpq[11]);  //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , r12  , gpq[12]);  //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , r13  , gpq[13]);  //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , r14  , gpq[14]);  //!< 64-bit Gpq register (X64)
-ASMJIT_DEF_REG(X86GpReg , r15  , gpq[15]);  //!< 64-bit Gpq register (X64)
-
-ASMJIT_DEF_REG(X86FpReg , fp0  , fp[0]);    //!< 80-bit Fp register.
-ASMJIT_DEF_REG(X86FpReg , fp1  , fp[1]);    //!< 80-bit Fp register.
-ASMJIT_DEF_REG(X86FpReg , fp2  , fp[2]);    //!< 80-bit Fp register.
-ASMJIT_DEF_REG(X86FpReg , fp3  , fp[3]);    //!< 80-bit Fp register.
-ASMJIT_DEF_REG(X86FpReg , fp4  , fp[4]);    //!< 80-bit Fp register.
-ASMJIT_DEF_REG(X86FpReg , fp5  , fp[5]);    //!< 80-bit Fp register.
-ASMJIT_DEF_REG(X86FpReg , fp6  , fp[6]);    //!< 80-bit Fp register.
-ASMJIT_DEF_REG(X86FpReg , fp7  , fp[7]);    //!< 80-bit Fp register.
-
-ASMJIT_DEF_REG(X86MmReg , mm0  , mm[0]);    //!< 64-bit Mm register.
-ASMJIT_DEF_REG(X86MmReg , mm1  , mm[1]);    //!< 64-bit Mm register.
-ASMJIT_DEF_REG(X86MmReg , mm2  , mm[2]);    //!< 64-bit Mm register.
-ASMJIT_DEF_REG(X86MmReg , mm3  , mm[3]);    //!< 64-bit Mm register.
-ASMJIT_DEF_REG(X86MmReg , mm4  , mm[4]);    //!< 64-bit Mm register.
-ASMJIT_DEF_REG(X86MmReg , mm5  , mm[5]);    //!< 64-bit Mm register.
-ASMJIT_DEF_REG(X86MmReg , mm6  , mm[6]);    //!< 64-bit Mm register.
-ASMJIT_DEF_REG(X86MmReg , mm7  , mm[7]);    //!< 64-bit Mm register.
-
-ASMJIT_DEF_REG(X86KReg  , k0   , k[0]);     //!< 64-bit K register.
-ASMJIT_DEF_REG(X86KReg  , k1   , k[1]);     //!< 64-bit K register.
-ASMJIT_DEF_REG(X86KReg  , k2   , k[2]);     //!< 64-bit K register.
-ASMJIT_DEF_REG(X86KReg  , k3   , k[3]);     //!< 64-bit K register.
-ASMJIT_DEF_REG(X86KReg  , k4   , k[4]);     //!< 64-bit K register.
-ASMJIT_DEF_REG(X86KReg  , k5   , k[5]);     //!< 64-bit K register.
-ASMJIT_DEF_REG(X86KReg  , k6   , k[6]);     //!< 64-bit K register.
-ASMJIT_DEF_REG(X86KReg  , k7   , k[7]);     //!< 64-bit K register.
-
-ASMJIT_DEF_REG(X86XmmReg, xmm0 , xmm[0]);   //!< 128-bit Xmm register.
-ASMJIT_DEF_REG(X86XmmReg, xmm1 , xmm[1]);   //!< 128-bit Xmm register.
-ASMJIT_DEF_REG(X86XmmReg, xmm2 , xmm[2]);   //!< 128-bit Xmm register.
-ASMJIT_DEF_REG(X86XmmReg, xmm3 , xmm[3]);   //!< 128-bit Xmm register.
-ASMJIT_DEF_REG(X86XmmReg, xmm4 , xmm[4]);   //!< 128-bit Xmm register.
-ASMJIT_DEF_REG(X86XmmReg, xmm5 , xmm[5]);   //!< 128-bit Xmm register.
-ASMJIT_DEF_REG(X86XmmReg, xmm6 , xmm[6]);   //!< 128-bit Xmm register.
-ASMJIT_DEF_REG(X86XmmReg, xmm7 , xmm[7]);   //!< 128-bit Xmm register.
-ASMJIT_DEF_REG(X86XmmReg, xmm8 , xmm[8]);   //!< 128-bit Xmm register (X64).
-ASMJIT_DEF_REG(X86XmmReg, xmm9 , xmm[9]);   //!< 128-bit Xmm register (X64).
-ASMJIT_DEF_REG(X86XmmReg, xmm10, xmm[10]);  //!< 128-bit Xmm register (X64).
-ASMJIT_DEF_REG(X86XmmReg, xmm11, xmm[11]);  //!< 128-bit Xmm register (X64).
-ASMJIT_DEF_REG(X86XmmReg, xmm12, xmm[12]);  //!< 128-bit Xmm register (X64).
-ASMJIT_DEF_REG(X86XmmReg, xmm13, xmm[13]);  //!< 128-bit Xmm register (X64).
-ASMJIT_DEF_REG(X86XmmReg, xmm14, xmm[14]);  //!< 128-bit Xmm register (X64).
-ASMJIT_DEF_REG(X86XmmReg, xmm15, xmm[15]);  //!< 128-bit Xmm register (X64).
-ASMJIT_DEF_REG(X86XmmReg, xmm16, xmm[16]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm17, xmm[17]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm18, xmm[18]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm19, xmm[19]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm20, xmm[20]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm21, xmm[21]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm22, xmm[22]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm23, xmm[23]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm24, xmm[24]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm25, xmm[25]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm26, xmm[26]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm27, xmm[27]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm28, xmm[28]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm29, xmm[29]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm30, xmm[30]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86XmmReg, xmm31, xmm[31]);  //!< 128-bit Xmm register (X64 & AVX512VL+).
-
-ASMJIT_DEF_REG(X86YmmReg, ymm0 , ymm[0]);   //!< 256-bit Ymm register.
-ASMJIT_DEF_REG(X86YmmReg, ymm1 , ymm[1]);   //!< 256-bit Ymm register.
-ASMJIT_DEF_REG(X86YmmReg, ymm2 , ymm[2]);   //!< 256-bit Ymm register.
-ASMJIT_DEF_REG(X86YmmReg, ymm3 , ymm[3]);   //!< 256-bit Ymm register.
-ASMJIT_DEF_REG(X86YmmReg, ymm4 , ymm[4]);   //!< 256-bit Ymm register.
-ASMJIT_DEF_REG(X86YmmReg, ymm5 , ymm[5]);   //!< 256-bit Ymm register.
-ASMJIT_DEF_REG(X86YmmReg, ymm6 , ymm[6]);   //!< 256-bit Ymm register.
-ASMJIT_DEF_REG(X86YmmReg, ymm7 , ymm[7]);   //!< 256-bit Ymm register.
-ASMJIT_DEF_REG(X86YmmReg, ymm8 , ymm[8]);   //!< 256-bit Ymm register (X64).
-ASMJIT_DEF_REG(X86YmmReg, ymm9 , ymm[9]);   //!< 256-bit Ymm register (X64).
-ASMJIT_DEF_REG(X86YmmReg, ymm10, ymm[10]);  //!< 256-bit Ymm register (X64).
-ASMJIT_DEF_REG(X86YmmReg, ymm11, ymm[11]);  //!< 256-bit Ymm register (X64).
-ASMJIT_DEF_REG(X86YmmReg, ymm12, ymm[12]);  //!< 256-bit Ymm register (X64).
-ASMJIT_DEF_REG(X86YmmReg, ymm13, ymm[13]);  //!< 256-bit Ymm register (X64).
-ASMJIT_DEF_REG(X86YmmReg, ymm14, ymm[14]);  //!< 256-bit Ymm register (X64).
-ASMJIT_DEF_REG(X86YmmReg, ymm15, ymm[15]);  //!< 256-bit Ymm register (X64).
-ASMJIT_DEF_REG(X86YmmReg, ymm16, ymm[16]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm17, ymm[17]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm18, ymm[18]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm19, ymm[19]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm20, ymm[20]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm21, ymm[21]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm22, ymm[22]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm23, ymm[23]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm24, ymm[24]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm25, ymm[25]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm26, ymm[26]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm27, ymm[27]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm28, ymm[28]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm29, ymm[29]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm30, ymm[30]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-ASMJIT_DEF_REG(X86YmmReg, ymm31, ymm[31]);  //!< 256-bit Ymm register (X64 & AVX512VL+).
-
-ASMJIT_DEF_REG(X86ZmmReg, zmm0 , zmm[0]);   //!< 512-bit Zmm register.
-ASMJIT_DEF_REG(X86ZmmReg, zmm1 , zmm[1]);   //!< 512-bit Zmm register.
-ASMJIT_DEF_REG(X86ZmmReg, zmm2 , zmm[2]);   //!< 512-bit Zmm register.
-ASMJIT_DEF_REG(X86ZmmReg, zmm3 , zmm[3]);   //!< 512-bit Zmm register.
-ASMJIT_DEF_REG(X86ZmmReg, zmm4 , zmm[4]);   //!< 512-bit Zmm register.
-ASMJIT_DEF_REG(X86ZmmReg, zmm5 , zmm[5]);   //!< 512-bit Zmm register.
-ASMJIT_DEF_REG(X86ZmmReg, zmm6 , zmm[6]);   //!< 512-bit Zmm register.
-ASMJIT_DEF_REG(X86ZmmReg, zmm7 , zmm[7]);   //!< 512-bit Zmm register.
-ASMJIT_DEF_REG(X86ZmmReg, zmm8 , zmm[8]);   //!< 512-bit Zmm register (X64).
-ASMJIT_DEF_REG(X86ZmmReg, zmm9 , zmm[9]);   //!< 512-bit Zmm register (X64).
-ASMJIT_DEF_REG(X86ZmmReg, zmm10, zmm[10]);  //!< 512-bit Zmm register (X64).
-ASMJIT_DEF_REG(X86ZmmReg, zmm11, zmm[11]);  //!< 512-bit Zmm register (X64).
-ASMJIT_DEF_REG(X86ZmmReg, zmm12, zmm[12]);  //!< 512-bit Zmm register (X64).
-ASMJIT_DEF_REG(X86ZmmReg, zmm13, zmm[13]);  //!< 512-bit Zmm register (X64).
-ASMJIT_DEF_REG(X86ZmmReg, zmm14, zmm[14]);  //!< 512-bit Zmm register (X64).
-ASMJIT_DEF_REG(X86ZmmReg, zmm15, zmm[15]);  //!< 512-bit Zmm register (X64).
-ASMJIT_DEF_REG(X86ZmmReg, zmm16, zmm[16]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm17, zmm[17]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm18, zmm[18]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm19, zmm[19]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm20, zmm[20]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm21, zmm[21]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm22, zmm[22]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm23, zmm[23]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm24, zmm[24]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm25, zmm[25]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm26, zmm[26]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm27, zmm[27]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm28, zmm[28]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm29, zmm[29]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm30, zmm[30]);  //!< 512-bit Zmm register (X64 & AVX512+).
-ASMJIT_DEF_REG(X86ZmmReg, zmm31, zmm[31]);  //!< 512-bit Zmm register (X64 & AVX512+).
-#undef ASMJIT_DEF_REG
-
-// This is only defined by `x86operand_regs.cpp` when exporting registers.
-#if !defined(ASMJIT_EXPORTS_X86OPERAND_REGS)
-
-//! Create 8-bit Gpb-lo register operand.
-static ASMJIT_INLINE X86GpReg gpb_lo(uint32_t index) { return X86GpReg(kX86RegTypeGpbLo, index, 1); }
-//! Create 8-bit Gpb-hi register operand.
-static ASMJIT_INLINE X86GpReg gpb_hi(uint32_t index) { return X86GpReg(kX86RegTypeGpbHi, index, 1); }
-//! Create 16-bit Gpw register operand.
-static ASMJIT_INLINE X86GpReg gpw(uint32_t index) { return X86GpReg(kX86RegTypeGpw, index, 2); }
-//! Create 32-bit Gpd register operand.
-static ASMJIT_INLINE X86GpReg gpd(uint32_t index) { return X86GpReg(kX86RegTypeGpd, index, 4); }
-//! Create 64-bit Gpq register operand (X64).
-static ASMJIT_INLINE X86GpReg gpq(uint32_t index) { return X86GpReg(kX86RegTypeGpq, index, 8); }
-//! Create 80-bit Fp register operand.
-static ASMJIT_INLINE X86FpReg fp(uint32_t index) { return X86FpReg(kX86RegTypeFp, index, 10); }
-//! Create 64-bit Mm register operand.
-static ASMJIT_INLINE X86MmReg mm(uint32_t index) { return X86MmReg(kX86RegTypeMm, index, 8); }
-//! Create 64-bit K register operand.
-static ASMJIT_INLINE X86KReg k(uint32_t index) { return X86KReg(kX86RegTypeK, index, 8); }
-//! Create 128-bit Xmm register operand.
-static ASMJIT_INLINE X86XmmReg xmm(uint32_t index) { return X86XmmReg(kX86RegTypeXmm, index, 16); }
-//! Create 256-bit Ymm register operand.
-static ASMJIT_INLINE X86YmmReg ymm(uint32_t index) { return X86YmmReg(kX86RegTypeYmm, index, 32); }
-//! Create 512-bit Zmm register operand.
-static ASMJIT_INLINE X86ZmmReg zmm(uint32_t index) { return X86ZmmReg(kX86RegTypeZmm, index, 64); }
+#if !defined(ASMJIT_EXPORTS_X86_OPERAND)
+namespace {
+#define ASMJIT_X86_PHYS_REG(TYPE, NAME, PROPERTY) \
+  static const TYPE& NAME = x86OpData.PROPERTY
+
+ASMJIT_X86_PHYS_REG(X86Rip , rip  , rip[0]);    //!< RIP register.
+ASMJIT_X86_PHYS_REG(X86Seg , es   , seg[1]);    //!< CS segment register.
+ASMJIT_X86_PHYS_REG(X86Seg , cs   , seg[2]);    //!< SS segment register.
+ASMJIT_X86_PHYS_REG(X86Seg , ss   , seg[3]);    //!< DS segment register.
+ASMJIT_X86_PHYS_REG(X86Seg , ds   , seg[4]);    //!< ES segment register.
+ASMJIT_X86_PHYS_REG(X86Seg , fs   , seg[5]);    //!< FS segment register.
+ASMJIT_X86_PHYS_REG(X86Seg , gs   , seg[6]);    //!< GS segment register.
+
+ASMJIT_X86_PHYS_REG(X86Gp  , al   , gpbLo[0]);  //!< 8-bit low GPB register.
+ASMJIT_X86_PHYS_REG(X86Gp  , cl   , gpbLo[1]);  //!< 8-bit low GPB register.
+ASMJIT_X86_PHYS_REG(X86Gp  , dl   , gpbLo[2]);  //!< 8-bit low GPB register.
+ASMJIT_X86_PHYS_REG(X86Gp  , bl   , gpbLo[3]);  //!< 8-bit low GPB register.
+ASMJIT_X86_PHYS_REG(X86Gp  , spl  , gpbLo[4]);  //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , bpl  , gpbLo[5]);  //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , sil  , gpbLo[6]);  //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , dil  , gpbLo[7]);  //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r8b  , gpbLo[8]);  //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r9b  , gpbLo[9]);  //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r10b , gpbLo[10]); //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r11b , gpbLo[11]); //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r12b , gpbLo[12]); //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r13b , gpbLo[13]); //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r14b , gpbLo[14]); //!< 8-bit low GPB register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r15b , gpbLo[15]); //!< 8-bit low GPB register (X64).
+
+ASMJIT_X86_PHYS_REG(X86Gp  , ah   , gpbHi[0]);  //!< 8-bit high GPB register.
+ASMJIT_X86_PHYS_REG(X86Gp  , ch   , gpbHi[1]);  //!< 8-bit high GPB register.
+ASMJIT_X86_PHYS_REG(X86Gp  , dh   , gpbHi[2]);  //!< 8-bit high GPB register.
+ASMJIT_X86_PHYS_REG(X86Gp  , bh   , gpbHi[3]);  //!< 8-bit high GPB register.
+
+ASMJIT_X86_PHYS_REG(X86Gp  , ax   , gpw[0]);    //!< 16-bit GPW register.
+ASMJIT_X86_PHYS_REG(X86Gp  , cx   , gpw[1]);    //!< 16-bit GPW register.
+ASMJIT_X86_PHYS_REG(X86Gp  , dx   , gpw[2]);    //!< 16-bit GPW register.
+ASMJIT_X86_PHYS_REG(X86Gp  , bx   , gpw[3]);    //!< 16-bit GPW register.
+ASMJIT_X86_PHYS_REG(X86Gp  , sp   , gpw[4]);    //!< 16-bit GPW register.
+ASMJIT_X86_PHYS_REG(X86Gp  , bp   , gpw[5]);    //!< 16-bit GPW register.
+ASMJIT_X86_PHYS_REG(X86Gp  , si   , gpw[6]);    //!< 16-bit GPW register.
+ASMJIT_X86_PHYS_REG(X86Gp  , di   , gpw[7]);    //!< 16-bit GPW register.
+ASMJIT_X86_PHYS_REG(X86Gp  , r8w  , gpw[8]);    //!< 16-bit GPW register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r9w  , gpw[9]);    //!< 16-bit GPW register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r10w , gpw[10]);   //!< 16-bit GPW register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r11w , gpw[11]);   //!< 16-bit GPW register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r12w , gpw[12]);   //!< 16-bit GPW register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r13w , gpw[13]);   //!< 16-bit GPW register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r14w , gpw[14]);   //!< 16-bit GPW register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r15w , gpw[15]);   //!< 16-bit GPW register (X64).
+
+ASMJIT_X86_PHYS_REG(X86Gp  , eax  , gpd[0]);    //!< 32-bit GPD register.
+ASMJIT_X86_PHYS_REG(X86Gp  , ecx  , gpd[1]);    //!< 32-bit GPD register.
+ASMJIT_X86_PHYS_REG(X86Gp  , edx  , gpd[2]);    //!< 32-bit GPD register.
+ASMJIT_X86_PHYS_REG(X86Gp  , ebx  , gpd[3]);    //!< 32-bit GPD register.
+ASMJIT_X86_PHYS_REG(X86Gp  , esp  , gpd[4]);    //!< 32-bit GPD register.
+ASMJIT_X86_PHYS_REG(X86Gp  , ebp  , gpd[5]);    //!< 32-bit GPD register.
+ASMJIT_X86_PHYS_REG(X86Gp  , esi  , gpd[6]);    //!< 32-bit GPD register.
+ASMJIT_X86_PHYS_REG(X86Gp  , edi  , gpd[7]);    //!< 32-bit GPD register.
+ASMJIT_X86_PHYS_REG(X86Gp  , r8d  , gpd[8]);    //!< 32-bit GPD register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r9d  , gpd[9]);    //!< 32-bit GPD register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r10d , gpd[10]);   //!< 32-bit GPD register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r11d , gpd[11]);   //!< 32-bit GPD register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r12d , gpd[12]);   //!< 32-bit GPD register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r13d , gpd[13]);   //!< 32-bit GPD register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r14d , gpd[14]);   //!< 32-bit GPD register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r15d , gpd[15]);   //!< 32-bit GPD register (X64).
+
+ASMJIT_X86_PHYS_REG(X86Gp  , rax  , gpq[0]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , rcx  , gpq[1]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , rdx  , gpq[2]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , rbx  , gpq[3]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , rsp  , gpq[4]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , rbp  , gpq[5]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , rsi  , gpq[6]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , rdi  , gpq[7]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r8   , gpq[8]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r9   , gpq[9]);    //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r10  , gpq[10]);   //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r11  , gpq[11]);   //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r12  , gpq[12]);   //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r13  , gpq[13]);   //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r14  , gpq[14]);   //!< 64-bit GPQ register (X64).
+ASMJIT_X86_PHYS_REG(X86Gp  , r15  , gpq[15]);   //!< 64-bit GPQ register (X64).
+
+ASMJIT_X86_PHYS_REG(X86Fp  , fp0  , fp[0]);     //!< 80-bit FPU register.
+ASMJIT_X86_PHYS_REG(X86Fp  , fp1  , fp[1]);     //!< 80-bit FPU register.
+ASMJIT_X86_PHYS_REG(X86Fp  , fp2  , fp[2]);     //!< 80-bit FPU register.
+ASMJIT_X86_PHYS_REG(X86Fp  , fp3  , fp[3]);     //!< 80-bit FPU register.
+ASMJIT_X86_PHYS_REG(X86Fp  , fp4  , fp[4]);     //!< 80-bit FPU register.
+ASMJIT_X86_PHYS_REG(X86Fp  , fp5  , fp[5]);     //!< 80-bit FPU register.
+ASMJIT_X86_PHYS_REG(X86Fp  , fp6  , fp[6]);     //!< 80-bit FPU register.
+ASMJIT_X86_PHYS_REG(X86Fp  , fp7  , fp[7]);     //!< 80-bit FPU register.
+
+ASMJIT_X86_PHYS_REG(X86Mm  , mm0  , mm[0]);     //!< 64-bit MMX register.
+ASMJIT_X86_PHYS_REG(X86Mm  , mm1  , mm[1]);     //!< 64-bit MMX register.
+ASMJIT_X86_PHYS_REG(X86Mm  , mm2  , mm[2]);     //!< 64-bit MMX register.
+ASMJIT_X86_PHYS_REG(X86Mm  , mm3  , mm[3]);     //!< 64-bit MMX register.
+ASMJIT_X86_PHYS_REG(X86Mm  , mm4  , mm[4]);     //!< 64-bit MMX register.
+ASMJIT_X86_PHYS_REG(X86Mm  , mm5  , mm[5]);     //!< 64-bit MMX register.
+ASMJIT_X86_PHYS_REG(X86Mm  , mm6  , mm[6]);     //!< 64-bit MMX register.
+ASMJIT_X86_PHYS_REG(X86Mm  , mm7  , mm[7]);     //!< 64-bit MMX register.
+
+ASMJIT_X86_PHYS_REG(X86KReg, k0   , k[0]);      //!< 64-bit K register.
+ASMJIT_X86_PHYS_REG(X86KReg, k1   , k[1]);      //!< 64-bit K register.
+ASMJIT_X86_PHYS_REG(X86KReg, k2   , k[2]);      //!< 64-bit K register.
+ASMJIT_X86_PHYS_REG(X86KReg, k3   , k[3]);      //!< 64-bit K register.
+ASMJIT_X86_PHYS_REG(X86KReg, k4   , k[4]);      //!< 64-bit K register.
+ASMJIT_X86_PHYS_REG(X86KReg, k5   , k[5]);      //!< 64-bit K register.
+ASMJIT_X86_PHYS_REG(X86KReg, k6   , k[6]);      //!< 64-bit K register.
+ASMJIT_X86_PHYS_REG(X86KReg, k7   , k[7]);      //!< 64-bit K register.
+
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm0 , xmm[0]);    //!< 128-bit XMM register.
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm1 , xmm[1]);    //!< 128-bit XMM register.
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm2 , xmm[2]);    //!< 128-bit XMM register.
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm3 , xmm[3]);    //!< 128-bit XMM register.
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm4 , xmm[4]);    //!< 128-bit XMM register.
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm5 , xmm[5]);    //!< 128-bit XMM register.
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm6 , xmm[6]);    //!< 128-bit XMM register.
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm7 , xmm[7]);    //!< 128-bit XMM register.
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm8 , xmm[8]);    //!< 128-bit XMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm9 , xmm[9]);    //!< 128-bit XMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm10, xmm[10]);   //!< 128-bit XMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm11, xmm[11]);   //!< 128-bit XMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm12, xmm[12]);   //!< 128-bit XMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm13, xmm[13]);   //!< 128-bit XMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm14, xmm[14]);   //!< 128-bit XMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm15, xmm[15]);   //!< 128-bit XMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm16, xmm[16]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm17, xmm[17]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm18, xmm[18]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm19, xmm[19]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm20, xmm[20]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm21, xmm[21]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm22, xmm[22]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm23, xmm[23]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm24, xmm[24]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm25, xmm[25]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm26, xmm[26]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm27, xmm[27]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm28, xmm[28]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm29, xmm[29]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm30, xmm[30]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Xmm , xmm31, xmm[31]);   //!< 128-bit XMM register (X64 & AVX512_VL+).
+
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm0 , ymm[0]);    //!< 256-bit YMM register.
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm1 , ymm[1]);    //!< 256-bit YMM register.
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm2 , ymm[2]);    //!< 256-bit YMM register.
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm3 , ymm[3]);    //!< 256-bit YMM register.
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm4 , ymm[4]);    //!< 256-bit YMM register.
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm5 , ymm[5]);    //!< 256-bit YMM register.
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm6 , ymm[6]);    //!< 256-bit YMM register.
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm7 , ymm[7]);    //!< 256-bit YMM register.
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm8 , ymm[8]);    //!< 256-bit YMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm9 , ymm[9]);    //!< 256-bit YMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm10, ymm[10]);   //!< 256-bit YMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm11, ymm[11]);   //!< 256-bit YMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm12, ymm[12]);   //!< 256-bit YMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm13, ymm[13]);   //!< 256-bit YMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm14, ymm[14]);   //!< 256-bit YMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm15, ymm[15]);   //!< 256-bit YMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm16, ymm[16]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm17, ymm[17]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm18, ymm[18]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm19, ymm[19]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm20, ymm[20]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm21, ymm[21]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm22, ymm[22]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm23, ymm[23]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm24, ymm[24]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm25, ymm[25]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm26, ymm[26]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm27, ymm[27]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm28, ymm[28]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm29, ymm[29]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm30, ymm[30]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+ASMJIT_X86_PHYS_REG(X86Ymm , ymm31, ymm[31]);   //!< 256-bit YMM register (X64 & AVX512_VL+).
+
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm0 , zmm[0]);    //!< 512-bit ZMM register.
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm1 , zmm[1]);    //!< 512-bit ZMM register.
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm2 , zmm[2]);    //!< 512-bit ZMM register.
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm3 , zmm[3]);    //!< 512-bit ZMM register.
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm4 , zmm[4]);    //!< 512-bit ZMM register.
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm5 , zmm[5]);    //!< 512-bit ZMM register.
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm6 , zmm[6]);    //!< 512-bit ZMM register.
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm7 , zmm[7]);    //!< 512-bit ZMM register.
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm8 , zmm[8]);    //!< 512-bit ZMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm9 , zmm[9]);    //!< 512-bit ZMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm10, zmm[10]);   //!< 512-bit ZMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm11, zmm[11]);   //!< 512-bit ZMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm12, zmm[12]);   //!< 512-bit ZMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm13, zmm[13]);   //!< 512-bit ZMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm14, zmm[14]);   //!< 512-bit ZMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm15, zmm[15]);   //!< 512-bit ZMM register (X64).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm16, zmm[16]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm17, zmm[17]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm18, zmm[18]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm19, zmm[19]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm20, zmm[20]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm21, zmm[21]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm22, zmm[22]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm23, zmm[23]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm24, zmm[24]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm25, zmm[25]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm26, zmm[26]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm27, zmm[27]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm28, zmm[28]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm29, zmm[29]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm30, zmm[30]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+ASMJIT_X86_PHYS_REG(X86Zmm , zmm31, zmm[31]);   //!< 512-bit ZMM register (X64 & AVX512_F+).
+
+ASMJIT_X86_PHYS_REG(X86Bnd , bnd0 , bnd[0]);    //!< 128-bit bound register.
+ASMJIT_X86_PHYS_REG(X86Bnd , bnd1 , bnd[1]);    //!< 128-bit bound register.
+ASMJIT_X86_PHYS_REG(X86Bnd , bnd2 , bnd[2]);    //!< 128-bit bound register.
+ASMJIT_X86_PHYS_REG(X86Bnd , bnd3 , bnd[3]);    //!< 128-bit bound register.
+
+ASMJIT_X86_PHYS_REG(X86CReg, cr0  , cr[0]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr1  , cr[1]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr2  , cr[2]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr3  , cr[3]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr4  , cr[4]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr5  , cr[5]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr6  , cr[6]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr7  , cr[7]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr8  , cr[8]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr9  , cr[9]);     //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr10 , cr[10]);    //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr11 , cr[11]);    //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr12 , cr[12]);    //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr13 , cr[13]);    //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr14 , cr[14]);    //!< 32-bit or 64-bit control register.
+ASMJIT_X86_PHYS_REG(X86CReg, cr15 , cr[15]);    //!< 32-bit or 64-bit control register.
+
+ASMJIT_X86_PHYS_REG(X86DReg, dr0  , dr[0]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr1  , dr[1]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr2  , dr[2]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr3  , dr[3]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr4  , dr[4]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr5  , dr[5]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr6  , dr[6]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr7  , dr[7]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr8  , dr[8]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr9  , dr[9]);     //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr10 , dr[10]);    //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr11 , dr[11]);    //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr12 , dr[12]);    //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr13 , dr[13]);    //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr14 , dr[14]);    //!< 32-bit or 64-bit debug register.
+ASMJIT_X86_PHYS_REG(X86DReg, dr15 , dr[15]);    //!< 32-bit or 64-bit debug register.
+
+#undef ASMJIT_X86_PHYS_REG
+} // anonymous namespace
+#endif // !ASMJIT_EXPORTS_X86_OPERAND
+
+//! Create an 8-bit low GPB register operand.
+static ASMJIT_INLINE X86GpbLo gpb(uint32_t rId) noexcept { return X86GpbLo(rId); }
+//! Create an 8-bit low GPB register operand.
+static ASMJIT_INLINE X86GpbLo gpb_lo(uint32_t rId) noexcept { return X86GpbLo(rId); }
+//! Create an 8-bit high GPB register operand.
+static ASMJIT_INLINE X86GpbHi gpb_hi(uint32_t rId) noexcept { return X86GpbHi(rId); }
+//! Create a 16-bit GPW register operand.
+static ASMJIT_INLINE X86Gpw gpw(uint32_t rId) noexcept { return X86Gpw(rId); }
+//! Create a 32-bit GPD register operand.
+static ASMJIT_INLINE X86Gpd gpd(uint32_t rId) noexcept { return X86Gpd(rId); }
+//! Create a 64-bit GPQ register operand (X64).
+static ASMJIT_INLINE X86Gpq gpq(uint32_t rId) noexcept { return X86Gpq(rId); }
+//! Create an 80-bit Fp register operand.
+static ASMJIT_INLINE X86Fp fp(uint32_t rId) noexcept { return X86Fp(rId); }
+//! Create a 64-bit Mm register operand.
+static ASMJIT_INLINE X86Mm mm(uint32_t rId) noexcept { return X86Mm(rId); }
+//! Create a 64-bit K register operand.
+static ASMJIT_INLINE X86KReg k(uint32_t rId) noexcept { return X86KReg(rId); }
+//! Create a 128-bit XMM register operand.
+static ASMJIT_INLINE X86Xmm xmm(uint32_t rId) noexcept { return X86Xmm(rId); }
+//! Create a 256-bit YMM register operand.
+static ASMJIT_INLINE X86Ymm ymm(uint32_t rId) noexcept { return X86Ymm(rId); }
+//! Create a 512-bit ZMM register operand.
+static ASMJIT_INLINE X86Zmm zmm(uint32_t rId) noexcept { return X86Zmm(rId); }
+//! Create a 128-bit bound register operand.
+static ASMJIT_INLINE X86Bnd bnd(uint32_t rId) noexcept { return X86Bnd(rId); }
+//! Create a 32-bit or 64-bit control register operand.
+static ASMJIT_INLINE X86CReg cr(uint32_t rId) noexcept { return X86CReg(rId); }
+//! Create a 32-bit or 64-bit debug register operand.
+static ASMJIT_INLINE X86DReg dr(uint32_t rId) noexcept { return X86DReg(rId); }
 
 // ============================================================================
 // [asmjit::x86 - Ptr (Reg)]
 // ============================================================================
 
-//! Create `[base.reg + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const X86GpReg& base, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(base, disp, size);
+//! Create a `[base.reg + offset]` memory operand.
+static ASMJIT_INLINE X86Mem ptr(const X86Gp& base, int32_t offset = 0, uint32_t size = 0) noexcept {
+  return X86Mem(base, offset, size);
 }
-//! Create `[base.reg + (index.reg << shift) + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const X86GpReg& base, const X86GpReg& index, uint32_t shift = 0, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(base, index, shift, disp, size);
+//! Create a `[base.reg + (index << shift) + offset]` memory operand (scalar index).
+static ASMJIT_INLINE X86Mem ptr(const X86Gp& base, const X86Gp& index, uint32_t shift = 0, int32_t offset = 0, uint32_t size = 0) noexcept {
+  return X86Mem(base, index, shift, offset, size);
 }
-//! Create `[base.reg + (xmm.reg << shift) + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const X86GpReg& base, const X86XmmReg& index, uint32_t shift = 0, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(base, index, shift, disp, size);
+//! Create a `[base.reg + (index << shift) + offset]` memory operand (vector index).
+static ASMJIT_INLINE X86Mem ptr(const X86Gp& base, const X86Vec& index, uint32_t shift = 0, int32_t offset = 0, uint32_t size = 0) noexcept {
+  return X86Mem(base, index, shift, offset, size);
 }
-//! Create `[base.reg + (ymm.reg << shift) + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const X86GpReg& base, const X86YmmReg& index, uint32_t shift = 0, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(base, index, shift, disp, size);
+
+//! Create a `[base + offset]` memory operand.
+static ASMJIT_INLINE X86Mem ptr(const Label& base, int32_t offset = 0, uint32_t size = 0) noexcept {
+  return X86Mem(base, offset, size);
 }
-//! Create `[label + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const Label& label, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(label, disp, size);
+//! Create a `[base + (index << shift) + offset]` memory operand.
+static ASMJIT_INLINE X86Mem ptr(const Label& base, const X86Gp& index, uint32_t shift, int32_t offset = 0, uint32_t size = 0) noexcept {
+  return X86Mem(base, index, shift, offset, size);
 }
-//! Create `[label + (index.reg << shift) + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const Label& label, const X86GpReg& index, uint32_t shift, int32_t disp = 0, uint32_t size = 0) { \
-  return X86Mem(label, index, shift, disp, size); \
+//! Create a `[base + (index << shift) + offset]` memory operand.
+static ASMJIT_INLINE X86Mem ptr(const Label& base, const X86Vec& index, uint32_t shift, int32_t offset = 0, uint32_t size = 0) noexcept {
+  return X86Mem(base, index, shift, offset, size);
 }
 
-//! Create `[pAbs + disp]` absolute memory operand with no/custom size information.
-ASMJIT_API X86Mem ptr_abs(Ptr pAbs, int32_t disp = 0, uint32_t size = 0);
-//! Create `[pAbs + (index.reg << shift) + disp]` absolute memory operand with no/custom size information.
-ASMJIT_API X86Mem ptr_abs(Ptr pAbs, const X86Reg& index, uint32_t shift = 0, int32_t disp = 0, uint32_t size = 0);
-
-//! \internal
-#define ASMJIT_EXPAND_PTR_REG(_Prefix_, _Size_) \
-  /*! Create `[base.reg + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const X86GpReg& base, int32_t disp = 0) { \
-    return X86Mem(base, disp, _Size_); \
-  } \
-  /*! Create `[base.reg + (index.reg << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const X86GpReg& base, const X86GpReg& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr(base, index, shift, disp, _Size_); \
-  } \
-  /*! Create `[base.reg + (xmm.reg << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const X86GpReg& base, const X86XmmReg& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr(base, index, shift, disp, _Size_); \
-  } \
-  /*! Create `[base.reg + (ymm.reg << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const X86GpReg& base, const X86YmmReg& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr(base, index, shift, disp, _Size_); \
-  } \
-  /*! Create `[label + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const Label& label, int32_t disp = 0) { \
-    return ptr(label, disp, _Size_); \
-  } \
-  /*! Create `[label + (index.reg << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const Label& label, const X86GpReg& index, uint32_t shift, int32_t disp = 0) { \
-    return ptr(label, index, shift, disp, _Size_); \
-  } \
-  /*! Create `[pAbs + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr##_abs(Ptr pAbs, int32_t disp = 0) { \
-    return ptr_abs(pAbs, disp, _Size_); \
-  } \
-  /*! Create `[pAbs + (index.reg << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr##_abs(Ptr pAbs, const X86GpReg& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr_abs(pAbs, index, shift, disp, _Size_); \
-  } \
-  /*! Create `[pAbs + (xmm.reg << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr##_abs(Ptr pAbs, const X86XmmReg& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr_abs(pAbs, index, shift, disp, _Size_); \
-  } \
-  /*! Create `[pAbs + (ymm.reg << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr##_abs(Ptr pAbs, const X86YmmReg& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr_abs(pAbs, index, shift, disp, _Size_); \
-  }
-
-ASMJIT_EXPAND_PTR_REG(byte, 1)
-ASMJIT_EXPAND_PTR_REG(word, 2)
-ASMJIT_EXPAND_PTR_REG(dword, 4)
-ASMJIT_EXPAND_PTR_REG(qword, 8)
-ASMJIT_EXPAND_PTR_REG(tword, 10)
-ASMJIT_EXPAND_PTR_REG(oword, 16)
-ASMJIT_EXPAND_PTR_REG(yword, 32)
-ASMJIT_EXPAND_PTR_REG(zword, 64)
-#undef ASMJIT_EXPAND_PTR_REG
-
-// ============================================================================
-// [asmjit::x86 - Ptr (Var)]
-// ============================================================================
-
-#if !defined(ASMJIT_DISABLE_COMPILER)
-//! Create `[base.var + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const X86GpVar& base, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(base, disp, size);
-}
-//! Create `[base.var + (index.var << shift) + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const X86GpVar& base, const X86GpVar& index, uint32_t shift = 0, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(base, index, shift, disp, size);
+//! Create `[rip + offset]` memory operand.
+static ASMJIT_INLINE X86Mem ptr(const X86Rip& rip_, int32_t offset = 0, uint32_t size = 0) noexcept {
+  return X86Mem(rip_, offset, size);
 }
-//! Create `[base.var + (xmm.var << shift) + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const X86GpVar& base, const X86XmmVar& index, uint32_t shift = 0, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(base, index, shift, disp, size);
+
+//! Create an `[base]` absolute memory operand.
+static ASMJIT_INLINE X86Mem ptr(uint64_t base, uint32_t size = 0) noexcept {
+  return X86Mem(base, size);
 }
-//! Create `[base.var + (ymm.var << shift) + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const X86GpVar& base, const X86YmmVar& index, uint32_t shift = 0, int32_t disp = 0, uint32_t size = 0) {
-  return X86Mem(base, index, shift, disp, size);
+//! Create an `[abs + (index.reg << shift)]` absolute memory operand.
+static ASMJIT_INLINE X86Mem ptr(uint64_t base, const X86Reg& index, uint32_t shift = 0, uint32_t size = 0) noexcept {
+  return X86Mem(base, index, shift, size);
 }
-//! Create `[label + (index.var << shift) + disp]` memory operand with no/custom size information.
-static ASMJIT_INLINE X86Mem ptr(const Label& label, const X86GpVar& index, uint32_t shift, int32_t disp = 0, uint32_t size = 0) { \
-  return X86Mem(label, index, shift, disp, size); \
+//! Create an `[abs + (index.reg << shift)]` absolute memory operand.
+static ASMJIT_INLINE X86Mem ptr(uint64_t base, const X86Vec& index, uint32_t shift = 0, uint32_t size = 0) noexcept {
+  return X86Mem(base, index, shift, size);
 }
 
-//! Create `[pAbs + (index.var << shift) + disp]` absolute memory operand with no/custom size information.
-ASMJIT_API X86Mem ptr_abs(Ptr pAbs, const X86Var& index, uint32_t shift = 0, int32_t disp = 0, uint32_t size = 0);
-
 //! \internal
-#define ASMJIT_EXPAND_PTR_VAR(_Prefix_, _Size_) \
-  /*! Create `[base.var + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const X86GpVar& base, int32_t disp = 0) { \
-    return X86Mem(base, disp, _Size_); \
+#define ASMJIT_X86_PTR_FN(FUNC, SIZE)                                                 \
+  /*! Create a `[base + offset]` memory operand. */                                   \
+  static ASMJIT_INLINE X86Mem FUNC(const X86Gp& base, int32_t offset = 0) noexcept {  \
+    return X86Mem(base, offset, SIZE);                                                \
   }                                                                                   \
-  /*! Create `[base.var + (index.var << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const X86GpVar& base, const X86GpVar& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr(base, index, shift, disp, _Size_); \
+  /*! Create a `[base + (index << shift) + offset]` memory operand. */                \
+  static ASMJIT_INLINE X86Mem FUNC(const X86Gp& base, const X86Gp& index, uint32_t shift = 0, int32_t offset = 0) noexcept { \
+    return X86Mem(base, index, shift, offset, SIZE);                                  \
   }                                                                                   \
-  /*! Create `[base.var + (xmm.var << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const X86GpVar& base, const X86XmmVar& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr(base, index, shift, disp, _Size_); \
+  /*! Create a `[base + (vec_index << shift) + offset]` memory operand. */            \
+  static ASMJIT_INLINE X86Mem FUNC(const X86Gp& base, const X86Vec& index, uint32_t shift = 0, int32_t offset = 0) noexcept { \
+    return X86Mem(base, index, shift, offset, SIZE);                                  \
   }                                                                                   \
-  /*! Create `[base.var + (ymm.var << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const X86GpVar& base, const X86YmmVar& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr(base, index, shift, disp, _Size_); \
+  /*! Create a `[base + offset]` memory operand. */                                   \
+  static ASMJIT_INLINE X86Mem FUNC(const Label& base, int32_t offset = 0) noexcept {  \
+    return X86Mem(base, offset, SIZE);                                                \
   }                                                                                   \
-  /*! Create `[label + (index.var << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr(const Label& label, const X86GpVar& index, uint32_t shift, int32_t disp = 0) { \
-    return ptr(label, index, shift, disp, _Size_); \
+  /*! Create a `[base + (index << shift) + offset]` memory operand. */                \
+  static ASMJIT_INLINE X86Mem FUNC(const Label& base, const X86Gp& index, uint32_t shift, int32_t offset = 0) noexcept { \
+    return X86Mem(base, index, shift, offset, SIZE);                                  \
   }                                                                                   \
-  /*! Create `[pAbs + (index.var << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr##_abs(Ptr pAbs, const X86GpVar& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr_abs(pAbs, reinterpret_cast<const X86Var&>(index), shift, disp, _Size_); \
+  /*! Create a `[rip + offset]` memory operand. */                                    \
+  static ASMJIT_INLINE X86Mem FUNC(const X86Rip& rip_, int32_t offset = 0) noexcept { \
+    return X86Mem(rip_, offset, SIZE);                                                \
   }                                                                                   \
-  /*! Create `[pAbs + (xmm.var << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr##_abs(Ptr pAbs, const X86XmmVar& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr_abs(pAbs, reinterpret_cast<const X86Var&>(index), shift, disp, _Size_); \
+  /*! Create a `[base + offset]` memory operand. */                                   \
+  static ASMJIT_INLINE X86Mem FUNC(uint64_t base) noexcept {                          \
+    return X86Mem(base, SIZE);                                                        \
   }                                                                                   \
-  /*! Create `[pAbs + (ymm.var << shift) + disp]` memory operand. */ \
-  static ASMJIT_INLINE X86Mem _Prefix_##_ptr##_abs(Ptr pAbs, const X86YmmVar& index, uint32_t shift = 0, int32_t disp = 0) { \
-    return ptr_abs(pAbs, reinterpret_cast<const X86Var&>(index), shift, disp, _Size_); \
-  }
-
-ASMJIT_EXPAND_PTR_VAR(byte, 1)
-ASMJIT_EXPAND_PTR_VAR(word, 2)
-ASMJIT_EXPAND_PTR_VAR(dword, 4)
-ASMJIT_EXPAND_PTR_VAR(qword, 8)
-ASMJIT_EXPAND_PTR_VAR(tword, 10)
-ASMJIT_EXPAND_PTR_VAR(oword, 16)
-ASMJIT_EXPAND_PTR_VAR(yword, 32)
-ASMJIT_EXPAND_PTR_VAR(zword, 64)
-#undef ASMJIT_EXPAND_PTR_VAR
-#endif // !ASMJIT_DISABLE_COMPILER
-
-#endif // !ASMJIT_EXPORTS_X86OPERAND_REGS
+  /*! Create a `[base + (index << shift) + offset]` memory operand. */                \
+  static ASMJIT_INLINE X86Mem FUNC(uint64_t base, const X86Gp& index, uint32_t shift = 0) noexcept { \
+    return X86Mem(base, index, shift, SIZE);                                          \
+  }                                                                                   \
+  /*! Create a `[base + (vec_index << shift) + offset]` memory operand. */            \
+  static ASMJIT_INLINE X86Mem FUNC(uint64_t base, const X86Vec& index, uint32_t shift = 0) noexcept { \
+    return X86Mem(base, index, shift, SIZE, Mem::kSignatureMemAbs);                   \
+  }                                                                                   \
+  /*! Create a `[base + offset]` memory operand. */                                   \
+  static ASMJIT_INLINE X86Mem FUNC##_abs(uint64_t base) noexcept {                    \
+    return X86Mem(base, SIZE);                                                        \
+  }                                                                                   \
+  /*! Create a `[base + (index << shift) + offset]` memory operand. */                \
+  static ASMJIT_INLINE X86Mem FUNC##_abs(uint64_t base, const X86Gp& index, uint32_t shift = 0) noexcept { \
+    return X86Mem(base, index, shift, SIZE, Mem::kSignatureMemAbs);                   \
+  }                                                                                   \
+  /*! Create a `[base + (vec_index << shift) + offset]` memory operand. */            \
+  static ASMJIT_INLINE X86Mem FUNC##_abs(uint64_t base, const X86Vec& index, uint32_t shift = 0) noexcept { \
+    return X86Mem(base, index, shift, SIZE, Mem::kSignatureMemAbs);                   \
+  }
+
+// Define memory operand constructors that use platform independent naming.
+ASMJIT_X86_PTR_FN(ptr_8, 1)
+ASMJIT_X86_PTR_FN(ptr_16, 2)
+ASMJIT_X86_PTR_FN(ptr_32, 4)
+ASMJIT_X86_PTR_FN(ptr_48, 6)
+ASMJIT_X86_PTR_FN(ptr_64, 8)
+ASMJIT_X86_PTR_FN(ptr_80, 10)
+ASMJIT_X86_PTR_FN(ptr_128, 16)
+ASMJIT_X86_PTR_FN(ptr_256, 32)
+ASMJIT_X86_PTR_FN(ptr_512, 64)
+
+// Define memory operand constructors that use X86/X64 specific naming.
+ASMJIT_X86_PTR_FN(byte_ptr, 1)
+ASMJIT_X86_PTR_FN(word_ptr, 2)
+ASMJIT_X86_PTR_FN(dword_ptr, 4)
+ASMJIT_X86_PTR_FN(qword_ptr, 8)
+ASMJIT_X86_PTR_FN(tword_ptr, 10)
+ASMJIT_X86_PTR_FN(oword_ptr, 16)
+ASMJIT_X86_PTR_FN(dqword_ptr, 16)
+ASMJIT_X86_PTR_FN(yword_ptr, 32)
+ASMJIT_X86_PTR_FN(zword_ptr, 64)
+
+#undef ASMJIT_X86_PTR_FN
 
 } // x86 namespace
 
@@ -2020,11 +1097,8 @@ ASMJIT_EXPAND_PTR_VAR(zword, 64)
 
 } // asmjit namespace
 
-// [Cleanup]
-#undef ASMJIT_OP_ID
-
 // [Api-End]
-#include "../apiend.h"
+#include "../asmjit_apiend.h"
 
 // [Guard]
 #endif // _ASMJIT_X86_X86OPERAND_H

libraries/asmjit/x86/x86operand_regs.cpp

@@ -6,280 +6,117 @@
 
 // [Export]
 #define ASMJIT_EXPORTS
-#define ASMJIT_EXPORTS_X86OPERAND_REGS
+#define ASMJIT_EXPORTS_X86_OPERAND
 
 // [Guard]
-#include "../build.h"
-#if defined(ASMJIT_BUILD_X86) || defined(ASMJIT_BUILD_X64)
+#include "../asmjit_build.h"
+#if defined(ASMJIT_BUILD_X86)
 
-// [Dependencies - AsmJit]
+// [Dependencies]
+#include "../base/misc_p.h"
 #include "../x86/x86operand.h"
 
 // [Api-Begin]
-#include "../apibegin.h"
+#include "../asmjit_apibegin.h"
 
 namespace asmjit {
 
-#define REG(_Type_, _Index_, _Size_) {{ \
-  kOperandTypeReg, _Size_, { ((_Type_) << 8) + _Index_ }, kInvalidValue, {{ kInvalidVar, 0 }} \
-}}
+// ============================================================================
+// [asmjit::X86OpData]
+// ============================================================================
+
+// Register Operand {
+//   uint32_t signature;
+//   uint32_t id;
+//   uint32_t reserved8_4;
+//   uint32_t reserved12_4;
+// }
+#define ASMJIT_X86_REG_01(TYPE, ID)         \
+{{{                                         \
+  uint32_t(X86RegTraits<TYPE>::kSignature), \
+  uint32_t(ID),                             \
+  uint32_t(0),                              \
+  uint32_t(0)                               \
+}}}
+
+#define ASMJIT_X86_REG_04(TYPE, ID) \
+  ASMJIT_X86_REG_01(TYPE, ID + 0 ), \
+  ASMJIT_X86_REG_01(TYPE, ID + 1 ), \
+  ASMJIT_X86_REG_01(TYPE, ID + 2 ), \
+  ASMJIT_X86_REG_01(TYPE, ID + 3 )
+
+#define ASMJIT_X86_REG_07(TYPE, ID) \
+  ASMJIT_X86_REG_04(TYPE, ID + 0 ), \
+  ASMJIT_X86_REG_01(TYPE, ID + 4 ), \
+  ASMJIT_X86_REG_01(TYPE, ID + 5 ), \
+  ASMJIT_X86_REG_01(TYPE, ID + 6 )
+
+#define ASMJIT_X86_REG_08(TYPE, ID) \
+  ASMJIT_X86_REG_04(TYPE, ID + 0 ), \
+  ASMJIT_X86_REG_04(TYPE, ID + 4 )
+
+#define ASMJIT_X86_REG_16(TYPE, ID) \
+  ASMJIT_X86_REG_08(TYPE, ID + 0 ), \
+  ASMJIT_X86_REG_08(TYPE, ID + 8 )
+
+#define ASMJIT_X86_REG_32(TYPE, ID) \
+  ASMJIT_X86_REG_16(TYPE, ID + 0 ), \
+  ASMJIT_X86_REG_16(TYPE, ID + 16)
+
+const X86OpData x86OpData = {
+  // --------------------------------------------------------------------------
+  // [ArchRegs]
+  // --------------------------------------------------------------------------
 
-const X86RegData x86RegData = {
-  // RIP.
-  REG(kX86RegTypeRip, 0, 0),
-  // NpGp.
-  REG(kInvalidReg, kInvalidReg, 0),
-
-  // Segments.
-  {
-    REG(kX86RegTypeSeg, 0, 2), // Default.
-    REG(kX86RegTypeSeg, 1, 2), // ES.
-    REG(kX86RegTypeSeg, 2, 2), // CS.
-    REG(kX86RegTypeSeg, 3, 2), // SS.
-    REG(kX86RegTypeSeg, 4, 2), // DS.
-    REG(kX86RegTypeSeg, 5, 2), // FS.
-    REG(kX86RegTypeSeg, 6, 2)  // GS.
-  },
-
-  // GpbLo.
   {
-    REG(kX86RegTypeGpbLo, 0, 1),
-    REG(kX86RegTypeGpbLo, 1, 1),
-    REG(kX86RegTypeGpbLo, 2, 1),
-    REG(kX86RegTypeGpbLo, 3, 1),
-    REG(kX86RegTypeGpbLo, 4, 1),
-    REG(kX86RegTypeGpbLo, 5, 1),
-    REG(kX86RegTypeGpbLo, 6, 1),
-    REG(kX86RegTypeGpbLo, 7, 1),
-    REG(kX86RegTypeGpbLo, 8, 1),
-    REG(kX86RegTypeGpbLo, 9, 1),
-    REG(kX86RegTypeGpbLo, 10, 1),
-    REG(kX86RegTypeGpbLo, 11, 1),
-    REG(kX86RegTypeGpbLo, 12, 1),
-    REG(kX86RegTypeGpbLo, 13, 1),
-    REG(kX86RegTypeGpbLo, 14, 1),
-    REG(kX86RegTypeGpbLo, 15, 1)
-  },
-
-  // GpbHi.
     {
-    REG(kX86RegTypeGpbHi, 0, 1),
-    REG(kX86RegTypeGpbHi, 1, 1),
-    REG(kX86RegTypeGpbHi, 2, 1),
-    REG(kX86RegTypeGpbHi, 3, 1)
+#define ASMJIT_X86_REG_SIGNATURE(TYPE) { X86RegTraits<TYPE>::kSignature }
+      ASMJIT_TABLE_16(ASMJIT_X86_REG_SIGNATURE,  0),
+      ASMJIT_TABLE_16(ASMJIT_X86_REG_SIGNATURE, 16)
+#undef ASMJIT_X86_REG_SIGNATURE
     },
 
-  // Gpw.
-  {
-    REG(kX86RegTypeGpw, 0, 2),
-    REG(kX86RegTypeGpw, 1, 2),
-    REG(kX86RegTypeGpw, 2, 2),
-    REG(kX86RegTypeGpw, 3, 2),
-    REG(kX86RegTypeGpw, 4, 2),
-    REG(kX86RegTypeGpw, 5, 2),
-    REG(kX86RegTypeGpw, 6, 2),
-    REG(kX86RegTypeGpw, 7, 2),
-    REG(kX86RegTypeGpw, 8, 2),
-    REG(kX86RegTypeGpw, 9, 2),
-    REG(kX86RegTypeGpw, 10, 2),
-    REG(kX86RegTypeGpw, 11, 2),
-    REG(kX86RegTypeGpw, 12, 2),
-    REG(kX86RegTypeGpw, 13, 2),
-    REG(kX86RegTypeGpw, 14, 2),
-    REG(kX86RegTypeGpw, 15, 2)
-  },
+    // RegCount[]
+    { ASMJIT_TABLE_T_32(X86RegTraits, kCount, 0) },
 
-  // Gpd.
-  {
-    REG(kX86RegTypeGpd, 0, 4),
-    REG(kX86RegTypeGpd, 1, 4),
-    REG(kX86RegTypeGpd, 2, 4),
-    REG(kX86RegTypeGpd, 3, 4),
-    REG(kX86RegTypeGpd, 4, 4),
-    REG(kX86RegTypeGpd, 5, 4),
-    REG(kX86RegTypeGpd, 6, 4),
-    REG(kX86RegTypeGpd, 7, 4),
-    REG(kX86RegTypeGpd, 8, 4),
-    REG(kX86RegTypeGpd, 9, 4),
-    REG(kX86RegTypeGpd, 10, 4),
-    REG(kX86RegTypeGpd, 11, 4),
-    REG(kX86RegTypeGpd, 12, 4),
-    REG(kX86RegTypeGpd, 13, 4),
-    REG(kX86RegTypeGpd, 14, 4),
-    REG(kX86RegTypeGpd, 15, 4)
-  },
-
-  // Gpq.
-  {
-    REG(kX86RegTypeGpq, 0, 8),
-    REG(kX86RegTypeGpq, 1, 8),
-    REG(kX86RegTypeGpq, 2, 8),
-    REG(kX86RegTypeGpq, 3, 8),
-    REG(kX86RegTypeGpq, 4, 8),
-    REG(kX86RegTypeGpq, 5, 8),
-    REG(kX86RegTypeGpq, 6, 8),
-    REG(kX86RegTypeGpq, 7, 8),
-    REG(kX86RegTypeGpq, 8, 8),
-    REG(kX86RegTypeGpq, 9, 8),
-    REG(kX86RegTypeGpq, 10, 8),
-    REG(kX86RegTypeGpq, 11, 8),
-    REG(kX86RegTypeGpq, 12, 8),
-    REG(kX86RegTypeGpq, 13, 8),
-    REG(kX86RegTypeGpq, 14, 8),
-    REG(kX86RegTypeGpq, 15, 8)
-  },
-
-  // Fp.
-  {
-    REG(kX86RegTypeFp, 0, 10),
-    REG(kX86RegTypeFp, 1, 10),
-    REG(kX86RegTypeFp, 2, 10),
-    REG(kX86RegTypeFp, 3, 10),
-    REG(kX86RegTypeFp, 4, 10),
-    REG(kX86RegTypeFp, 5, 10),
-    REG(kX86RegTypeFp, 6, 10),
-    REG(kX86RegTypeFp, 7, 10)
+    // RegTypeToTypeId[]
+    { ASMJIT_TABLE_T_32(X86RegTraits, kTypeId, 0) }
   },
 
-  // Mm.
-  {
-    REG(kX86RegTypeMm, 0, 8),
-    REG(kX86RegTypeMm, 1, 8),
-    REG(kX86RegTypeMm, 2, 8),
-    REG(kX86RegTypeMm, 3, 8),
-    REG(kX86RegTypeMm, 4, 8),
-    REG(kX86RegTypeMm, 5, 8),
-    REG(kX86RegTypeMm, 6, 8),
-    REG(kX86RegTypeMm, 7, 8)
-  },
-
-  // K.
-  {
-    REG(kX86RegTypeK, 0, 8),
-    REG(kX86RegTypeK, 1, 8),
-    REG(kX86RegTypeK, 2, 8),
-    REG(kX86RegTypeK, 3, 8),
-    REG(kX86RegTypeK, 4, 8),
-    REG(kX86RegTypeK, 5, 8),
-    REG(kX86RegTypeK, 6, 8),
-    REG(kX86RegTypeK, 7, 8)
-  },
-
-  // Xmm.
-  {
-    REG(kX86RegTypeXmm, 0, 16),
-    REG(kX86RegTypeXmm, 1, 16),
-    REG(kX86RegTypeXmm, 2, 16),
-    REG(kX86RegTypeXmm, 3, 16),
-    REG(kX86RegTypeXmm, 4, 16),
-    REG(kX86RegTypeXmm, 5, 16),
-    REG(kX86RegTypeXmm, 6, 16),
-    REG(kX86RegTypeXmm, 7, 16),
-    REG(kX86RegTypeXmm, 8, 16),
-    REG(kX86RegTypeXmm, 9, 16),
-    REG(kX86RegTypeXmm, 10, 16),
-    REG(kX86RegTypeXmm, 11, 16),
-    REG(kX86RegTypeXmm, 12, 16),
-    REG(kX86RegTypeXmm, 13, 16),
-    REG(kX86RegTypeXmm, 14, 16),
-    REG(kX86RegTypeXmm, 15, 16),
-    REG(kX86RegTypeXmm, 16, 16),
-    REG(kX86RegTypeXmm, 17, 16),
-    REG(kX86RegTypeXmm, 18, 16),
-    REG(kX86RegTypeXmm, 19, 16),
-    REG(kX86RegTypeXmm, 20, 16),
-    REG(kX86RegTypeXmm, 21, 16),
-    REG(kX86RegTypeXmm, 22, 16),
-    REG(kX86RegTypeXmm, 23, 16),
-    REG(kX86RegTypeXmm, 24, 16),
-    REG(kX86RegTypeXmm, 25, 16),
-    REG(kX86RegTypeXmm, 26, 16),
-    REG(kX86RegTypeXmm, 27, 16),
-    REG(kX86RegTypeXmm, 28, 16),
-    REG(kX86RegTypeXmm, 29, 16),
-    REG(kX86RegTypeXmm, 30, 16),
-    REG(kX86RegTypeXmm, 31, 16)
-  },
-
-  // Ymm.
-  {
-    REG(kX86RegTypeYmm, 0, 32),
-    REG(kX86RegTypeYmm, 1, 32),
-    REG(kX86RegTypeYmm, 2, 32),
-    REG(kX86RegTypeYmm, 3, 32),
-    REG(kX86RegTypeYmm, 4, 32),
-    REG(kX86RegTypeYmm, 5, 32),
-    REG(kX86RegTypeYmm, 6, 32),
-    REG(kX86RegTypeYmm, 7, 32),
-    REG(kX86RegTypeYmm, 8, 32),
-    REG(kX86RegTypeYmm, 9, 32),
-    REG(kX86RegTypeYmm, 10, 32),
-    REG(kX86RegTypeYmm, 11, 32),
-    REG(kX86RegTypeYmm, 12, 32),
-    REG(kX86RegTypeYmm, 13, 32),
-    REG(kX86RegTypeYmm, 14, 32),
-    REG(kX86RegTypeYmm, 15, 32),
-    REG(kX86RegTypeYmm, 16, 32),
-    REG(kX86RegTypeYmm, 17, 32),
-    REG(kX86RegTypeYmm, 18, 32),
-    REG(kX86RegTypeYmm, 19, 32),
-    REG(kX86RegTypeYmm, 20, 32),
-    REG(kX86RegTypeYmm, 21, 32),
-    REG(kX86RegTypeYmm, 22, 32),
-    REG(kX86RegTypeYmm, 23, 32),
-    REG(kX86RegTypeYmm, 24, 32),
-    REG(kX86RegTypeYmm, 25, 32),
-    REG(kX86RegTypeYmm, 26, 32),
-    REG(kX86RegTypeYmm, 27, 32),
-    REG(kX86RegTypeYmm, 28, 32),
-    REG(kX86RegTypeYmm, 29, 32),
-    REG(kX86RegTypeYmm, 30, 32),
-    REG(kX86RegTypeYmm, 31, 32)
-  },
-
-  // Zmm.
-  {
-    REG(kX86RegTypeZmm, 0, 64),
-    REG(kX86RegTypeZmm, 1, 64),
-    REG(kX86RegTypeZmm, 2, 64),
-    REG(kX86RegTypeZmm, 3, 64),
-    REG(kX86RegTypeZmm, 4, 64),
-    REG(kX86RegTypeZmm, 5, 64),
-    REG(kX86RegTypeZmm, 6, 64),
-    REG(kX86RegTypeZmm, 7, 64),
-    REG(kX86RegTypeZmm, 8, 64),
-    REG(kX86RegTypeZmm, 9, 64),
-    REG(kX86RegTypeZmm, 10, 64),
-    REG(kX86RegTypeZmm, 11, 64),
-    REG(kX86RegTypeZmm, 12, 64),
-    REG(kX86RegTypeZmm, 13, 64),
-    REG(kX86RegTypeZmm, 14, 64),
-    REG(kX86RegTypeZmm, 15, 64),
-    REG(kX86RegTypeZmm, 16, 64),
-    REG(kX86RegTypeZmm, 17, 64),
-    REG(kX86RegTypeZmm, 18, 64),
-    REG(kX86RegTypeZmm, 19, 64),
-    REG(kX86RegTypeZmm, 20, 64),
-    REG(kX86RegTypeZmm, 21, 64),
-    REG(kX86RegTypeZmm, 22, 64),
-    REG(kX86RegTypeZmm, 23, 64),
-    REG(kX86RegTypeZmm, 24, 64),
-    REG(kX86RegTypeZmm, 25, 64),
-    REG(kX86RegTypeZmm, 26, 64),
-    REG(kX86RegTypeZmm, 27, 64),
-    REG(kX86RegTypeZmm, 28, 64),
-    REG(kX86RegTypeZmm, 29, 64),
-    REG(kX86RegTypeZmm, 30, 64),
-    REG(kX86RegTypeZmm, 31, 64)
-  }
+  // --------------------------------------------------------------------------
+  // [Registers]
+  // --------------------------------------------------------------------------
+
+  { ASMJIT_X86_REG_01(X86Reg::kRegRip  , 0) },
+  { ASMJIT_X86_REG_07(X86Reg::kRegSeg  , 0) },
+  { ASMJIT_X86_REG_16(X86Reg::kRegGpbLo, 0) },
+  { ASMJIT_X86_REG_04(X86Reg::kRegGpbHi, 0) },
+  { ASMJIT_X86_REG_16(X86Reg::kRegGpw  , 0) },
+  { ASMJIT_X86_REG_16(X86Reg::kRegGpd  , 0) },
+  { ASMJIT_X86_REG_16(X86Reg::kRegGpq  , 0) },
+  { ASMJIT_X86_REG_08(X86Reg::kRegFp   , 0) },
+  { ASMJIT_X86_REG_08(X86Reg::kRegMm   , 0) },
+  { ASMJIT_X86_REG_08(X86Reg::kRegK    , 0) },
+  { ASMJIT_X86_REG_32(X86Reg::kRegXmm  , 0) },
+  { ASMJIT_X86_REG_32(X86Reg::kRegYmm  , 0) },
+  { ASMJIT_X86_REG_32(X86Reg::kRegZmm  , 0) },
+  { ASMJIT_X86_REG_04(X86Reg::kRegBnd  , 0) },
+  { ASMJIT_X86_REG_16(X86Reg::kRegCr   , 0) },
+  { ASMJIT_X86_REG_16(X86Reg::kRegDr   , 0) }
 };
 
-#undef REG
+#undef ASMJIT_X86_REG_32
+#undef ASMJIT_X86_REG_16
+#undef ASMJIT_X86_REG_08
+#undef ASMJIT_X86_REG_04
+#undef ASMJIT_X86_REG_01
+
+#undef ASMJIT_X86_REG_SIGNATURE
 
 } // asmjit namespace
 
 // [Api-End]
-#include "../apiend.h"
+#include "../asmjit_apiend.h"
 
 // [Guard]
-#endif // ASMJIT_BUILD_X86 || ASMJIT_BUILD_X64
+#endif // ASMJIT_BUILD_X86