JIT compilation on ARM processors (#3517)

* Upgraded to new version of asmjit

* JIT compilation for ARM

* Updated CMake script

CMakeLists.txt

@@ -247,8 +247,8 @@ IF(X86)
 ELSE()
     SET_SOURCE_FILES_PROPERTIES(${CMAKE_SOURCE_DIR}/libraries/sfmt/src/SFMT.cpp PROPERTIES COMPILE_FLAGS "-UHAVE_SSE2")
 ENDIF()
-IF(X86 AND NOT (WIN32 AND OPENMM_BUILD_STATIC_LIB))
-    FILE(GLOB src_files ${CMAKE_CURRENT_SOURCE_DIR}/libraries/asmjit/*/*.cpp)
+IF((X86 OR ARM) AND NOT (WIN32 AND OPENMM_BUILD_STATIC_LIB))
+    FILE(GLOB src_files ${CMAKE_CURRENT_SOURCE_DIR}/libraries/asmjit/asmjit/*/*.cpp)
     FILE(GLOB incl_files ${CMAKE_CURRENT_SOURCE_DIR}/libraries/asmjit/*.h)
     SET(SOURCE_FILES ${SOURCE_FILES} ${src_files})
     SET(SOURCE_INCLUDE_FILES ${SOURCE_INCLUDE_FILES} ${incl_files})

libraries/asmjit/arm.h

-// [AsmJit]
-// Complete x86/x64 JIT and Remote Assembler for C++.
-//
-// [License]
-// Zlib - See LICENSE.md file in the package.
-
-// [Guard]
-#ifndef _ASMJIT_ARM_H
-#define _ASMJIT_ARM_H
-
-// [Dependencies]
-#include "./base.h"
-
-#include "./arm/armassembler.h"
-#include "./arm/armbuilder.h"
-#include "./arm/armcompiler.h"
-#include "./arm/arminst.h"
-#include "./arm/armoperand.h"
-
-// [Guard]
-#endif // _ASMJIT_ARM_H

libraries/asmjit/asmjit.h

-// [AsmJit]
-// Complete x86/x64 JIT and Remote Assembler for C++.
-//
-// [License]
-// Zlib - See LICENSE.md file in the package.
-
-// [Guard]
-#ifndef _ASMJIT_ASMJIT_H
-#define _ASMJIT_ASMJIT_H
-
-// ============================================================================
-// [asmjit_mainpage]
-// ============================================================================
-
-//! \mainpage
-//!
-//! AsmJit - Complete x86/x64 JIT and Remote Assembler for C++.
-//!
-//! Introduction provided by the project page at https://github.com/asmjit/asmjit.
-
-//! \defgroup asmjit_base AsmJit Base API (architecture independent)
-//!
-//! \brief Backend Neutral API.
-
-//! \defgroup asmjit_x86 AsmJit X86/X64 API
-//!
-//! \brief X86/X64 Backend API.
-
-//! \defgroup asmjit_arm AsmJit ARM32/ARM64 API
-//!
-//! \brief ARM32/ARM64 Backend API.
-
-// [Dependencies]
-#include "./base.h"
-
-// [X86/X64]
-#if defined(ASMJIT_BUILD_X86)
-#include "./x86.h"
-#endif // ASMJIT_BUILD_X86
-
-// [ARM32/ARM64]
-#if defined(ASMJIT_BUILD_ARM)
-#include "./arm.h"
-#endif // ASMJIT_BUILD_ARM
-
-// [Guard]
-#endif // _ASMJIT_ASMJIT_H

libraries/asmjit/LICENSE.md → libraries/asmjit/asmjit/LICENSE.md

-Copyright (c) 2008-2017, Petr Kobalicek
+Copyright (c) 2008-2020 The AsmJit Authors
 
 This software is provided 'as-is', without any express or implied
 warranty. In no event will the authors be held liable for any damages

libraries/asmjit/asmjit/a64.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_A64_H_INCLUDED
+#define ASMJIT_A64_H_INCLUDED
+
+//! \addtogroup asmjit_a64
+//!
+//! ### Emitters
+//!
+//!   - \ref a64::Assembler - AArch64 assembler (must read, provides examples).
+//!   - \ref a64::Builder - AArch64 builder.
+//!   - \ref a64::Compiler - AArch64 compiler.
+//!   - \ref a64::Emitter - AArch64 emitter (abstract).
+//!
+//! ### Supported Instructions
+//!
+//!   - Emitters:
+//!     - \ref a64::EmitterExplicitT - Provides all instructions that use explicit
+//!       operands, provides also utility functions. The member functions provided
+//!       are part of all ARM/AArch64 emitters.
+//!
+//!   - Instruction representation:
+//!     - \ref a64::Inst::Id - instruction identifiers.
+//!
+//! ### Register Operands
+//!
+//!   - \ref arm::Reg - Base class for any AArch32/AArch64 register.
+//!     - \ref arm::Gp - General purpose register:
+//!       - \ref arm::GpW - 32-bit register.
+//!       - \ref arm::GpX - 64-bit register.
+//!     - \ref arm::Vec - Vector (SIMD) register:
+//!       - \ref arm::VecB - 8-bit SIMD register (AArch64 only).
+//!       - \ref arm::VecH - 16-bit SIMD register (AArch64 only).
+//!       - \ref arm::VecS - 32-bit SIMD register.
+//!       - \ref arm::VecD - 64-bit SIMD register.
+//!       - \ref arm::VecV - 128-bit SIMD register.
+//!
+//! ### Memory Operands
+//!
+//!   - \ref arm::Mem - AArch32/AArch64 memory operand that provides support for all ARM addressing features
+//!     including base, index, pre/post increment, and ARM-specific shift addressing and index extending.
+//!
+//! ### Other
+//!
+//!   - \ref arm::Shift - Shift operation and value.
+//!   - \ref a64::Utils - Utilities that can help during code generation for AArch64.
+
+#include "./arm.h"
+#include "./arm/a64assembler.h"
+#include "./arm/a64builder.h"
+#include "./arm/a64compiler.h"
+#include "./arm/a64emitter.h"
+#include "./arm/a64globals.h"
+#include "./arm/a64instdb.h"
+#include "./arm/a64operand.h"
+#include "./arm/a64utils.h"
+
+#endif // ASMJIT_A64_H_INCLUDED
+

libraries/asmjit/asmjit/arm.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_H_INCLUDED
+#define ASMJIT_ARM_H_INCLUDED
+
+//! \addtogroup asmjit_arm
+//!
+//! ### Namespaces
+//!
+//!   - \ref arm - arm namespace provides common functionality for both AArch32 and AArch64 backends.
+//!   - \ref a64 - a64 namespace provides support for AArch64 architecture. In addition it includes
+//!     \ref arm namespace, so you can only use a single namespace when targeting AArch64 architecture.
+//!
+//! ### Emitters
+//!
+//!   - AArch64
+//!     - \ref a64::Assembler - AArch64 assembler (must read, provides examples).
+//!     - \ref a64::Builder - AArch64 builder.
+//!     - \ref a64::Compiler - AArch64 compiler.
+//!     - \ref a64::Emitter - AArch64 emitter (abstract).
+//!
+//! ### Supported Instructions
+//!
+//!   - AArch64:
+//!     - Emitters:
+//!       - \ref a64::EmitterExplicitT - Provides all instructions that use explicit operands, provides also
+//!         utility functions. The member functions provided are part of all AArch64 emitters.
+//!     - Instruction representation:
+//!       - \ref a64::Inst::Id - instruction identifiers.
+//!
+//! ### Register Operands
+//!
+//!   - \ref arm::Reg - Base class for any AArch32/AArch64 register.
+//!     - \ref arm::Gp - General purpose register:
+//!       - \ref arm::GpW - 32-bit register.
+//!       - \ref arm::GpX - 64-bit register.
+//!     - \ref arm::Vec - Vector (SIMD) register:
+//!       - \ref arm::VecB - 8-bit SIMD register (AArch64 only).
+//!       - \ref arm::VecH - 16-bit SIMD register (AArch64 only).
+//!       - \ref arm::VecS - 32-bit SIMD register.
+//!       - \ref arm::VecD - 64-bit SIMD register.
+//!       - \ref arm::VecV - 128-bit SIMD register.
+//!
+//! ### Memory Operands
+//!
+//!   - \ref arm::Mem - AArch32/AArch64 memory operand that provides support for all ARM addressing features
+//!     including base, index, pre/post increment, and ARM-specific shift addressing and index extending.
+//!
+//! ### Other
+//!
+//!   - \ref arm::Shift - Shift operation and value (both AArch32 and AArch64).
+//!   - \ref arm::DataType - Data type that is part of an instruction in AArch32 mode.
+//!   - \ref a64::Utils - Utilities that can help during code generation for AArch64.
+
+#include "./core.h"
+#include "./arm/armglobals.h"
+#include "./arm/armoperand.h"
+
+#endif // ASMJIT_ARM_H_INCLUDED

libraries/asmjit/asmjit/arm/a64archtraits_p.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64ARCHTRAITS_P_H_INCLUDED
+#define ASMJIT_ARM_A64ARCHTRAITS_P_H_INCLUDED
+
+#include "../core/archtraits.h"
+#include "../core/misc_p.h"
+#include "../core/type.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \cond INTERNAL
+//! \addtogroup asmjit_a64
+//! \{
+
+static const constexpr ArchTraits a64ArchTraits = {
+  // SP/FP/LR/PC.
+  Gp::kIdSp, Gp::kIdFp, Gp::kIdLr, 0xFF,
+
+  // Reserved.
+  { 0, 0, 0 },
+
+  // HW stack alignment (AArch64 requires stack aligned to 64 bytes).
+  16,
+
+  // Min/max stack offset - byte addressing is the worst, VecQ addressing the best.
+  4095, 65520,
+
+  // Instruction hints [Gp, Vec, ExtraVirt2, ExtraVirt3].
+  {{
+    InstHints::kPushPop,
+    InstHints::kPushPop,
+    InstHints::kNoHints,
+    InstHints::kNoHints
+  }},
+
+  // RegInfo.
+  #define V(index) OperandSignature{arm::RegTraits<RegType(index)>::kSignature}
+  {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }},
+  #undef V
+
+  // RegTypeToTypeId.
+  #define V(index) TypeId(arm::RegTraits<RegType(index)>::kTypeId)
+  {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }},
+  #undef V
+
+  // TypeIdToRegType.
+  #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8)    ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8)   ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16)   ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16)  ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32)   ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32)  ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt64)   ? RegType::kARM_GpX   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt64)  ? RegType::kARM_GpX   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr)  ? RegType::kARM_GpX   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kARM_GpX   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kARM_VecS  : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kARM_VecD  : RegType::kNone)
+  {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }},
+  #undef V
+
+  // Word names of 8-bit, 16-bit, 32-bit, and 64-bit quantities.
+  {
+    ArchTypeNameId::kByte,
+    ArchTypeNameId::kHWord,
+    ArchTypeNameId::kWord,
+    ArchTypeNameId::kXWord
+  }
+};
+
+//! \}
+//! \endcond
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_A64ARCHTRAITS_P_H_INCLUDED

libraries/asmjit/asmjit/arm/a64assembler.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64)
+
+#include "../core/codewriter_p.h"
+#include "../core/cpuinfo.h"
+#include "../core/emitterutils_p.h"
+#include "../core/formatter.h"
+#include "../core/logger.h"
+#include "../core/misc_p.h"
+#include "../core/support.h"
+#include "../arm/armformatter_p.h"
+#include "../arm/a64assembler.h"
+#include "../arm/a64emithelper_p.h"
+#include "../arm/a64instdb_p.h"
+#include "../arm/a64utils.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::Assembler - Cond
+// =====================
+
+static inline uint32_t condCodeToOpcodeCond(uint32_t cond) noexcept {
+  return (uint32_t(cond) - 2u) & 0xFu;
+}
+
+// a64::Assembler - Bits
+// =====================
+
+template<typename T>
+static inline constexpr uint32_t B(const T& index) noexcept { return uint32_t(1u) << uint32_t(index); }
+
+static constexpr uint32_t kSP = Gp::kIdSp;
+static constexpr uint32_t kZR = Gp::kIdZr;
+static constexpr uint32_t kWX = InstDB::kWX;
+
+// a64::Assembler - ShiftOpToLdStOptMap
+// ====================================
+
+// Table that maps ShiftOp to OPT part in LD/ST (register) opcode.
+#define VALUE(index) index == uint32_t(ShiftOp::kUXTW) ? 2u : \
+                     index == uint32_t(ShiftOp::kLSL)  ? 3u : \
+                     index == uint32_t(ShiftOp::kSXTW) ? 6u : \
+                     index == uint32_t(ShiftOp::kSXTX) ? 7u : 0xFF
+static const uint8_t armShiftOpToLdStOptMap[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, 0) };
+#undef VALUE
+
+static inline constexpr uint32_t diff(RegType a, RegType b) noexcept {
+  return uint32_t(a) - uint32_t(b);
+}
+
+// asmjit::a64::Assembler - SizeOp
+// ===============================
+
+//! Struct that contains Size (2 bits), Q flag, and S (scalar) flag. These values
+//! are used to encode Q, Size, and Scalar fields in an opcode.
+struct SizeOp {
+  enum : uint8_t {
+    k128BitShift = 0,
+    kScalarShift = 1,
+    kSizeShift = 2,
+
+    kQ = uint8_t(1u << k128BitShift),
+    kS = uint8_t(1u << kScalarShift),
+
+    k00 = uint8_t(0 << kSizeShift),
+    k01 = uint8_t(1 << kSizeShift),
+    k10 = uint8_t(2 << kSizeShift),
+    k11 = uint8_t(3 << kSizeShift),
+
+    k00Q = k00 | kQ,
+    k01Q = k01 | kQ,
+    k10Q = k10 | kQ,
+    k11Q = k11 | kQ,
+
+    k00S = k00 | kS,
+    k01S = k01 | kS,
+    k10S = k10 | kS,
+    k11S = k11 | kS,
+
+    kInvalid = 0xFFu,
+
+    // Masks used by SizeOpMap.
+    kSzQ = (0x3u << kSizeShift) | kQ,
+    kSzS = (0x3u << kSizeShift) | kS,
+    kSzQS = (0x3u << kSizeShift) | kQ | kS
+  };
+
+  uint8_t value;
+
+  inline bool isValid() const noexcept { return value != kInvalid; }
+  inline void makeInvalid() noexcept { value = kInvalid; }
+
+  inline uint32_t q() const noexcept { return (value >> k128BitShift) & 0x1u; }
+  inline uint32_t qs() const noexcept { return ((value >> k128BitShift) | (value >> kScalarShift)) & 0x1u; }
+  inline uint32_t scalar() const noexcept { return (value >> kScalarShift) & 0x1u; }
+  inline uint32_t size() const noexcept { return (value >> kSizeShift) & 0x3u; }
+
+  inline void decrementSize() noexcept {
+    ASMJIT_ASSERT(size() > 0);
+    value = uint8_t(value - (1u << kSizeShift));
+  }
+};
+
+struct SizeOpTable {
+  enum TableId : uint8_t {
+    kTableBin = 0,
+    kTableAny,
+    kCount
+  };
+
+  // 40 elements for each combination.
+  SizeOp array[(uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB) + 1) * 8];
+};
+
+#define VALUE_BIN(x) { \
+  x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k00  : \
+  x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k00Q : \
+  x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeB   )) ? SizeOp::k00  : \
+  x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeB   )) ? SizeOp::k00Q : SizeOp::kInvalid \
+}
+
+#define VALUE_ANY(x) { \
+  x == (((uint32_t(RegType::kARM_VecB) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k00S : \
+  x == (((uint32_t(RegType::kARM_VecH) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k01S : \
+  x == (((uint32_t(RegType::kARM_VecS) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k10S : \
+  x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k11S : \
+  x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeB   )) ? SizeOp::k00  : \
+  x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeB   )) ? SizeOp::k00Q : \
+  x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeH   )) ? SizeOp::k01  : \
+  x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeH   )) ? SizeOp::k01Q : \
+  x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeS   )) ? SizeOp::k10  : \
+  x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeS   )) ? SizeOp::k10Q : \
+  x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeD   )) ? SizeOp::k11S : \
+  x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeD   )) ? SizeOp::k11Q : SizeOp::kInvalid \
+}
+
+static const SizeOpTable sizeOpTable[SizeOpTable::kCount] = {
+  {{ ASMJIT_LOOKUP_TABLE_40(VALUE_BIN, 0) }},
+  {{ ASMJIT_LOOKUP_TABLE_40(VALUE_ANY, 0) }}
+};
+
+#undef VALUE_ANY
+#undef VALUE_BIN
+
+struct SizeOpMap {
+  uint8_t tableId;
+  uint8_t sizeOpMask;
+  uint16_t acceptMask;
+};
+
+static const constexpr SizeOpMap sizeOpMap[InstDB::kVO_Count] = {
+  { // kVO_V_B:
+    SizeOpTable::kTableBin, SizeOp::kQ   , uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q))
+  },
+
+  { // kVO_V_BH:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q))
+  },
+
+  { // kVO_V_BH_4S:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10Q))
+  },
+
+  { // kVO_V_BHS:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10) | B(SizeOp::k10Q))
+  },
+
+  { // kVO_V_BHS_D2:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k11Q))
+  },
+
+  { // kVO_V_HS:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10) | B(SizeOp::k10Q))
+  },
+
+  { // kVO_V_S:
+    SizeOpTable::kTableAny, SizeOp::kQ   , uint16_t(B(SizeOp::k10) | B(SizeOp::k10Q))
+  },
+
+  { // kVO_V_B8H4:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k01))
+  },
+
+  { // kVO_V_B8H4S2:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k01) | B(SizeOp::k10))
+  },
+
+  { // kVO_V_B8D1:
+    SizeOpTable::kTableAny, SizeOp::kSzQ , uint16_t(B(SizeOp::k00) | B(SizeOp::k11S))
+  },
+
+  { // kVO_V_H4S2:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k01) | B(SizeOp::k10))
+  },
+
+  { // kVO_V_B16:
+    SizeOpTable::kTableBin, SizeOp::kQ   , uint16_t(B(SizeOp::k00Q))
+  },
+
+  { // kVO_V_B16H8:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00Q) | B(SizeOp::k01Q))
+  },
+
+  { // kVO_V_B16H8S4:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00Q) | B(SizeOp::k01Q) | B(SizeOp::k10Q))
+  },
+
+  { // kVO_V_B16D2:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00Q) | B(SizeOp::k11Q))
+  },
+
+  { // kVO_V_H8S4:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k01Q) | B(SizeOp::k10Q))
+  },
+
+  { // kVO_V_S4:
+    SizeOpTable::kTableAny, 0            , uint16_t(B(SizeOp::k10Q))
+  },
+
+  { // kVO_V_D2:
+    SizeOpTable::kTableAny, 0            , uint16_t(B(SizeOp::k11Q))
+  },
+
+  { // kVO_SV_BHS:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k00S) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k01S) | B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k10S))
+  },
+
+  { // kVO_SV_B8H4S2:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00S) | B(SizeOp::k01) | B(SizeOp::k01S) | B(SizeOp::k10) | B(SizeOp::k10S))
+  },
+
+  { // kVO_SV_HS:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k01S) | B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k10S))
+  },
+
+  { // kVO_V_Any:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k11S) | B(SizeOp::k11Q))
+  },
+
+  { // kVO_SV_Any:
+    SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k00S) |
+                                                    B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k01S) |
+                                                    B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k10S) |
+                                                    B(SizeOp::k11) | B(SizeOp::k11Q) | B(SizeOp::k11S))
+  }
+};
+
+static const Operand_& significantSimdOp(const Operand_& o0, const Operand_& o1, uint32_t instFlags) noexcept {
+  return !(instFlags & InstDB::kInstFlagLong) ? o0 : o1;
+}
+
+static inline SizeOp armElementTypeToSizeOp(uint32_t vecOpType, RegType regType, uint32_t elementType) noexcept {
+  // Instruction data or Assembler is wrong if this triggers an assertion failure.
+  ASMJIT_ASSERT(vecOpType < InstDB::kVO_Count);
+  // ElementType uses 3 bits in the operand signature, it should never overflow.
+  ASMJIT_ASSERT(elementType <= 0x7u);
+
+  const SizeOpMap& map = sizeOpMap[vecOpType];
+  const SizeOpTable& table = sizeOpTable[map.tableId];
+
+  size_t index = (Support::min<uint32_t>(diff(regType, RegType::kARM_VecB), diff(RegType::kARM_VecV, RegType::kARM_VecB) + 1) << 3) | elementType;
+  SizeOp op = table.array[index];
+  SizeOp modifiedOp { uint8_t(op.value & map.sizeOpMask) };
+
+  if (!Support::bitTest(map.acceptMask, op.value))
+    modifiedOp.makeInvalid();
+
+  return modifiedOp;
+}
+
+// a64::Assembler - Immediate Encoding Utilities (Integral)
+// ========================================================
+
+using Utils::LogicalImm;
+
+struct HalfWordImm {
+  uint32_t hw;
+  uint32_t inv;
+  uint32_t imm;
+};
+
+struct LMHImm {
+  uint32_t lm;
+  uint32_t h;
+  uint32_t maxRmId;
+};
+
+static inline uint32_t countZeroHalfWords64(uint64_t imm) noexcept {
+  return uint32_t((imm & 0x000000000000FFFFu) == 0) +
+         uint32_t((imm & 0x00000000FFFF0000u) == 0) +
+         uint32_t((imm & 0x0000FFFF00000000u) == 0) +
+         uint32_t((imm & 0xFFFF000000000000u) == 0) ;
+}
+
+static uint32_t encodeMovSequence32(uint32_t out[2], uint32_t imm, uint32_t rd, uint32_t x) noexcept {
+  ASMJIT_ASSERT(rd <= 31);
+
+  uint32_t kMovZ = 0b01010010100000000000000000000000 | (x << 31);
+  uint32_t kMovN = 0b00010010100000000000000000000000;
+  uint32_t kMovK = 0b01110010100000000000000000000000;
+
+  if ((imm & 0xFFFF0000u) == 0x00000000u) {
+    out[0] = kMovZ | (0 << 21) | ((imm & 0xFFFFu) << 5) | rd;
+    return 1;
+  }
+
+  if ((imm & 0xFFFF0000u) == 0xFFFF0000u) {
+    out[0] = kMovN | (0 << 21) | ((~imm & 0xFFFFu) << 5) | rd;
+    return 1;
+  }
+
+  if ((imm & 0x0000FFFFu) == 0x00000000u) {
+    out[0] = kMovZ | (1 << 21) | ((imm >> 16) << 5) | rd;
+    return 1;
+  }
+
+  if ((imm & 0x0000FFFFu) == 0x0000FFFFu) {
+    out[0] = kMovN | (1 << 21) | ((~imm >> 16) << 5) | rd;
+    return 1;
+  }
+
+  out[0] = kMovZ | (0 << 21) | ((imm & 0xFFFFu) << 5) | rd;
+  out[1] = kMovK | (1 << 21) | ((imm     >> 16) << 5) | rd;
+  return 2;
+}
+
+static uint32_t encodeMovSequence64(uint32_t out[4], uint64_t imm, uint32_t rd, uint32_t x) noexcept {
+  ASMJIT_ASSERT(rd <= 31);
+
+  uint32_t kMovZ = 0b11010010100000000000000000000000;
+  uint32_t kMovN = 0b10010010100000000000000000000000;
+  uint32_t kMovK = 0b11110010100000000000000000000000;
+
+  if (imm <= 0xFFFFFFFFu)
+    return encodeMovSequence32(out, uint32_t(imm), rd, x);
+
+  uint32_t zhw = countZeroHalfWords64( imm);
+  uint32_t ohw = countZeroHalfWords64(~imm);
+
+  if (zhw >= ohw) {
+    uint32_t op = kMovZ;
+    uint32_t count = 0;
+
+    for (uint32_t hwIndex = 0; hwIndex < 4; hwIndex++, imm >>= 16) {
+      uint32_t hwImm = uint32_t(imm & 0xFFFFu);
+      if (hwImm == 0)
+        continue;
+
+      out[count++] = op | (hwIndex << 21) | (hwImm << 5) | rd;
+      op = kMovK;
+    }
+
+    // This should not happen - zero should be handled by encodeMovSequence32().
+    ASMJIT_ASSERT(count > 0);
+
+    return count;
+  }
+  else {
+    uint32_t op = kMovN;
+    uint32_t count = 0;
+    uint32_t negMask = 0xFFFFu;
+
+    for (uint32_t hwIndex = 0; hwIndex < 4; hwIndex++, imm >>= 16) {
+      uint32_t hwImm = uint32_t(imm & 0xFFFFu);
+      if (hwImm == 0xFFFFu)
+        continue;
+
+      out[count++] = op | (hwIndex << 21) | ((hwImm ^ negMask) << 5) | rd;
+      op = kMovK;
+      negMask = 0;
+    }
+
+    if (count == 0) {
+      out[count++] = kMovN | ((0xFFFF ^ negMask) << 5) | rd;
+    }
+
+    return count;
+  }
+}
+
+static inline bool encodeLMH(uint32_t sizeField, uint32_t elementIndex, LMHImm* out) noexcept {
+  if (sizeField != 1 && sizeField != 2)
+    return false;
+
+  uint32_t hShift = 3u - sizeField;
+  uint32_t lmShift = sizeField - 1u;
+  uint32_t maxElementIndex = 15u >> sizeField;
+
+  out->h = elementIndex >> hShift;
+  out->lm = (elementIndex << lmShift) & 0x3u;
+  out->maxRmId = (8u << sizeField) - 1;
+
+  return elementIndex <= maxElementIndex;
+}
+
+// [.......A|B.......|.......C|D.......|.......E|F.......|.......G|H.......]
+static inline uint32_t encodeImm64ByteMaskToImm8(uint64_t imm) noexcept {
+  return uint32_t(((imm >> (7  - 0)) & 0b00000011) | // [.......G|H.......]
+                  ((imm >> (23 - 2)) & 0b00001100) | // [.......E|F.......]
+                  ((imm >> (39 - 4)) & 0b00110000) | // [.......C|D.......]
+                  ((imm >> (55 - 6)) & 0b11000000)); // [.......A|B.......]
+}
+
+// a64::Assembler - Opcode
+// =======================
+
+//! Helper class to store and manipulate ARM opcode.
+struct Opcode {
+  uint32_t v;
+
+  enum Bits : uint32_t {
+    kN = (1u << 22),
+    kQ = (1u << 30),
+    kX = (1u << 31)
+  };
+
+  // --------------------------------------------------------------------------
+  // [Opcode Builder]
+  // --------------------------------------------------------------------------
+
+  inline uint32_t get() const noexcept { return v; }
+  inline void reset(uint32_t value) noexcept { v = value; }
+
+  inline bool hasQ() const noexcept { return (v & kQ) != 0; }
+  inline bool hasX() const noexcept { return (v & kX) != 0; }
+
+  template<typename T>
+  inline Opcode& addImm(T value, uint32_t bitIndex) noexcept { return operator|=(uint32_t(value) << bitIndex); }
+
+  template<typename T>
+  inline Opcode& xorImm(T value, uint32_t bitIndex) noexcept { return operator^=(uint32_t(value) << bitIndex); }
+
+  template<typename T, typename Condition>
+  inline Opcode& addIf(T value, const Condition& condition) noexcept { return operator|=(condition ? uint32_t(value) : uint32_t(0)); }
+
+  inline Opcode& addLogicalImm(const LogicalImm& logicalImm) noexcept {
+    addImm(logicalImm.n, 22);
+    addImm(logicalImm.r, 16);
+    addImm(logicalImm.s, 10);
+    return *this;
+  }
+
+  inline Opcode& addReg(uint32_t id, uint32_t bitIndex) noexcept { return operator|=((id & 31u) << bitIndex); }
+  inline Opcode& addReg(const Operand_& op, uint32_t bitIndex) noexcept { return addReg(op.id(), bitIndex); }
+
+  inline Opcode& operator=(uint32_t x) noexcept { v = x; return *this; }
+  inline Opcode& operator&=(uint32_t x) noexcept { v &= x; return *this; }
+  inline Opcode& operator|=(uint32_t x) noexcept { v |= x; return *this; }
+  inline Opcode& operator^=(uint32_t x) noexcept { v ^= x; return *this; }
+
+  inline uint32_t operator&(uint32_t x) const noexcept { return v & x; }
+  inline uint32_t operator|(uint32_t x) const noexcept { return v | x; }
+  inline uint32_t operator^(uint32_t x) const noexcept { return v ^ x; }
+};
+
+// a64::Assembler - Signature Utilities
+// ====================================
+
+// TODO: [ARM] Deprecate matchSignature.
+static inline bool matchSignature(const Operand_& o0, const Operand_& o1, uint32_t instFlags) noexcept {
+  if (!(instFlags & (InstDB::kInstFlagLong | InstDB::kInstFlagNarrow)))
+    return o0.signature() == o1.signature();
+
+  // TODO: [ARM] Something smart to validate this.
+  return true;
+}
+
+static inline bool matchSignature(const Operand_& o0, const Operand_& o1, const Operand_& o2, uint32_t instFlags) noexcept {
+  return matchSignature(o0, o1, instFlags) && o1.signature() == o2.signature();
+}
+
+static inline bool matchSignature(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, uint32_t instFlags) noexcept {
+  return matchSignature(o0, o1, instFlags) && o1.signature() == o2.signature() && o2.signature() == o3.signature();;
+}
+
+// Memory must be either:
+// 1. Absolute address, which will be converted to relative.
+// 2. Relative displacement (Label).
+// 3. Base register + either offset or index.
+static inline bool armCheckMemBaseIndexRel(const Mem& mem) noexcept {
+  // Allowed base types (Nothing, Label, and GpX).
+  constexpr uint32_t kBaseMask  = B(0) |
+                                  B(RegType::kLabelTag) |
+                                  B(RegType::kARM_GpX);
+
+  // Allowed index types (Nothing, GpW, and GpX).
+  constexpr uint32_t kIndexMask = B(0) |
+                                  B(RegType::kARM_GpW) |
+                                  B(RegType::kARM_GpX) ;
+
+  RegType baseType = mem.baseType();
+  RegType indexType = mem.indexType();
+
+  if (!Support::bitTest(kBaseMask, baseType))
+    return false;
+
+  if (baseType > RegType::kLabelTag) {
+    // Index allows either GpW or GpX.
+    if (!Support::bitTest(kIndexMask, indexType))
+      return false;
+
+    if (indexType == RegType::kNone)
+      return true;
+    else
+      return !mem.hasOffset();
+  }
+  else {
+    // No index register allowed if this is a PC relative address (literal).
+    return indexType == RegType::kNone;
+  }
+}
+
+struct EncodeFpOpcodeBits {
+  uint32_t sizeMask;
+  uint32_t mask[3];
+};
+
+static inline bool pickFpOpcode(const Vec& reg, uint32_t sOp, uint32_t sHf, uint32_t vOp, uint32_t vHf, Opcode* opcode, uint32_t* szOut) noexcept {
+  static constexpr uint32_t kQBitIndex = 30;
+
+  static const EncodeFpOpcodeBits szBits[InstDB::kHF_Count] = {
+    { B(2) | B(1)       , { 0u                           , 0u, B(22) } },
+    { B(2) | B(1) | B(0), { 0u                           , 0u, 0u    } },
+    { B(2) | B(1) | B(0), { B(23) | B(22)                , 0u, B(22) } },
+    { B(2) | B(1) | B(0), { B(22) | B(20) | B(19)        , 0u, B(22) } },
+    { B(2) | B(1) | B(0), { B(22) | B(21) | B(15) | B(14), 0u, B(22) } },
+    { B(2) | B(1) | B(0), { B(23)                        , 0u, B(22) } }
+  };
+
+  if (!reg.hasElementType()) {
+    // Scalar operation [HSD].
+    uint32_t sz = diff(reg.type(), RegType::kARM_VecH);
+    if (sz > 2u || !Support::bitTest(szBits[sHf].sizeMask, sz))
+      return false;
+
+    opcode->reset(szBits[sHf].mask[sz] ^ sOp);
+    *szOut = sz;
+    return sOp != 0;
+  }
+  else {
+    // Vector operation [HSD].
+    uint32_t q = diff(reg.type(), RegType::kARM_VecD);
+    uint32_t sz = reg.elementType() - Vec::kElementTypeH;
+
+    if (q > 1u || sz > 2u || !Support::bitTest(szBits[vHf].sizeMask, sz))
+      return false;
+
+    opcode->reset(szBits[vHf].mask[sz] ^ (vOp | (q << kQBitIndex)));
+    *szOut = sz;
+    return vOp != 0;
+  }
+}
+
+static inline bool pickFpOpcode(const Vec& reg, uint32_t sOp, uint32_t sHf, uint32_t vOp, uint32_t vHf, Opcode* opcode) noexcept {
+  uint32_t sz;
+  return pickFpOpcode(reg, sOp, sHf, vOp, vHf, opcode, &sz);
+}
+
+// a64::Assembler - Operand Checks
+// ===============================
+
+// Checks whether all operands have the same signature.
+static inline bool checkSignature(const Operand_& o0, const Operand_& o1) noexcept {
+  return o0.signature() == o1.signature();
+}
+
+static inline bool checkSignature(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept {
+  return o0.signature() == o1.signature() &&
+         o1.signature() == o2.signature();
+}
+
+static inline bool checkSignature(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept {
+  return o0.signature() == o1.signature() &&
+         o1.signature() == o2.signature() &&
+         o2.signature() == o3.signature();
+}
+
+// Checks whether the register is GP register of the allowed types.
+//
+// Allowed is a 2-bit mask, where the first bits allows GpW and the second bit
+// allows GpX. These bits are usually stored within the instruction, but could
+// be also hardcoded in the assembler for instructions where GP types are not
+// selectable.
+static inline bool checkGpType(const Operand_& op, uint32_t allowed) noexcept {
+  RegType type = op.as<Reg>().type();
+  return Support::bitTest(allowed << uint32_t(RegType::kARM_GpW), type);
+}
+
+static inline bool checkGpType(const Operand_& op, uint32_t allowed, uint32_t* x) noexcept {
+  // NOTE: We set 'x' to one only when GpW is allowed, otherwise the X is part
+  // of the opcode and we cannot set it. This is why this works without requiring
+  // additional logic.
+  RegType type = op.as<Reg>().type();
+  *x = diff(type, RegType::kARM_GpW) & allowed;
+  return Support::bitTest(allowed << uint32_t(RegType::kARM_GpW), type);
+}
+
+static inline bool checkGpType(const Operand_& o0, const Operand_& o1, uint32_t allowed, uint32_t* x) noexcept {
+  return checkGpType(o0, allowed, x) && checkSignature(o0, o1);
+}
+
+static inline bool checkGpType(const Operand_& o0, const Operand_& o1, const Operand_& o2, uint32_t allowed, uint32_t* x) noexcept {
+  return checkGpType(o0, allowed, x) && checkSignature(o0, o1, o2);
+}
+
+static inline bool checkGpId(const Operand_& op, uint32_t hiId = kZR) noexcept {
+  uint32_t id = op.as<Reg>().id();
+  return id < 31u || id == hiId;
+}
+
+static inline bool checkGpId(const Operand_& o0, const Operand_& o1, uint32_t hiId = kZR) noexcept {
+  uint32_t id0 = o0.as<Reg>().id();
+  uint32_t id1 = o1.as<Reg>().id();
+
+  return (id0 < 31u || id0 == hiId) && (id1 < 31u || id1 == hiId);
+}
+
+static inline bool checkGpId(const Operand_& o0, const Operand_& o1, const Operand_& o2, uint32_t hiId = kZR) noexcept {
+  uint32_t id0 = o0.as<Reg>().id();
+  uint32_t id1 = o1.as<Reg>().id();
+  uint32_t id2 = o2.as<Reg>().id();
+
+  return (id0 < 31u || id0 == hiId) && (id1 < 31u || id1 == hiId) && (id2 < 31u || id2 == hiId);
+}
+
+static inline bool checkVecId(const Operand_& op) noexcept {
+  uint32_t id = op.as<Reg>().id();
+  return id <= 31u;
+}
+
+static inline bool checkVecId(const Operand_& o0, const Operand_& o1) noexcept {
+  uint32_t id0 = o0.as<Reg>().id();
+  uint32_t id1 = o1.as<Reg>().id();
+
+  return (id0 | id1) <= 31u;
+}
+
+/* Unused at the moment.
+static inline bool checkVecId(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept {
+  uint32_t id0 = o0.as<Reg>().id();
+  uint32_t id1 = o1.as<Reg>().id();
+  uint32_t id2 = o2.as<Reg>().id();
+
+  return (id0 | id1 | id2) <= 31u;
+}
+
+static inline bool checkVecId(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept {
+  uint32_t id0 = o0.as<Reg>().id();
+  uint32_t id1 = o1.as<Reg>().id();
+  uint32_t id2 = o2.as<Reg>().id();
+  uint32_t id3 = o3.as<Reg>().id();
+
+  return (id0 | id1 | id2 | id3) <= 31u;
+}
+*/
+
+static inline bool checkMemBase(const Mem& mem) noexcept {
+  return mem.baseType() == RegType::kARM_GpX && mem.baseId() <= 31;
+}
+
+static inline bool checkEven(const Operand_& o0, const Operand_& o1) noexcept {
+  return ((o0.id() | o1.id()) & 1) == 0;
+}
+
+static inline bool checkConsecutive(const Operand_& o0, const Operand_& o1) noexcept {
+  return ((o0.id() + 1u) & 0x1Fu) == o1.id();
+}
+
+static inline bool checkConsecutive(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept {
+  return ((o0.id() + 1u) & 0x1Fu) == o1.id() &&
+         ((o0.id() + 2u) & 0x1Fu) == o2.id();
+}
+
+static inline bool checkConsecutive(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept {
+  return ((o0.id() + 1u) & 0x1Fu) == o1.id() &&
+         ((o0.id() + 2u) & 0x1Fu) == o2.id() &&
+         ((o0.id() + 3u) & 0x1Fu) == o3.id();
+}
+
+// a64::Assembler - CheckReg
+// =========================
+
+#define V(index) (index == uint32_t(RegType::kARM_GpW)  ? Gp::kIdZr :  \
+                  index == uint32_t(RegType::kARM_GpX)  ? Gp::kIdZr :  \
+                  index == uint32_t(RegType::kARM_VecB) ? 31u       :  \
+                  index == uint32_t(RegType::kARM_VecH) ? 31u       :  \
+                  index == uint32_t(RegType::kARM_VecS) ? 31u       :  \
+                  index == uint32_t(RegType::kARM_VecD) ? 31u       :  \
+                  index == uint32_t(RegType::kARM_VecV) ? 31u       : 0)
+static const Support::Array<uint8_t, 32> commonHiRegIdOfType = {{
+  ASMJIT_LOOKUP_TABLE_32(V, 0)
+}};
+#undef V
+
+static inline bool checkValidRegs(const Operand_& o0) noexcept {
+  return ((o0.id() < 31) | (o0.id() == commonHiRegIdOfType[o0.as<Reg>().type()]));
+}
+
+static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1) noexcept {
+  return ((o0.id() < 31) | (o0.id() == commonHiRegIdOfType[o0.as<Reg>().type()])) &
+         ((o1.id() < 31) | (o1.id() == commonHiRegIdOfType[o1.as<Reg>().type()])) ;
+}
+
+static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept {
+  return ((o0.id() < 31) | (o0.id() == commonHiRegIdOfType[o0.as<Reg>().type()])) &
+         ((o1.id() < 31) | (o1.id() == commonHiRegIdOfType[o1.as<Reg>().type()])) &
+         ((o2.id() < 31) | (o2.id() == commonHiRegIdOfType[o2.as<Reg>().type()])) ;
+}
+
+static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept {
+  return ((o0.id() < 31) | (o0.id() == commonHiRegIdOfType[o0.as<Reg>().type()])) &
+         ((o1.id() < 31) | (o1.id() == commonHiRegIdOfType[o1.as<Reg>().type()])) &
+         ((o2.id() < 31) | (o2.id() == commonHiRegIdOfType[o2.as<Reg>().type()])) &
+         ((o3.id() < 31) | (o3.id() == commonHiRegIdOfType[o3.as<Reg>().type()])) ;
+}
+
+// a64::Assembler - Construction & Destruction
+// ===========================================
+
+Assembler::Assembler(CodeHolder* code) noexcept : BaseAssembler() {
+  _archMask = uint64_t(1) << uint32_t(Arch::kAArch64);
+  assignEmitterFuncs(this);
+
+  if (code)
+    code->attach(this);
+}
+
+Assembler::~Assembler() noexcept {}
+
+// a64::Assembler - Emit
+// =====================
+
+#define ENC_OPS1(OP0) \
+  (uint32_t(OperandType::k##OP0))
+
+#define ENC_OPS2(OP0, OP1) \
+  (uint32_t(OperandType::k##OP0) + \
+  (uint32_t(OperandType::k##OP1) << 3))
+
+#define ENC_OPS3(OP0, OP1, OP2) \
+  (uint32_t(OperandType::k##OP0) + \
+  (uint32_t(OperandType::k##OP1) << 3) + \
+  (uint32_t(OperandType::k##OP2) << 6))
+
+#define ENC_OPS4(OP0, OP1, OP2, OP3) \
+  (uint32_t(OperandType::k##OP0) + \
+  (uint32_t(OperandType::k##OP1) << 3) + \
+  (uint32_t(OperandType::k##OP2) << 6) + \
+  (uint32_t(OperandType::k##OP3) << 9))
+
+Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) {
+  // Logging/Validation/Error.
+  constexpr InstOptions kRequiresSpecialHandling = InstOptions::kReserved;
+
+  Error err;
+  CodeWriter writer(this);
+
+  // Combine all instruction options and also check whether the instruction
+  // is valid. All options that require special handling (including invalid
+  // instruction) are handled by the next branch.
+  InstOptions options = InstOptions(instId - 1 >= Inst::_kIdCount - 1) | InstOptions((size_t)(_bufferEnd - writer.cursor()) < 4) | instOptions() | forcedInstOptions();
+
+  CondCode instCC = BaseInst::extractARMCondCode(instId);
+  instId = instId & uint32_t(InstIdParts::kRealId);
+
+  if (instId >= Inst::_kIdCount)
+    instId = 0;
+
+  const InstDB::InstInfo* instInfo = &InstDB::_instInfoTable[instId];
+  uint32_t encodingIndex = instInfo->_encodingDataIndex;
+
+  Opcode opcode;
+  uint32_t isign4;
+  uint32_t instFlags;
+
+  const Operand_& o3 = opExt[EmitterUtils::kOp3];
+  const Operand_* rmRel = nullptr;
+
+  uint32_t multipleOpData[4];
+  uint32_t multipleOpCount;
+
+  // These are only used when instruction uses a relative displacement.
+  OffsetFormat offsetFormat;     // Offset format.
+  uint64_t offsetValue;          // Offset value (if known).
+
+  if (ASMJIT_UNLIKELY(Support::test(options, kRequiresSpecialHandling))) {
+    if (ASMJIT_UNLIKELY(!_code))
+      return reportError(DebugUtils::errored(kErrorNotInitialized));
+
+    // Unknown instruction.
+    if (ASMJIT_UNLIKELY(instId == 0))
+      goto InvalidInstruction;
+
+    // Condition code can only be used with 'B' instruction.
+    if (ASMJIT_UNLIKELY(instCC != CondCode::kAL && instId != Inst::kIdB))
+      goto InvalidInstruction;
+
+    // Grow request, happens rarely.
+    err = writer.ensureSpace(this, 4);
+    if (ASMJIT_UNLIKELY(err))
+      goto Failed;
+
+#ifndef ASMJIT_NO_VALIDATION
+    // Strict validation.
+    if (hasDiagnosticOption(DiagnosticOptions::kValidateAssembler)) {
+      Operand_ opArray[Globals::kMaxOpCount];
+      EmitterUtils::opArrayFromEmitArgs(opArray, o0, o1, o2, opExt);
+
+      err = _funcs.validate(arch(), BaseInst(instId, options, _extraReg), opArray, Globals::kMaxOpCount, ValidationFlags::kNone);
+      if (ASMJIT_UNLIKELY(err))
+        goto Failed;
+    }
+#endif
+  }
+
+  // Signature of the first 4 operands.
+  isign4 = (uint32_t(o0.opType())     ) +
+           (uint32_t(o1.opType()) << 3) +
+           (uint32_t(o2.opType()) << 6) +
+           (uint32_t(o3.opType()) << 9);
+  instFlags = instInfo->flags();
+
+  switch (instInfo->_encoding) {
+    // ------------------------------------------------------------------------
+    // [Base - Universal]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseOp: {
+      const InstDB::EncodingData::BaseOp& opData = InstDB::EncodingData::baseOp[encodingIndex];
+
+      if (isign4 == 0) {
+        opcode.reset(opData.opcode);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseOpImm: {
+      const InstDB::EncodingData::BaseOpImm& opData = InstDB::EncodingData::baseOpImm[encodingIndex];
+
+      if (isign4 == ENC_OPS1(Imm)) {
+        uint64_t imm = o0.as<Imm>().valueAs<uint64_t>();
+        uint32_t immMax = 1u << opData.immBits;
+
+        if (imm >= immMax)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(imm, opData.immOffset);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseR: {
+      const InstDB::EncodingData::BaseR& opData = InstDB::EncodingData::baseR[encodingIndex];
+
+      if (isign4 == ENC_OPS1(Reg)) {
+        if (!checkGpType(o0, opData.rType))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, opData.rHiId))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode);
+        opcode.addReg(o0, opData.rShift);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseRR: {
+      const InstDB::EncodingData::BaseRR& opData = InstDB::EncodingData::baseRR[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        uint32_t x;
+        if (!checkGpType(o0, opData.aType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpType(o1, opData.bType))
+          goto InvalidInstruction;
+
+        if (opData.uniform && !checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, opData.aHiId))
+          goto InvalidPhysId;
+
+        if (!checkGpId(o1, opData.bHiId))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addReg(o1, opData.bShift);
+        opcode.addReg(o0, opData.aShift);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseRRR: {
+      const InstDB::EncodingData::BaseRRR& opData = InstDB::EncodingData::baseRRR[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        uint32_t x;
+        if (!checkGpType(o0, opData.aType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpType(o1, opData.bType))
+          goto InvalidInstruction;
+
+        if (!checkGpType(o2, opData.cType))
+          goto InvalidInstruction;
+
+        if (opData.uniform && !checkSignature(o0, o1, o2))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, opData.aHiId))
+          goto InvalidPhysId;
+
+        if (!checkGpId(o1, opData.bHiId))
+          goto InvalidPhysId;
+
+        if (!checkGpId(o2, opData.cHiId))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, 31);
+        opcode.addReg(o2, 16);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseRRRR: {
+      const InstDB::EncodingData::BaseRRRR& opData = InstDB::EncodingData::baseRRRR[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
+        uint32_t x;
+        if (!checkGpType(o0, opData.aType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpType(o1, opData.bType))
+          goto InvalidInstruction;
+
+        if (!checkGpType(o2, opData.cType))
+          goto InvalidInstruction;
+
+        if (!checkGpType(o3, opData.dType))
+          goto InvalidInstruction;
+
+        if (opData.uniform && !checkSignature(o0, o1, o2, o3))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, opData.aHiId))
+          goto InvalidPhysId;
+
+        if (!checkGpId(o1, opData.bHiId))
+          goto InvalidPhysId;
+
+        if (!checkGpId(o2, opData.cHiId))
+          goto InvalidPhysId;
+
+        if (!checkGpId(o3, opData.dHiId))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, 31);
+        opcode.addReg(o2, 16);
+        opcode.addReg(o3, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseRRII: {
+      const InstDB::EncodingData::BaseRRII& opData = InstDB::EncodingData::baseRRII[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+        if (!checkGpType(o0, opData.aType))
+          goto InvalidInstruction;
+
+        if (!checkGpType(o1, opData.bType))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, opData.aHiId))
+          goto InvalidPhysId;
+
+        if (!checkGpId(o1, opData.bHiId))
+          goto InvalidPhysId;
+
+        if (o2.as<Imm>().valueAs<uint64_t>() >= Support::bitMask(opData.aImmSize + opData.aImmDiscardLsb) ||
+            o3.as<Imm>().valueAs<uint64_t>() >= Support::bitMask(opData.bImmSize + opData.bImmDiscardLsb))
+          goto InvalidImmediate;
+
+        uint32_t aImm = o2.as<Imm>().valueAs<uint32_t>() >> opData.aImmDiscardLsb;
+        uint32_t bImm = o3.as<Imm>().valueAs<uint32_t>() >> opData.bImmDiscardLsb;
+
+        if ((aImm << opData.aImmDiscardLsb) != o2.as<Imm>().valueAs<uint32_t>() ||
+            (bImm << opData.bImmDiscardLsb) != o3.as<Imm>().valueAs<uint32_t>())
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(aImm, opData.aImmOffset);
+        opcode.addImm(bImm, opData.bImmOffset);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Base - Mov]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseMov: {
+      // MOV is a pseudo instruction that uses various instructions depending on its signature.
+      uint32_t x = diff(o0.as<Reg>().type(), RegType::kARM_GpW);
+      if (x > 1)
+        goto InvalidInstruction;
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        if (!o0.as<Reg>().isGp())
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        bool hasSP = o0.as<Gp>().isSP() || o1.as<Gp>().isSP();
+        if (hasSP) {
+          // Cannot be combined with ZR.
+          if (!checkGpId(o0, o1, kSP))
+            goto InvalidPhysId;
+
+          // MOV Rd, Rm -> ADD Rd, Rn, #0.
+          opcode.reset(0b00010001000000000000000000000000);
+          opcode.addImm(x, 31);
+          opcode.addReg(o1, 5);
+          opcode.addReg(o0, 0);
+          goto EmitOp;
+        }
+        else {
+          if (!checkGpId(o0, o1, kZR))
+            goto InvalidPhysId;
+
+          // MOV Rd, Rm -> ORR Rd, <ZR>, Rm.
+          opcode.reset(0b00101010000000000000001111100000);
+          opcode.addImm(x, 31);
+          opcode.addReg(o1, 16);
+          opcode.addReg(o0, 0);
+          goto EmitOp;
+        }
+      }
+
+      if (isign4 == ENC_OPS2(Reg, Imm)) {
+        if (!o0.as<Reg>().isGp())
+          goto InvalidInstruction;
+
+        uint64_t immValue = o1.as<Imm>().valueAs<uint64_t>();
+        if (!x)
+          immValue &= 0xFFFFFFFFu;
+
+        // Prefer a single MOVN/MOVZ instruction over a logical instruction.
+        multipleOpCount = encodeMovSequence64(multipleOpData, immValue, o0.id() & 31, x);
+        if (multipleOpCount == 1 && !o0.as<Gp>().isSP()) {
+          opcode.reset(multipleOpData[0]);
+          goto EmitOp;
+        }
+
+        // Logical instructions use 13-bit immediate pattern encoded as N:ImmR:ImmS.
+        LogicalImm logicalImm;
+        if (!o0.as<Gp>().isZR()) {
+          if (Utils::encodeLogicalImm(immValue, x ? 64 : 32, &logicalImm)) {
+            if (!checkGpId(o0, kSP))
+              goto InvalidPhysId;
+
+            opcode.reset(0b00110010000000000000001111100000);
+            opcode.addImm(x, 31);
+            opcode.addLogicalImm(logicalImm);
+            opcode.addReg(o0, 0);
+            goto EmitOp;
+          }
+        }
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        goto EmitOp_Multiple;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseMovKNZ: {
+      const InstDB::EncodingData::BaseMovKNZ& opData = InstDB::EncodingData::baseMovKNZ[encodingIndex];
+
+      uint32_t x = diff(o0.as<Reg>().type(), RegType::kARM_GpW);
+      if (x > 1)
+        goto InvalidInstruction;
+
+      if (!checkGpId(o0, kZR))
+        goto InvalidPhysId;
+
+      opcode.reset(opData.opcode);
+      opcode.addImm(x, 31);
+
+      if (isign4 == ENC_OPS2(Reg, Imm)) {
+        uint64_t imm16 = o1.as<Imm>().valueAs<uint64_t>();
+        if (imm16 > 0xFFFFu)
+          goto InvalidImmediate;
+
+        opcode.addImm(imm16, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Imm, Imm)) {
+        uint64_t imm16 = o1.as<Imm>().valueAs<uint64_t>();
+        uint32_t shiftType = o2.as<Imm>().predicate();
+        uint64_t shiftValue = o2.as<Imm>().valueAs<uint64_t>();
+
+        if (imm16 > 0xFFFFu || shiftValue > 48 || shiftType != uint32_t(ShiftOp::kLSL))
+          goto InvalidImmediate;
+
+        // Convert shift value to 'hw' field.
+        uint32_t hw = uint32_t(shiftValue) >> 4;
+        if ((hw << 4) != uint32_t(shiftValue))
+          goto InvalidImmediate;
+
+        opcode.addImm(hw, 21);
+        opcode.addImm(imm16, 5);
+        opcode.addReg(o0, 0);
+
+        if (!x && hw > 1u)
+          goto InvalidImmediate;
+
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Base - Adr]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseAdr: {
+      const InstDB::EncodingData::BaseAdr& opData = InstDB::EncodingData::baseAdr[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Label) || isign4 == ENC_OPS2(Reg, Imm)) {
+        if (!o0.as<Reg>().isGpX())
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addReg(o0, 0);
+        offsetFormat.resetToImmValue(opData.offsetType, 4, 5, 21, 0);
+
+        if (instId == Inst::kIdAdrp)
+          offsetFormat._immDiscardLsb = 12;
+
+        rmRel = &o1;
+        goto EmitOp_Rel;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Base - Arithmetic and Logical]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseAddSub: {
+      const InstDB::EncodingData::BaseAddSub& opData = InstDB::EncodingData::baseAddSub[encodingIndex];
+
+      uint32_t x;
+      if (!checkGpType(o0, o1, kWX, &x))
+        goto InvalidInstruction;
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm) || isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+        opcode.reset(uint32_t(opData.immediateOp) << 24);
+
+        // ADD | SUB (immediate) - ZR is not allowed.
+        // ADDS|SUBS (immediate) - ZR allowed in Rd, SP allowed in Rn.
+        uint32_t aHiId = opcode.get() & B(29) ? kZR : kSP;
+        uint32_t bHiId = kSP;
+
+        if (!checkGpId(o0, aHiId) || !checkGpId(o1, bHiId))
+          goto InvalidPhysId;
+
+        // ADD|SUB (immediate) use 12-bit immediate optionally shifted by 'LSL #12'.
+        uint64_t imm = o2.as<Imm>().valueAs<uint64_t>();
+        uint32_t shift = 0;
+
+        if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+          if (o3.as<Imm>().predicate() != uint32_t(ShiftOp::kLSL))
+            goto InvalidImmediate;
+
+          if (o3.as<Imm>().value() != 0 && o3.as<Imm>().value() != 12)
+            goto InvalidImmediate;
+
+          shift = uint32_t(o3.as<Imm>().value() != 0);
+        }
+
+        // Accept immediate value of '0x00XXX000' by setting 'shift' to 12.
+        if (imm > 0xFFFu) {
+          if (shift || (imm & ~uint64_t(0xFFFu << 12)) != 0)
+            goto InvalidImmediate;
+          shift = 1;
+          imm >>= 12;
+        }
+
+        opcode.addImm(x, 31);
+        opcode.addImm(shift, 22);
+        opcode.addImm(imm, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg) || isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+        if (!checkSignature(o1, o2))
+          goto InvalidInstruction;
+
+        uint32_t opSize = x ? 64 : 32;
+        uint64_t shift = 0;
+        uint32_t sType = uint32_t(ShiftOp::kLSL);
+
+        if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+          sType = o3.as<Imm>().predicate();
+          shift = o3.as<Imm>().valueAs<uint64_t>();
+        }
+
+        if (!checkGpId(o2, kZR))
+          goto InvalidPhysId;
+
+        // Shift operation - LSL, LSR, ASR.
+        if (sType <= uint32_t(ShiftOp::kASR)) {
+          bool hasSP = o0.as<Gp>().isSP() || o1.as<Gp>().isSP();
+          if (!hasSP) {
+            if (!checkGpId(o0, o1, kZR))
+              goto InvalidPhysId;
+
+            if (shift >= opSize)
+              goto InvalidImmediate;
+
+            opcode.reset(uint32_t(opData.shiftedOp) << 21);
+            opcode.addImm(x, 31);
+            opcode.addImm(sType, 22);
+            opcode.addReg(o2, 16);
+            opcode.addImm(shift, 10);
+            opcode.addReg(o1, 5);
+            opcode.addReg(o0, 0);
+            goto EmitOp;
+          }
+
+          // SP register can only be used with LSL or Extend.
+          if (sType != uint32_t(ShiftOp::kLSL))
+            goto InvalidImmediate;
+          sType = x ? uint32_t(ShiftOp::kUXTX) : uint32_t(ShiftOp::kUXTW);
+        }
+
+        // Extend operation - UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX.
+        opcode.reset(uint32_t(opData.extendedOp) << 21);
+        sType -= uint32_t(ShiftOp::kUXTB);
+
+        if (sType > 7 || shift > 4)
+          goto InvalidImmediate;
+
+        if (!(opcode.get() & B(29))) {
+          // ADD|SUB (extend) - ZR is not allowed.
+          if (!checkGpId(o0, o1, kSP))
+            goto InvalidPhysId;
+        }
+        else {
+          // ADDS|SUBS (extend) - ZR allowed in Rd, SP allowed in Rn.
+          if (!checkGpId(o0, kZR) || !checkGpId(o1, kSP))
+            goto InvalidPhysId;
+        }
+
+        opcode.addImm(x, 31);
+        opcode.addReg(o2, 16);
+        opcode.addImm(sType, 13);
+        opcode.addImm(shift, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseLogical: {
+      const InstDB::EncodingData::BaseLogical& opData = InstDB::EncodingData::baseLogical[encodingIndex];
+
+      uint32_t x;
+      if (!checkGpType(o0, o1, kWX, &x))
+        goto InvalidInstruction;
+
+      if (!checkSignature(o0, o1))
+        goto InvalidInstruction;
+
+      uint32_t opSize = x ? 64 : 32;
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm) && opData.immediateOp != 0) {
+        opcode.reset(uint32_t(opData.immediateOp) << 23);
+
+        // AND|ANDS|BIC|BICS|ORR|EOR (immediate) uses a LogicalImm format described by N:R:S values.
+        uint64_t immMask = Support::lsbMask<uint64_t>(opSize);
+        uint64_t immValue = o2.as<Imm>().valueAs<uint64_t>();
+
+        if (opData.negateImm)
+          immValue ^= immMask;
+
+        // Logical instructions use 13-bit immediate pattern encoded as N:ImmS:ImmR.
+        LogicalImm logicalImm;
+        if (!Utils::encodeLogicalImm(immValue & immMask, opSize, &logicalImm))
+          goto InvalidImmediate;
+
+        // AND|BIC|ORR|EOR (immediate) can have SP on destination, but ANDS|BICS (immediate) cannot.
+        uint32_t kOpANDS = 0x3 << 29;
+        bool isANDS = (opcode.get() & kOpANDS) == kOpANDS;
+
+        if (!checkGpId(o0, isANDS ? kZR : kSP) || !checkGpId(o1, kZR))
+          goto InvalidPhysId;
+
+        opcode.addImm(x, 31);
+        opcode.addLogicalImm(logicalImm);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      if (!checkSignature(o1, o2))
+        goto InvalidInstruction;
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (!checkGpId(o0, o1, o2, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(uint32_t(opData.shiftedOp) << 21);
+        opcode.addImm(x, 31);
+        opcode.addReg(o2, 16);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+        if (!checkGpId(o0, o1, o2, kZR))
+          goto InvalidPhysId;
+
+        uint32_t shiftType = o3.as<Imm>().predicate();
+        uint64_t opShift = o3.as<Imm>().valueAs<uint64_t>();
+
+        if (shiftType > 0x3 || opShift >= opSize)
+          goto InvalidImmediate;
+
+        opcode.reset(uint32_t(opData.shiftedOp) << 21);
+        opcode.addImm(x, 31);
+        opcode.addImm(shiftType, 22);
+        opcode.addReg(o2, 16);
+        opcode.addImm(opShift, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseCmpCmn: {
+      const InstDB::EncodingData::BaseCmpCmn& opData = InstDB::EncodingData::baseCmpCmn[encodingIndex];
+
+      uint32_t x;
+      if (!checkGpType(o0, kWX, &x))
+        goto InvalidInstruction;
+
+      if (isign4 == ENC_OPS2(Reg, Imm)) {
+        // CMN|CMP (immediate) - ZR is not allowed.
+        if (!checkGpId(o0, kSP))
+          goto InvalidPhysId;
+
+        // CMN|CMP (immediate) use 12-bit immediate optionally shifted by 'LSL #12'.
+        const Imm& imm12 = o1.as<Imm>();
+        uint32_t immShift = 0;
+        uint64_t immValue = imm12.valueAs<uint64_t>();
+
+        if (immValue > 0xFFFu) {
+          if ((immValue & ~uint64_t(0xFFFu << 12)) != 0)
+            goto InvalidImmediate;
+          immShift = 1;
+          immValue >>= 12;
+        }
+
+        opcode.reset(uint32_t(opData.immediateOp) << 24);
+        opcode.addImm(x, 31);
+        opcode.addImm(immShift, 22);
+        opcode.addImm(immValue, 10);
+        opcode.addReg(o0, 5);
+        opcode.addReg(Gp::kIdZr, 0);
+        goto EmitOp;
+      }
+
+      if (isign4 == ENC_OPS2(Reg, Reg) || isign4 == ENC_OPS3(Reg, Reg, Imm)) {
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        uint32_t opSize = x ? 64 : 32;
+        uint32_t sType = 0;
+        uint64_t shift = 0;
+
+        if (isign4 == ENC_OPS3(Reg, Reg, Imm)) {
+          sType = o2.as<Imm>().predicate();
+          shift = o2.as<Imm>().valueAs<uint64_t>();
+        }
+
+        bool hasSP = o0.as<Gp>().isSP() || o1.as<Gp>().isSP();
+
+        // Shift operation - LSL, LSR, ASR.
+        if (sType <= uint32_t(ShiftOp::kASR)) {
+          if (!hasSP) {
+            if (shift >= opSize)
+              goto InvalidImmediate;
+
+            opcode.reset(uint32_t(opData.shiftedOp) << 21);
+            opcode.addImm(x, 31);
+            opcode.addImm(sType, 22);
+            opcode.addReg(o1, 16);
+            opcode.addImm(shift, 10);
+            opcode.addReg(o0, 5);
+            opcode.addReg(Gp::kIdZr, 0);
+            goto EmitOp;
+          }
+
+          // SP register can only be used with LSL or Extend.
+          if (sType != uint32_t(ShiftOp::kLSL))
+            goto InvalidImmediate;
+
+          sType = x ? uint32_t(ShiftOp::kUXTX) : uint32_t(ShiftOp::kUXTW);
+        }
+
+        // Extend operation - UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX.
+        sType -= uint32_t(ShiftOp::kUXTB);
+        if (sType > 7 || shift > 4)
+          goto InvalidImmediate;
+
+        opcode.reset(uint32_t(opData.extendedOp) << 21);
+        opcode.addImm(x, 31);
+        opcode.addReg(o1, 16);
+        opcode.addImm(sType, 13);
+        opcode.addImm(shift, 10);
+        opcode.addReg(o0, 5);
+        opcode.addReg(Gp::kIdZr, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseMvnNeg: {
+      const InstDB::EncodingData::BaseMvnNeg& opData = InstDB::EncodingData::baseMvnNeg[encodingIndex];
+
+      uint32_t x;
+      if (!checkGpType(o0, o1, kWX, &x))
+        goto InvalidInstruction;
+
+      opcode.reset(opData.opcode);
+      opcode.addImm(x, 31);
+      opcode.addReg(o1, 16);
+      opcode.addReg(o0, 0);
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        goto EmitOp;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm)) {
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        uint32_t opSize = x ? 64 : 32;
+        uint32_t shiftType = o2.as<Imm>().predicate();
+        uint64_t opShift = o2.as<Imm>().valueAs<uint64_t>();
+
+        if (shiftType > uint32_t(ShiftOp::kROR) || opShift >= opSize)
+          goto InvalidImmediate;
+
+        opcode.addImm(shiftType, 22);
+        opcode.addImm(opShift, 10);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseTst: {
+      const InstDB::EncodingData::BaseTst& opData = InstDB::EncodingData::baseTst[encodingIndex];
+
+      uint32_t x;
+      if (!checkGpType(o0, kWX, &x))
+        goto InvalidInstruction;
+
+      uint32_t opSize = x ? 64 : 32;
+
+      if (isign4 == ENC_OPS2(Reg, Imm) && opData.immediateOp != 0) {
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        // TST (immediate) uses a LogicalImm format described by N:R:S values.
+        uint64_t immMask = Support::lsbMask<uint64_t>(opSize);
+        uint64_t immValue = o1.as<Imm>().valueAs<uint64_t>();
+
+        // Logical instructions use 13-bit immediate pattern encoded as N:ImmS:ImmR.
+        LogicalImm logicalImm;
+        if (!Utils::encodeLogicalImm(immValue & immMask, opSize, &logicalImm))
+          goto InvalidImmediate;
+
+        opcode.reset(uint32_t(opData.immediateOp) << 22);
+        opcode.addLogicalImm(logicalImm);
+        opcode.addImm(x, 31);
+        opcode.addReg(o0, 5);
+        opcode.addReg(Gp::kIdZr, 0);
+        goto EmitOp;
+      }
+
+      opcode.reset(uint32_t(opData.shiftedOp) << 21);
+      opcode.addImm(x, 31);
+      opcode.addReg(o1, 16);
+      opcode.addReg(o0, 5);
+      opcode.addReg(Gp::kIdZr, 0);
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        goto EmitOp;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm)) {
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        uint32_t shiftType = o2.as<Imm>().predicate();
+        uint64_t opShift = o2.as<Imm>().valueAs<uint64_t>();
+
+        if (shiftType > 0x3 || opShift >= opSize)
+          goto InvalidImmediate;
+
+        opcode.addImm(shiftType, 22);
+        opcode.addImm(opShift, 10);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Base - Bit Manipulation]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseBfc: {
+      const InstDB::EncodingData::BaseBfc& opData = InstDB::EncodingData::baseBfc[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Imm, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0))
+          goto InvalidPhysId;
+
+        uint64_t lsb = o1.as<Imm>().valueAs<uint64_t>();
+        uint64_t width = o2.as<Imm>().valueAs<uint64_t>();
+        uint32_t opSize = x ? 64 : 32;
+
+        if (lsb >= opSize || width == 0 || width > opSize)
+          goto InvalidImmediate;
+
+        uint32_t lsb32 = Support::neg(uint32_t(lsb)) & (opSize - 1);
+        uint32_t width32 = uint32_t(width) - 1;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addImm(x, 22);
+        opcode.addImm(lsb32, 16);
+        opcode.addImm(width32, 10);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseBfi: {
+      const InstDB::EncodingData::BaseBfi& opData = InstDB::EncodingData::baseBfi[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1))
+          goto InvalidPhysId;
+
+        uint64_t lsb = o2.as<Imm>().valueAs<uint64_t>();
+        uint64_t width = o3.as<Imm>().valueAs<uint64_t>();
+        uint32_t opSize = x ? 64 : 32;
+
+        if (lsb >= opSize || width == 0 || width > opSize)
+          goto InvalidImmediate;
+
+        uint32_t lImm = Support::neg(uint32_t(lsb)) & (opSize - 1);
+        uint32_t wImm = uint32_t(width) - 1;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addImm(x, 22);
+        opcode.addImm(lImm, 16);
+        opcode.addImm(wImm, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseBfm: {
+      const InstDB::EncodingData::BaseBfm& opData = InstDB::EncodingData::baseBfm[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1))
+          goto InvalidPhysId;
+
+        uint64_t immR = o2.as<Imm>().valueAs<uint64_t>();
+        uint64_t immS = o3.as<Imm>().valueAs<uint64_t>();
+        uint32_t opSize = x ? 64 : 32;
+
+        if ((immR | immS) >= opSize)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addImm(x, 22);
+        opcode.addImm(immR, 16);
+        opcode.addImm(immS, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseBfx: {
+      const InstDB::EncodingData::BaseBfx& opData = InstDB::EncodingData::baseBfx[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1))
+          goto InvalidPhysId;
+
+        uint64_t lsb = o2.as<Imm>().valueAs<uint64_t>();
+        uint64_t width = o3.as<Imm>().valueAs<uint64_t>();
+        uint32_t opSize = x ? 64 : 32;
+
+        if (lsb >= opSize || width == 0 || width > opSize)
+          goto InvalidImmediate;
+
+        uint32_t lsb32 = uint32_t(lsb);
+        uint32_t width32 = lsb32 + uint32_t(width) - 1u;
+
+        if (width32 >= opSize)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addImm(x, 22);
+        opcode.addImm(lsb32, 16);
+        opcode.addImm(width32, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseExtend: {
+      const InstDB::EncodingData::BaseExtend& opData = InstDB::EncodingData::baseExtend[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        uint32_t x;
+        if (!checkGpType(o0, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!o1.as<Reg>().isGpW())
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, 31);
+        opcode.addImm(x, 22);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseExtract: {
+      const InstDB::EncodingData::BaseExtract& opData = InstDB::EncodingData::baseExtract[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1, o2))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, o2))
+          goto InvalidPhysId;
+
+        uint64_t lsb = o3.as<Imm>().valueAs<uint64_t>();
+        uint32_t opSize = x ? 64 : 32;
+
+        if (lsb >= opSize)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addImm(x, 22);
+        opcode.addReg(o2, 16);
+        opcode.addImm(lsb, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseRev: {
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        uint32_t x;
+        if (!checkGpType(o0, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1))
+          goto InvalidPhysId;
+
+        opcode.reset(0b01011010110000000000100000000000);
+        opcode.addImm(x, 31);
+        opcode.addImm(x, 10);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseShift: {
+      const InstDB::EncodingData::BaseShift& opData = InstDB::EncodingData::baseShift[encodingIndex];
+
+      uint32_t x;
+      if (!checkGpType(o0, kWX, &x))
+        goto InvalidInstruction;
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (!checkSignature(o0, o1, o2))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, o2, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.registerOp());
+        opcode.addImm(x, 31);
+        opcode.addReg(o2, 16);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm) && opData.immediateOp()) {
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        uint64_t immR = o2.as<Imm>().valueAs<uint64_t>();
+        uint32_t opSize = x ? 64 : 32;
+
+        if (immR >= opSize)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.immediateOp());
+        opcode.addImm(x, 31);
+        opcode.addImm(x, 22);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+
+        if (opcode.get() & B(10)) {
+          // ASR and LSR (immediate) has the same logic.
+          opcode.addImm(x, 15);
+          opcode.addImm(immR, 16);
+          goto EmitOp;
+        }
+
+        if (opData.ror == 0) {
+          // LSL (immediate) is an alias to UBFM
+          uint32_t ubfmImmR = Support::neg(uint32_t(immR)) & (opSize - 1);
+          uint32_t ubfmImmS = opSize - 1 - uint32_t(immR);
+
+          opcode.addImm(ubfmImmR, 16);
+          opcode.addImm(ubfmImmS, 10);
+          goto EmitOp;
+        }
+        else {
+          // ROR (immediate) is an alias to EXTR.
+          opcode.addImm(immR, 10);
+          opcode.addReg(o1, 16);
+          goto EmitOp;
+        }
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Base - Conditionals]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseCCmp: {
+      const InstDB::EncodingData::BaseCCmp& opData = InstDB::EncodingData::baseCCmp[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm) || isign4 == ENC_OPS4(Reg, Imm, Imm, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        uint64_t nzcv = o2.as<Imm>().valueAs<uint64_t>();
+        uint64_t cond = o3.as<Imm>().valueAs<uint64_t>();
+
+        if ((nzcv | cond) > 0xFu)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addImm(condCodeToOpcodeCond(uint32_t(cond)), 12);
+        opcode.addImm(nzcv, 0);
+
+        if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+          // CCMN|CCMP (register) form.
+          if (!checkSignature(o0, o1))
+            goto InvalidInstruction;
+
+          if (!checkGpId(o1, kZR))
+            goto InvalidPhysId;
+
+          opcode.addReg(o1, 16);
+          opcode.addReg(o0, 5);
+          goto EmitOp;
+        }
+        else {
+          // CCMN|CCMP (immediate) form.
+          uint64_t imm5 = o1.as<Imm>().valueAs<uint64_t>();
+          if (imm5 > 0x1F)
+            goto InvalidImmediate;
+
+          opcode.addImm(1, 11);
+          opcode.addImm(imm5, 16);
+          opcode.addReg(o0, 5);
+          goto EmitOp;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseCInc: {
+      const InstDB::EncodingData::BaseCInc& opData = InstDB::EncodingData::baseCInc[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, o1, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        uint64_t cond = o2.as<Imm>().valueAs<uint64_t>();
+        if (cond - 2u > 0xEu)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addReg(o1, 16);
+        opcode.addImm(condCodeToOpcodeCond(uint32_t(cond)) ^ 1u, 12);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseCSel: {
+      const InstDB::EncodingData::BaseCSel& opData = InstDB::EncodingData::baseCSel[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, o1, o2, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, o2, kZR))
+          goto InvalidPhysId;
+
+        uint64_t cond = o3.as<Imm>().valueAs<uint64_t>();
+        if (cond > 0xFu)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addReg(o2, 16);
+        opcode.addImm(condCodeToOpcodeCond(uint32_t(cond)), 12);
+        opcode.addReg(o1, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseCSet: {
+      const InstDB::EncodingData::BaseCSet& opData = InstDB::EncodingData::baseCSet[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, InstDB::kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        uint64_t cond = o1.as<Imm>().valueAs<uint64_t>();
+        if (cond - 2u >= 0xEu)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addImm(condCodeToOpcodeCond(uint32_t(cond)) ^ 1u, 12);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Base - Special]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseAtDcIcTlbi: {
+      const InstDB::EncodingData::BaseAtDcIcTlbi& opData = InstDB::EncodingData::baseAtDcIcTlbi[encodingIndex];
+
+      if (isign4 == ENC_OPS1(Imm) || isign4 == ENC_OPS2(Imm, Reg)) {
+        if (opData.mandatoryReg && isign4 != ENC_OPS2(Imm, Reg))
+          goto InvalidInstruction;
+
+        if (o0.as<Imm>().valueAs<uint64_t>() > 0x7FFFu)
+          goto InvalidImmediate;
+
+        uint32_t imm = o0.as<Imm>().valueAs<uint32_t>();
+        if ((imm & opData.immVerifyMask) != opData.immVerifyData)
+          goto InvalidImmediate;
+
+        uint32_t rt = 31;
+        if (o1.isReg()) {
+          if (!o1.as<Reg>().isGpX())
+            goto InvalidInstruction;
+
+          if (!checkGpId(o1, kZR))
+            goto InvalidPhysId;
+
+          rt = o1.id() & 31;
+        }
+
+        opcode.reset(0b11010101000010000000000000000000);
+        opcode.addImm(imm, 5);
+        opcode.addReg(rt, 0);
+        goto EmitOp;
+      }
+      break;
+    }
+
+    case InstDB::kEncodingBaseMrs: {
+      if (isign4 == ENC_OPS2(Reg, Imm)) {
+        if (!o0.as<Reg>().isGpX())
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        if (o1.as<Imm>().valueAs<uint64_t>() > 0xFFFFu)
+          goto InvalidImmediate;
+
+        uint32_t imm = o1.as<Imm>().valueAs<uint32_t>();
+        if (!(imm & B(15)))
+          goto InvalidImmediate;
+
+        opcode.reset(0b11010101001100000000000000000000);
+        opcode.addImm(imm, 5);
+        opcode.addReg(o0, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseMsr: {
+      if (isign4 == ENC_OPS2(Imm, Reg)) {
+        if (!o1.as<Reg>().isGpX())
+          goto InvalidInstruction;
+
+        if (o0.as<Imm>().valueAs<uint64_t>() > 0xFFFFu)
+          goto InvalidImmediate;
+
+        uint32_t imm = o0.as<Imm>().valueAs<uint32_t>();
+        if (!(imm & B(15)))
+          goto InvalidImmediate;
+
+        if (!checkGpId(o1, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(0b11010101000100000000000000000000);
+        opcode.addImm(imm, 5);
+        opcode.addReg(o1, 0);
+        goto EmitOp;
+      }
+
+      if (isign4 == ENC_OPS2(Imm, Imm)) {
+        if (o0.as<Imm>().valueAs<uint64_t>() > 0x1Fu)
+          goto InvalidImmediate;
+
+        if (o1.as<Imm>().valueAs<uint64_t>() > 0xFu)
+          goto InvalidImmediate;
+
+        uint32_t op = o0.as<Imm>().valueAs<uint32_t>();
+        uint32_t cRm = o1.as<Imm>().valueAs<uint32_t>();
+
+        uint32_t op1 = uint32_t(op) >> 3;
+        uint32_t op2 = uint32_t(op) & 0x7u;
+
+        opcode.reset(0b11010101000000000100000000011111);
+        opcode.addImm(op1, 16);
+        opcode.addImm(cRm, 8);
+        opcode.addImm(op2, 5);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseSys: {
+      if (isign4 == ENC_OPS4(Imm, Imm, Imm, Imm)) {
+        if (o0.as<Imm>().valueAs<uint64_t>() > 0x7u ||
+            o1.as<Imm>().valueAs<uint64_t>() > 0xFu ||
+            o2.as<Imm>().valueAs<uint64_t>() > 0xFu ||
+            o3.as<Imm>().valueAs<uint64_t>() > 0x7u)
+          goto InvalidImmediate;
+
+        uint32_t op1 = o0.as<Imm>().valueAs<uint32_t>();
+        uint32_t cRn = o1.as<Imm>().valueAs<uint32_t>();
+        uint32_t cRm = o2.as<Imm>().valueAs<uint32_t>();
+        uint32_t op2 = o3.as<Imm>().valueAs<uint32_t>();
+        uint32_t rt = 31;
+
+        const Operand_& o4 = opExt[EmitterUtils::kOp4];
+        if (o4.isReg()) {
+          if (!o4.as<Reg>().isGpX())
+            goto InvalidInstruction;
+
+          if (!checkGpId(o4, kZR))
+            goto InvalidPhysId;
+
+          rt = o4.id() & 31;
+        }
+        else if (!o4.isNone()) {
+          goto InvalidInstruction;
+        }
+
+        opcode.reset(0b11010101000010000000000000000000);
+        opcode.addImm(op1, 16);
+        opcode.addImm(cRn, 12);
+        opcode.addImm(cRm, 8);
+        opcode.addImm(op2, 5);
+        opcode.addImm(rt, 0);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Base - Branch]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseBranchReg: {
+      const InstDB::EncodingData::BaseBranchReg& opData = InstDB::EncodingData::baseBranchReg[encodingIndex];
+
+      if (isign4 == ENC_OPS1(Reg)) {
+        if (!o0.as<Reg>().isGpX())
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode);
+        opcode.addReg(o0, 5);
+        goto EmitOp;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseBranchRel: {
+      const InstDB::EncodingData::BaseBranchRel& opData = InstDB::EncodingData::baseBranchRel[encodingIndex];
+
+      if (isign4 == ENC_OPS1(Label) || isign4 == ENC_OPS1(Imm)) {
+        opcode.reset(opData.opcode);
+        rmRel = &o0;
+
+        if (instCC != CondCode::kAL) {
+          opcode |= B(30);
+          opcode.addImm(condCodeToOpcodeCond(uint32_t(instCC)), 0);
+          offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 5, 19, 2);
+          goto EmitOp_Rel;
+        }
+
+        offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 0, 26, 2);
+        goto EmitOp_Rel;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseBranchCmp: {
+      const InstDB::EncodingData::BaseBranchCmp& opData = InstDB::EncodingData::baseBranchCmp[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Label) || isign4 == ENC_OPS2(Reg, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode);
+        opcode.addImm(x, 31);
+        opcode.addReg(o0, 0);
+        offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 5, 19, 2);
+
+        rmRel = &o1;
+        goto EmitOp_Rel;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseBranchTst: {
+      const InstDB::EncodingData::BaseBranchTst& opData = InstDB::EncodingData::baseBranchTst[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Imm, Label) || isign4 == ENC_OPS3(Reg, Imm, Imm)) {
+        uint32_t x;
+        if (!checkGpType(o0, kWX, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        uint64_t imm = o1.as<Imm>().valueAs<uint64_t>();
+
+        opcode.reset(opData.opcode);
+        if (imm >= 32) {
+          if (!x)
+            goto InvalidImmediate;
+          opcode.addImm(x, 31);
+          imm &= 0x1F;
+        }
+
+        opcode.addReg(o0, 0);
+        opcode.addImm(imm, 19);
+        offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 5, 14, 2);
+
+        rmRel = &o2;
+        goto EmitOp_Rel;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Base - Load / Store]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingBaseLdSt: {
+      const InstDB::EncodingData::BaseLdSt& opData = InstDB::EncodingData::baseLdSt[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Mem)) {
+        const Mem& m = o1.as<Mem>();
+        rmRel = &m;
+
+        uint32_t x;
+        if (!checkGpType(o0, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        // Instructions that work with either word or dword have the unsigned
+        // offset shift set to 2 (word), so we set it to 3 (dword) if this is
+        // X version of the instruction.
+        uint32_t xShiftMask = uint32_t(opData.uOffsetShift == 2);
+        uint32_t immShift = uint32_t(opData.uOffsetShift) + (x & xShiftMask);
+
+        if (!armCheckMemBaseIndexRel(m))
+          goto InvalidAddress;
+
+        int64_t offset = m.offset();
+        if (m.hasBaseReg()) {
+          // [Base {Offset | Index}]
+          if (m.hasIndex()) {
+            uint32_t opt = armShiftOpToLdStOptMap[m.predicate()];
+            if (opt == 0xFF)
+              goto InvalidAddress;
+
+            uint32_t shift = m.shift();
+            uint32_t s = shift != 0;
+
+            if (s && shift != immShift)
+              goto InvalidAddressScale;
+
+            opcode.reset(uint32_t(opData.registerOp) << 21);
+            opcode.xorImm(x, opData.xOffset);
+            opcode.addImm(opt, 13);
+            opcode.addImm(s, 12);
+            opcode |= B(11);
+            opcode.addReg(o0, 0);
+            goto EmitOp_MemBaseIndex_Rn5_Rm16;
+          }
+
+          // Makes it easier to work with the offset especially on 32-bit arch.
+          if (!Support::isInt32(offset))
+            goto InvalidDisplacement;
+          int32_t offset32 = int32_t(offset);
+
+          if (m.isPreOrPost()) {
+            if (!Support::isInt9(offset32))
+              goto InvalidDisplacement;
+
+            opcode.reset(uint32_t(opData.prePostOp) << 21);
+            opcode.xorImm(x, opData.xOffset);
+            opcode.addImm(offset32 & 0x1FF, 12);
+            opcode.addImm(m.isPreIndex(), 11);
+            opcode |= B(10);
+            opcode.addReg(o0, 0);
+            goto EmitOp_MemBase_Rn5;
+          }
+          else {
+            uint32_t imm12 = uint32_t(offset32) >> immShift;
+
+            // Alternative form of LDUR/STUR and related instructions as described by AArch64 reference manual:
+            //
+            // If this instruction is not encodable with scaled unsigned offset, try unscaled signed offset.
+            if (!Support::isUInt12(imm12) || (imm12 << immShift) != uint32_t(offset32)) {
+              instId = opData.uAltInstId;
+              instInfo = &InstDB::_instInfoTable[instId];
+              encodingIndex = instInfo->_encodingDataIndex;
+              goto Case_BaseLdurStur;
+            }
+
+            opcode.reset(uint32_t(opData.uOffsetOp) << 22);
+            opcode.xorImm(x, opData.xOffset);
+            opcode.addImm(imm12, 10);
+            opcode.addReg(o0, 0);
+            goto EmitOp_MemBase_Rn5;
+          }
+        }
+        else {
+          if (!opData.literalOp)
+            goto InvalidAddress;
+
+          opcode.reset(uint32_t(opData.literalOp) << 24);
+          opcode.xorImm(x, opData.xOffset);
+          opcode.addReg(o0, 0);
+          offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 5, 19, 2);
+          goto EmitOp_Rel;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseLdpStp: {
+      const InstDB::EncodingData::BaseLdpStp& opData = InstDB::EncodingData::baseLdpStp[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Mem)) {
+        const Mem& m = o2.as<Mem>();
+        rmRel = &m;
+
+        uint32_t x;
+        if (!checkGpType(o0, o1, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        if (m.baseType() != RegType::kARM_GpX || m.hasIndex())
+          goto InvalidAddress;
+
+        if (m.isOffset64Bit())
+          goto InvalidDisplacement;
+
+        uint32_t offsetShift = opData.offsetShift + x;
+        int32_t offset32 = m.offsetLo32() >> offsetShift;
+
+        // Make sure we didn't lose bits by applying the mandatory offset shift.
+        if (uint32_t(offset32) << offsetShift != uint32_t(m.offsetLo32()))
+          goto InvalidDisplacement;
+
+        // Offset is encoded as 7-bit immediate.
+        if (!Support::isInt7(offset32))
+          goto InvalidDisplacement;
+
+        if (m.isPreOrPost() && offset32 != 0) {
+          if (!opData.prePostOp)
+            goto InvalidAddress;
+
+          opcode.reset(uint32_t(opData.prePostOp) << 22);
+          opcode.addImm(m.isPreIndex(), 24);
+        }
+        else {
+          opcode.reset(uint32_t(opData.offsetOp) << 22);
+        }
+
+        opcode.addImm(x, opData.xOffset);
+        opcode.addImm(offset32 & 0x7F, 15);
+        opcode.addReg(o1, 10);
+        opcode.addReg(o0, 0);
+        goto EmitOp_MemBase_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseStx: {
+      const InstDB::EncodingData::BaseStx& opData = InstDB::EncodingData::baseStx[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Mem)) {
+        const Mem& m = o2.as<Mem>();
+        uint32_t x;
+
+        if (!o0.as<Reg>().isGpW() || !checkGpType(o1, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, opData.xOffset);
+        opcode.addReg(o0, 16);
+        opcode.addReg(o1, 0);
+
+        rmRel = &m;
+        goto EmitOp_MemBaseNoImm_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseLdxp: {
+      const InstDB::EncodingData::BaseLdxp& opData = InstDB::EncodingData::baseLdxp[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Mem)) {
+        const Mem& m = o2.as<Mem>();
+        uint32_t x;
+
+        if (!checkGpType(o0, opData.rType, &x) || !checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, opData.xOffset);
+        opcode.addReg(o1, 10);
+        opcode.addReg(o0, 0);
+
+        rmRel = &m;
+        goto EmitOp_MemBaseNoImm_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseStxp: {
+      const InstDB::EncodingData::BaseStxp& opData = InstDB::EncodingData::baseStxp[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Mem)) {
+        const Mem& m = o3.as<Mem>();
+        uint32_t x;
+
+        if (!o0.as<Reg>().isGpW() || !checkGpType(o1, opData.rType, &x) || !checkSignature(o1, o2))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, o2, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, opData.xOffset);
+        opcode.addReg(o0, 16);
+        opcode.addReg(o2, 10);
+        opcode.addReg(o1, 0);
+
+        rmRel = &m;
+        goto EmitOp_MemBaseNoImm_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseRM_NoImm: {
+      const InstDB::EncodingData::BaseRM_NoImm& opData = InstDB::EncodingData::baseRM_NoImm[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Mem)) {
+        const Mem& m = o1.as<Mem>();
+        rmRel = &m;
+
+        uint32_t x;
+        if (!checkGpType(o0, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, opData.rHiId))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, opData.xOffset);
+        opcode.addReg(o0, 0);
+        goto EmitOp_MemBaseNoImm_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseRM_SImm9: {
+Case_BaseLdurStur:
+      const InstDB::EncodingData::BaseRM_SImm9& opData = InstDB::EncodingData::baseRM_SImm9[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Mem)) {
+        const Mem& m = o1.as<Mem>();
+        rmRel = &m;
+
+        uint32_t x;
+        if (!checkGpType(o0, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, opData.rHiId))
+          goto InvalidPhysId;
+
+        if (m.hasBaseReg() && !m.hasIndex()) {
+          if (m.isOffset64Bit())
+            goto InvalidDisplacement;
+
+          int32_t offset32 = m.offsetLo32() >> opData.immShift;
+          if (Support::shl(offset32, opData.immShift) != m.offsetLo32())
+            goto InvalidDisplacement;
+
+          if (!Support::isInt9(offset32))
+            goto InvalidDisplacement;
+
+          if (m.isFixedOffset()) {
+            opcode.reset(opData.offsetOp());
+          }
+          else {
+            if (!opData.prePostOp())
+              goto InvalidInstruction;
+
+            opcode.reset(opData.prePostOp());
+            opcode.xorImm(m.isPreIndex(), 11);
+          }
+
+          opcode.xorImm(x, opData.xOffset);
+          opcode.addImm(offset32 & 0x1FF, 12);
+          opcode.addReg(o0, 0);
+          goto EmitOp_MemBase_Rn5;
+        }
+
+        goto InvalidAddress;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseRM_SImm10: {
+      const InstDB::EncodingData::BaseRM_SImm10& opData = InstDB::EncodingData::baseRM_SImm10[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Mem)) {
+        const Mem& m = o1.as<Mem>();
+        rmRel = &m;
+
+        uint32_t x;
+        if (!checkGpType(o0, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, opData.rHiId))
+          goto InvalidPhysId;
+
+        if (m.hasBaseReg() && !m.hasIndex()) {
+          if (m.isOffset64Bit())
+            goto InvalidDisplacement;
+
+          int32_t offset32 = m.offsetLo32() >> opData.immShift;
+          if (Support::shl(offset32, opData.immShift) != m.offsetLo32())
+            goto InvalidDisplacement;
+
+          if (!Support::isInt10(offset32))
+            goto InvalidDisplacement;
+
+          if (m.isPostIndex())
+            goto InvalidAddress;
+
+          // Offset has 10 bits, sign is stored in the 10th bit.
+          offset32 &= 0x3FF;
+
+          opcode.reset(opData.opcode());
+          opcode.xorImm(m.isPreIndex(), 11);
+          opcode.xorImm(x, opData.xOffset);
+          opcode.addImm(offset32 >> 9, 22);
+          opcode.addImm(offset32, 12);
+          opcode.addReg(o0, 0);
+          goto EmitOp_MemBase_Rn5;
+        }
+
+        goto InvalidAddress;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseAtomicOp: {
+      const InstDB::EncodingData::BaseAtomicOp& opData = InstDB::EncodingData::baseAtomicOp[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Mem)) {
+        const Mem& m = o2.as<Mem>();
+        uint32_t x;
+
+        if (!checkGpType(o0, opData.rType, &x) || !checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, o1, kZR))
+          goto InvalidInstruction;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, opData.xOffset);
+        opcode.addReg(o0, 16);
+        opcode.addReg(o1, 0);
+
+        rmRel = &m;
+        goto EmitOp_MemBaseNoImm_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseAtomicSt: {
+      const InstDB::EncodingData::BaseAtomicSt& opData = InstDB::EncodingData::baseAtomicSt[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Mem)) {
+        const Mem& m = o1.as<Mem>();
+        uint32_t x;
+
+        if (!checkGpType(o0, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!checkGpId(o0, kZR))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, opData.xOffset);
+        opcode.addReg(o0, 16);
+        opcode.addReg(Gp::kIdZr, 0);
+
+        rmRel = &m;
+        goto EmitOp_MemBaseNoImm_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingBaseAtomicCasp: {
+      const InstDB::EncodingData::BaseAtomicCasp& opData = InstDB::EncodingData::baseAtomicCasp[encodingIndex];
+      const Operand_& o4 = opExt[EmitterUtils::kOp4];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg) && o4.isMem()) {
+        const Mem& m = o4.as<Mem>();
+        uint32_t x;
+
+        if (!checkGpType(o0, opData.rType, &x))
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1, o2, o3))
+          goto InvalidInstruction;
+
+        if (!checkEven(o0, o2) || !checkGpId(o0, o2, kZR))
+          goto InvalidPhysId;
+
+        if (!checkConsecutive(o0, o1) || !checkConsecutive(o2, o3))
+          goto InvalidPhysId;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(x, opData.xOffset);
+        opcode.addReg(o0, 16);
+        opcode.addReg(o2, 0);
+
+        rmRel = &m;
+        goto EmitOp_MemBaseNoImm_Rn5;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [FSimd - Instructions]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingFSimdSV: {
+      const InstDB::EncodingData::FSimdSV& opData = InstDB::EncodingData::fSimdSV[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        uint32_t q = diff(o1.as<Reg>().type(), RegType::kARM_VecD);
+        if (q > 1)
+          goto InvalidInstruction;
+
+        if (o0.as<Vec>().hasElementType())
+          goto InvalidInstruction;
+
+        // This operation is only defined for:
+        //   hD, vS.{4|8}h (16-bit)
+        //   sD, vS.4s     (32-bit)
+        uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+        uint32_t elementSz = o1.as<Vec>().elementType() - Vec::kElementTypeH;
+
+        // Size greater than 1 means 64-bit elements, not supported.
+        if ((sz | elementSz) > 1 || sz != elementSz)
+          goto InvalidInstruction;
+
+        // Size 1 (32-bit float) requires at least 4 elements.
+        if (sz && !q)
+          goto InvalidInstruction;
+
+        // Bit flipping according to sz.
+        static const uint32_t szBits[] = { B(29), 0 };
+
+        opcode.reset(opData.opcode << 10);
+        opcode ^= szBits[sz];
+        opcode.addImm(q, 30);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingFSimdVV: {
+      const InstDB::EncodingData::FSimdVV& opData = InstDB::EncodingData::fSimdVV[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        if (!matchSignature(o0, o1, instFlags))
+          goto InvalidInstruction;
+
+        if (!pickFpOpcode(o0.as<Vec>(), opData.scalarOp(), opData.scalarHf(), opData.vectorOp(), opData.vectorHf(), &opcode))
+          goto InvalidInstruction;
+
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingFSimdVVV: {
+      const InstDB::EncodingData::FSimdVVV& opData = InstDB::EncodingData::fSimdVVV[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (!matchSignature(o0, o1, o2, instFlags))
+          goto InvalidInstruction;
+
+        if (!pickFpOpcode(o0.as<Vec>(), opData.scalarOp(), opData.scalarHf(), opData.vectorOp(), opData.vectorHf(), &opcode))
+          goto InvalidInstruction;
+
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingFSimdVVVe: {
+      const InstDB::EncodingData::FSimdVVVe& opData = InstDB::EncodingData::fSimdVVVe[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (!o2.as<Vec>().hasElementIndex()) {
+          if (!matchSignature(o0, o1, o2, instFlags))
+            goto InvalidInstruction;
+
+          if (!pickFpOpcode(o0.as<Vec>(), opData.scalarOp(), opData.scalarHf(), opData.vectorOp(), opData.vectorHf(), &opcode))
+            goto InvalidInstruction;
+
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+        else {
+          if (!matchSignature(o0, o1, instFlags))
+            goto InvalidInstruction;
+
+          uint32_t q = o1.as<Reg>().isVecQ();
+          uint32_t sz;
+
+          if (!pickFpOpcode(o0.as<Vec>(), opData.elementScalarOp(), InstDB::kHF_D, opData.elementVectorOp(), InstDB::kHF_D, &opcode, &sz))
+            goto InvalidInstruction;
+
+          if (sz == 0 && o2.as<Reg>().id() > 15)
+            goto InvalidPhysId;
+
+          uint32_t elementIndex = o2.as<Vec>().elementIndex();
+          if (elementIndex > (7u >> sz))
+            goto InvalidElementIndex;
+
+          uint32_t hlm = elementIndex << sz;
+          opcode.addImm(q, 30);
+          opcode.addImm(hlm & 3u, 20);
+          opcode.addImm(hlm >> 2, 11);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingFSimdVVVV: {
+      const InstDB::EncodingData::FSimdVVVV& opData = InstDB::EncodingData::fSimdVVVV[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
+        if (!matchSignature(o0, o1, o2, o3, instFlags))
+          goto InvalidInstruction;
+
+        if (!pickFpOpcode(o0.as<Vec>(), opData.scalarOp(), opData.scalarHf(), opData.vectorOp(), opData.vectorHf(), &opcode))
+          goto InvalidInstruction;
+
+        goto EmitOp_Rd0_Rn5_Rm16_Ra10;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFcadd: {
+      const InstDB::EncodingData::SimdFcadd& opData = InstDB::EncodingData::simdFcadd[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+        if (!checkSignature(o0, o1, o2) || o0.as<Vec>().hasElementIndex())
+          goto InvalidInstruction;
+
+        uint32_t q = diff(o0.as<Reg>().type(), RegType::kARM_VecD);
+        if (q > 1)
+          goto InvalidInstruction;
+
+        uint32_t sz = o0.as<Vec>().elementType() - Vec::kElementTypeB;
+        if (sz == 0 || sz > 3)
+          goto InvalidInstruction;
+
+        // 0 <- 90deg.
+        // 1 <- 270deg.
+        uint32_t rot = 0;
+        if (o3.as<Imm>().value() == 270)
+          rot = 1;
+        else if (o3.as<Imm>().value() != 90)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(q, 30);
+        opcode.addImm(sz, 22);
+        opcode.addImm(rot, 12);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFccmpFccmpe: {
+      const InstDB::EncodingData::SimdFccmpFccmpe& opData = InstDB::EncodingData::simdFccmpFccmpe[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) {
+        uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+        if (sz > 2)
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1) || o0.as<Vec>().hasElementType())
+          goto InvalidInstruction;
+
+        uint64_t nzcv = o2.as<Imm>().valueAs<uint64_t>();
+        uint64_t cond = o3.as<Imm>().valueAs<uint64_t>();
+
+        if ((nzcv | cond) > 0xFu)
+          goto InvalidImmediate;
+
+        uint32_t type = (sz - 1) & 0x3u;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(type, 22);
+        opcode.addImm(condCodeToOpcodeCond(uint32_t(cond)), 12);
+        opcode.addImm(nzcv, 0);
+
+        goto EmitOp_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFcm: {
+      const InstDB::EncodingData::SimdFcm& opData = InstDB::EncodingData::simdFcm[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg) && opData.hasRegisterOp()) {
+        if (!matchSignature(o0, o1, o2, instFlags))
+          goto InvalidInstruction;
+
+        if (!pickFpOpcode(o0.as<Vec>(), opData.registerScalarOp(), opData.registerScalarHf(), opData.registerVectorOp(), opData.registerVectorHf(), &opcode))
+          goto InvalidInstruction;
+
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm) && opData.hasZeroOp()) {
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (o2.as<Imm>().value() != 0 || o2.as<Imm>().predicate() != 0)
+          goto InvalidImmediate;
+
+        if (!pickFpOpcode(o0.as<Vec>(), opData.zeroScalarOp(), InstDB::kHF_B, opData.zeroVectorOp(), InstDB::kHF_B, &opcode))
+          goto InvalidInstruction;
+
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFcmla: {
+      const InstDB::EncodingData::SimdFcmla& opData = InstDB::EncodingData::simdFcmla[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        uint32_t q = diff(o0.as<Reg>().type(), RegType::kARM_VecD);
+        if (q > 1)
+          goto InvalidInstruction;
+
+        uint32_t sz = o0.as<Vec>().elementType() - Vec::kElementTypeB;
+        if (sz == 0 || sz > 3)
+          goto InvalidInstruction;
+
+        uint32_t rot = 0;
+        switch (o3.as<Imm>().value()) {
+          case 0  : rot = 0; break;
+          case 90 : rot = 1; break;
+          case 180: rot = 2; break;
+          case 270: rot = 3; break;
+          default:
+            goto InvalidImmediate;
+        }
+
+        if (!o2.as<Vec>().hasElementIndex()) {
+          if (!checkSignature(o1, o2))
+            goto InvalidInstruction;
+
+          opcode.reset(opData.regularOp());
+          opcode.addImm(q, 30);
+          opcode.addImm(sz, 22);
+          opcode.addImm(rot, 11);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+        else {
+          if (o0.as<Vec>().elementType() != o2.as<Vec>().elementType())
+            goto InvalidInstruction;
+
+          // Only allowed vectors are: 4H, 8H, and 4S.
+          if (!(sz == 1 || (q == 1 && sz == 2)))
+            goto InvalidInstruction;
+
+          // Element index ranges:
+          //   4H - ElementIndex[0..1] (index 2..3 is UNDEFINED).
+          //   8H - ElementIndex[0..3].
+          //   4S - ElementIndex[0..1].
+          uint32_t elementIndex = o2.as<Vec>().elementIndex();
+          uint32_t hlFieldShift = sz == 1 ? 0u : 1u;
+          uint32_t maxElementIndex = q == 1 && sz == 1 ? 3u : 1u;
+
+          if (elementIndex > maxElementIndex)
+            goto InvalidElementIndex;
+
+          uint32_t hl = elementIndex << hlFieldShift;
+
+          opcode.reset(opData.elementOp());
+          opcode.addImm(q, 30);
+          opcode.addImm(sz, 22);
+          opcode.addImm(hl & 1u, 21); // L field.
+          opcode.addImm(hl >> 1, 11); // H field.
+          opcode.addImm(rot, 13);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFcmpFcmpe: {
+      const InstDB::EncodingData::SimdFcmpFcmpe& opData = InstDB::EncodingData::simdFcmpFcmpe[encodingIndex];
+
+      uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+      uint32_t type = (sz - 1) & 0x3u;
+
+      if (sz > 2)
+        goto InvalidInstruction;
+
+      if (o0.as<Vec>().hasElementType())
+        goto InvalidInstruction;
+
+      opcode.reset(opData.opcode());
+      opcode.addImm(type, 22);
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        goto EmitOp_Rn5_Rm16;
+      }
+
+      if (isign4 == ENC_OPS2(Reg, Imm)) {
+        if (o1.as<Imm>().value() != 0 || o1.as<Imm>().predicate() != 0)
+          goto InvalidInstruction;
+
+        opcode |= B(3);
+        goto EmitOp_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFcsel: {
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+        if (!checkSignature(o0, o1, o2))
+          goto InvalidInstruction;
+
+        uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+        uint32_t type = (sz - 1) & 0x3u;
+
+        if (sz > 2 || o0.as<Vec>().hasElementType())
+          goto InvalidInstruction;
+
+        uint64_t cond = o3.as<Imm>().valueAs<uint64_t>();
+        if (cond > 0xFu)
+          goto InvalidImmediate;
+
+        opcode.reset(0b00011110001000000000110000000000);
+        opcode.addImm(type, 22);
+        opcode.addImm(condCodeToOpcodeCond(uint32_t(cond)), 12);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFcvt: {
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        uint32_t dstSz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+        uint32_t srcSz = diff(o1.as<Reg>().type(), RegType::kARM_VecH);
+
+        if ((dstSz | srcSz) > 3)
+          goto InvalidInstruction;
+
+        if (o0.as<Vec>().hasElementType() || o1.as<Vec>().hasElementType())
+          goto InvalidInstruction;
+
+        // Table that provides 'type' and 'opc' according to the dst/src combination.
+        static const uint8_t table[] = {
+          0xFFu, // H <- H (Invalid).
+          0x03u, // H <- S (type=00 opc=11).
+          0x13u, // H <- D (type=01 opc=11).
+          0xFFu, // H <- Q (Invalid).
+          0x30u, // S <- H (type=11 opc=00).
+          0xFFu, // S <- S (Invalid).
+          0x10u, // S <- D (type=01 opc=00).
+          0xFFu, // S <- Q (Invalid).
+          0x31u, // D <- H (type=11 opc=01).
+          0x01u, // D <- S (type=00 opc=01).
+          0xFFu, // D <- D (Invalid).
+          0xFFu, // D <- Q (Invalid).
+          0xFFu, // Q <- H (Invalid).
+          0xFFu, // Q <- S (Invalid).
+          0xFFu, // Q <- D (Invalid).
+          0xFFu  // Q <- Q (Invalid).
+        };
+
+        uint32_t typeOpc = table[(dstSz << 2) | srcSz];
+        opcode.reset(0b0001111000100010010000 << 10);
+        opcode.addImm(typeOpc >> 4, 22);
+        opcode.addImm(typeOpc & 15, 15);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFcvtLN: {
+      const InstDB::EncodingData::SimdFcvtLN& opData = InstDB::EncodingData::simdFcvtLN[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        // Scalar form - only FCVTXN.
+        if (o0.as<Vec>().isVecS() && o1.as<Vec>().isVecD()) {
+          if (!opData.hasScalar())
+            goto InvalidInstruction;
+
+          if (o0.as<Vec>().hasElementType() || o1.as<Vec>().hasElementType())
+            goto InvalidInstruction;
+
+          opcode.reset(opData.scalarOp());
+          opcode |= B(22); // sz bit must be 1, the only supported combination of FCVTXN.
+          goto EmitOp_Rd0_Rn5;
+        }
+
+        opcode.reset(opData.vectorOp());
+
+        const Vec& rL = (instFlags & InstDB::kInstFlagLong) ? o0.as<Vec>() : o1.as<Vec>();
+        const Vec& rN = (instFlags & InstDB::kInstFlagLong) ? o1.as<Vec>() : o0.as<Vec>();
+
+        uint32_t q = diff(rN.type(), RegType::kARM_VecD);
+        if (uint32_t(opcode.hasQ()) != q)
+          goto InvalidInstruction;
+
+        if (rL.isVecS4() && rN.elementType() == Vec::kElementTypeH && !opData.isCvtxn()) {
+          goto EmitOp_Rd0_Rn5;
+        }
+
+        if (rL.isVecD2() && rN.elementType() == Vec::kElementTypeS) {
+          opcode |= B(22);
+          goto EmitOp_Rd0_Rn5;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFcvtSV: {
+      const InstDB::EncodingData::SimdFcvtSV& opData = InstDB::EncodingData::simdFcvtSV[encodingIndex];
+
+      // So we can support both IntToFloat and FloatToInt conversions.
+      const Operand_& oGp = opData.isFloatToInt() ? o0 : o1;
+      const Operand_& oVec = opData.isFloatToInt() ? o1 : o0;
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        if (oGp.as<Reg>().isGp() && oVec.as<Reg>().isVec()) {
+          uint32_t x = oGp.as<Reg>().isGpX();
+          uint32_t type = diff(oVec.as<Reg>().type(), RegType::kARM_VecH);
+
+          if (type > 2u)
+            goto InvalidInstruction;
+
+          type = (type - 1u) & 0x3;
+          opcode.reset(opData.generalOp());
+          opcode.addImm(type, 22);
+          opcode.addImm(x, 31);
+          goto EmitOp_Rd0_Rn5;
+        }
+
+        if (o0.as<Reg>().isVec() && o1.as<Reg>().isVec()) {
+          if (!checkSignature(o0, o1))
+            goto InvalidInstruction;
+
+          if (!pickFpOpcode(o0.as<Vec>(), opData.scalarIntOp(), InstDB::kHF_B, opData.vectorIntOp(), InstDB::kHF_B, &opcode))
+            goto InvalidInstruction;
+
+          goto EmitOp_Rd0_Rn5;
+        }
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm) && opData.isFixedPoint()) {
+        if (o2.as<Imm>().valueAs<uint64_t>() >= 64)
+          goto InvalidInstruction;
+
+        uint32_t scale = o2.as<Imm>().valueAs<uint32_t>();
+        if (scale == 0)
+          goto InvalidInstruction;
+
+        if (oGp.as<Reg>().isGp() && oVec.as<Reg>().isVec()) {
+          uint32_t x = oGp.as<Reg>().isGpX();
+          uint32_t type = diff(oVec.as<Reg>().type(), RegType::kARM_VecH);
+
+          uint32_t scaleLimit = 32u << x;
+          if (scale > scaleLimit)
+            goto InvalidInstruction;
+
+          type = (type - 1u) & 0x3;
+          opcode.reset(opData.generalOp() ^ B(21));
+          opcode.addImm(type, 22);
+          opcode.addImm(x, 31);
+          opcode.addImm(64u - scale, 10);
+          goto EmitOp_Rd0_Rn5;
+        }
+
+        if (o0.as<Reg>().isVec() && o1.as<Reg>().isVec()) {
+          if (!checkSignature(o0, o1))
+            goto InvalidInstruction;
+
+          uint32_t sz;
+          if (!pickFpOpcode(o0.as<Vec>(), opData.scalarFpOp(), InstDB::kHF_0, opData.vectorFpOp(), InstDB::kHF_0, &opcode, &sz))
+            goto InvalidInstruction;
+
+          uint32_t scaleLimit = 16u << sz;
+          if (scale > scaleLimit)
+            goto InvalidInstruction;
+
+          uint32_t imm = Support::neg(scale) & Support::lsbMask<uint32_t>(sz + 4 + 1);
+          opcode.addImm(imm, 16);
+          goto EmitOp_Rd0_Rn5;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFmlal: {
+      const InstDB::EncodingData::SimdFmlal& opData = InstDB::EncodingData::simdFmlal[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        uint32_t q = diff(o0.as<Reg>().type(), RegType::kARM_VecD);
+        uint32_t qIsOptional = opData.optionalQ();
+
+        if (qIsOptional) {
+          // This instruction works with either 64-bit or 128-bit registers,
+          // encoded by Q bit.
+          if (q > 1)
+            goto InvalidInstruction;
+        }
+        else {
+          // This instruction requires 128-bit vector registers.
+          if (q != 1)
+            goto InvalidInstruction;
+
+          // The instruction is ehtier B (bottom) or T (top), which is part of
+          // the opcode, which uses Q bit, so we have to clear it explicitly.
+          q = 0;
+        }
+
+        if (uint32_t(o0.as<Reg>().type()) != uint32_t(o1.as<Reg>().type()) + qIsOptional ||
+            o0.as<Vec>().elementType() != opData.tA ||
+            o1.as<Vec>().elementType() != opData.tB)
+          goto InvalidInstruction;
+
+        if (!o2.as<Vec>().hasElementIndex()) {
+          if (!checkSignature(o1, o2))
+            goto InvalidInstruction;
+
+          opcode.reset(opData.vectorOp());
+          opcode.addImm(q, 30);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+        else {
+          if (o2.as<Vec>().elementType() != opData.tElement)
+            goto InvalidInstruction;
+
+          if (o2.as<Reg>().id() > 15)
+            goto InvalidPhysId;
+
+          uint32_t elementIndex = o2.as<Vec>().elementIndex();
+          if (elementIndex > 7u)
+            goto InvalidElementIndex;
+
+          opcode.reset(opData.elementOp());
+          opcode.addImm(q, 30);
+          opcode.addImm(elementIndex & 3u, 20);
+          opcode.addImm(elementIndex >> 2, 11);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdFmov: {
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        // FMOV Gp <-> Vec opcode:
+        opcode.reset(0b00011110001001100000000000000000);
+
+        if (o0.as<Reg>().isGp() && o1.as<Reg>().isVec()) {
+          // FMOV Wd, Hn      (sf=0 type=11 rmode=00 op=110)
+          // FMOV Xd, Hn      (sf=1 type=11 rmode=00 op=110)
+          // FMOV Wd, Sn      (sf=0 type=00 rmode=00 op=110)
+          // FMOV Xd, Dn      (sf=1 type=11 rmode=00 op=110)
+          // FMOV Xd, Vn.d[1] (sf=1 type=10 rmode=01 op=110)
+          uint32_t x = o0.as<Reg>().isGpX();
+          uint32_t sz = diff(o1.as<Reg>().type(), RegType::kARM_VecH);
+
+          uint32_t type = (sz - 1) & 0x3u;
+          uint32_t rModeOp = 0b00110;
+
+          if (o1.as<Vec>().hasElementIndex()) {
+            // Special case.
+            if (!x || !o1.as<Vec>().isVecD2() || o1.as<Vec>().elementIndex() != 1)
+              goto InvalidInstruction;
+            type = 0b10;
+            rModeOp = 0b01110;
+          }
+          else {
+            // Must be scalar.
+            if (sz > 2)
+              goto InvalidInstruction;
+
+            if (o1.as<Vec>().hasElementType())
+              goto InvalidInstruction;
+
+            if (o1.as<Vec>().isVecS() && x)
+              goto InvalidInstruction;
+
+            if (o1.as<Vec>().isVecD() && !x)
+              goto InvalidInstruction;
+          }
+
+          opcode.addImm(x, 31);
+          opcode.addImm(type, 22);
+          opcode.addImm(rModeOp, 16);
+          goto EmitOp_Rd0_Rn5;
+        }
+
+        if (o0.as<Reg>().isVec() && o1.as<Reg>().isGp()) {
+          // FMOV Hd, Wn      (sf=0 type=11 rmode=00 op=111)
+          // FMOV Hd, Xn      (sf=1 type=11 rmode=00 op=111)
+          // FMOV Sd, Wn      (sf=0 type=00 rmode=00 op=111)
+          // FMOV Dd, Xn      (sf=1 type=11 rmode=00 op=111)
+          // FMOV Vd.d[1], Xn (sf=1 type=10 rmode=01 op=111)
+          uint32_t x = o1.as<Reg>().isGpX();
+          uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+
+          uint32_t type = (sz - 1) & 0x3u;
+          uint32_t rModeOp = 0b00111;
+
+          if (o0.as<Vec>().hasElementIndex()) {
+            // Special case.
+            if (!x || !o0.as<Vec>().isVecD2() || o0.as<Vec>().elementIndex() != 1)
+              goto InvalidInstruction;
+            type = 0b10;
+            rModeOp = 0b01111;
+          }
+          else {
+            // Must be scalar.
+            if (sz > 2)
+              goto InvalidInstruction;
+
+            if (o0.as<Vec>().hasElementType())
+              goto InvalidInstruction;
+
+            if (o0.as<Vec>().isVecS() && x)
+              goto InvalidInstruction;
+
+            if (o0.as<Vec>().isVecD() && !x)
+              goto InvalidInstruction;
+          }
+
+          opcode.addImm(x, 31);
+          opcode.addImm(type, 22);
+          opcode.addImm(rModeOp, 16);
+          goto EmitOp_Rd0_Rn5;
+        }
+
+        if (checkSignature(o0, o1)) {
+          uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+          if (sz > 2)
+            goto InvalidInstruction;
+
+          if (o0.as<Vec>().hasElementType())
+            goto InvalidInstruction;
+
+          uint32_t type = (sz - 1) & 0x3;
+          opcode.reset(0b00011110001000000100000000000000);
+          opcode.addImm(type, 22);
+          goto EmitOp_Rd0_Rn5;
+        }
+      }
+
+      if (isign4 == ENC_OPS2(Reg, Imm)) {
+        if (o0.as<Reg>().isVec()) {
+          double fpValue;
+          if (o1.as<Imm>().isDouble())
+            fpValue = o1.as<Imm>().valueAs<double>();
+          else if (o1.as<Imm>().isInt32())
+            fpValue = o1.as<Imm>().valueAs<int32_t>();
+          else
+            goto InvalidImmediate;
+
+          if (!Utils::isFP64Imm8(fpValue))
+            goto InvalidImmediate;
+
+          uint32_t imm8 = Utils::encodeFP64ToImm8(fpValue);
+          if (!o0.as<Vec>().hasElementType()) {
+            // FMOV (scalar, immediate).
+            uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+            uint32_t type = (sz - 1u) & 0x3u;
+
+            if (sz > 2)
+              goto InvalidInstruction;
+
+            opcode.reset(0b00011110001000000001000000000000);
+            opcode.addImm(type, 22);
+            opcode.addImm(imm8, 13);
+            goto EmitOp_Rd0;
+          }
+          else {
+            uint32_t q = diff(o0.as<Vec>().type(), RegType::kARM_VecD);
+            uint32_t sz = o0.as<Vec>().elementType() - Vec::kElementTypeH;
+
+            if (q > 1 || sz > 2)
+              goto InvalidInstruction;
+
+            static const uint32_t szBits[3] = { B(11), B(0), B(29) };
+            opcode.reset(0b00001111000000001111010000000000);
+            opcode ^= szBits[sz];
+            opcode.addImm(q, 30);
+            opcode.addImm(imm8 >> 5, 16);
+            opcode.addImm(imm8 & 31, 5);
+            goto EmitOp_Rd0;
+          }
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingFSimdPair: {
+      const InstDB::EncodingData::FSimdPair& opData = InstDB::EncodingData::fSimdPair[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        // This operation is only defined for:
+        //   hD, vS.2h (16-bit)
+        //   sD, vS.2s (32-bit)
+        //   dD, vS.2d (64-bit)
+        uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecH);
+        if (sz > 2)
+          goto InvalidInstruction;
+
+        static const uint32_t szSignatures[3] = {
+          VecS::kSignature | (Vec::kSignatureElementH),
+          VecD::kSignature | (Vec::kSignatureElementS),
+          VecV::kSignature | (Vec::kSignatureElementD)
+        };
+
+        if (o1.signature() != szSignatures[sz])
+          goto InvalidInstruction;
+
+        static const uint32_t szBits[] = { B(29), 0, B(22) };
+        opcode.reset(opData.scalarOp());
+        opcode ^= szBits[sz];
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (!checkSignature(o0, o1, o2))
+          goto InvalidInstruction;
+
+        uint32_t q = diff(o0.as<Reg>().type(), RegType::kARM_VecD);
+        if (q > 1)
+          goto InvalidInstruction;
+
+        uint32_t sz = o0.as<Vec>().elementType() - Vec::kElementTypeH;
+        if (sz > 2)
+          goto InvalidInstruction;
+
+        static const uint32_t szBits[3] = { B(22) | B(21) | B(15) | B(14), 0, B(22) };
+        opcode.reset(opData.vectorOp());
+        opcode ^= szBits[sz];
+        opcode.addImm(q, 30);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [ISimd - Instructions]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingISimdSV: {
+      const InstDB::EncodingData::ISimdSV& opData = InstDB::EncodingData::iSimdSV[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        // The first destination operand is scalar, which matches element-type of source vectors.
+        uint32_t L = (instFlags & InstDB::kInstFlagLong) != 0;
+        if (diff(o0.as<Vec>().type(), RegType::kARM_VecB) != o1.as<Vec>().elementType() - Vec::kElementTypeB + L)
+          goto InvalidInstruction;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as<Reg>().type(), o1.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(sizeOp.q(), 30);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdVV: {
+      const InstDB::EncodingData::ISimdVV& opData = InstDB::EncodingData::iSimdVV[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        const Operand_& sop = significantSimdOp(o0, o1, instFlags);
+        if (!matchSignature(o0, o1, instFlags))
+          goto InvalidInstruction;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as<Reg>().type(), sop.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdVVx: {
+      const InstDB::EncodingData::ISimdVVx& opData = InstDB::EncodingData::iSimdVVx[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        if (o0.signature() != opData.op0Signature ||
+            o1.signature() != opData.op1Signature)
+          goto InvalidInstruction;
+
+        opcode.reset(opData.opcode());
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdVVV: {
+      const InstDB::EncodingData::ISimdVVV& opData = InstDB::EncodingData::iSimdVVV[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        const Operand_& sop = significantSimdOp(o0, o1, instFlags);
+        if (!matchSignature(o0, o1, o2, instFlags))
+          goto InvalidInstruction;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as<Reg>().type(), sop.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdVVVx: {
+      const InstDB::EncodingData::ISimdVVVx& opData = InstDB::EncodingData::iSimdVVVx[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (o0.signature() != opData.op0Signature ||
+            o1.signature() != opData.op1Signature ||
+            o2.signature() != opData.op2Signature)
+          goto InvalidInstruction;
+
+        opcode.reset(opData.opcode());
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdWWV: {
+      // Special case for wide add/sub [s|b][add|sub][w]{2}.
+      const InstDB::EncodingData::ISimdWWV& opData = InstDB::EncodingData::iSimdWWV[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o2.as<Reg>().type(), o2.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1) || !o0.as<Reg>().isVecV() || o0.as<Vec>().elementType() != o2.as<Vec>().elementType() + 1)
+          goto InvalidInstruction;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdVVVe: {
+      const InstDB::EncodingData::ISimdVVVe& opData = InstDB::EncodingData::iSimdVVVe[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        const Operand_& sop = significantSimdOp(o0, o1, instFlags);
+        if (!matchSignature(o0, o1, instFlags))
+          goto InvalidInstruction;
+
+        if (!o2.as<Vec>().hasElementIndex()) {
+          SizeOp sizeOp = armElementTypeToSizeOp(opData.regularVecType, sop.as<Reg>().type(), sop.as<Vec>().elementType());
+          if (!sizeOp.isValid())
+            goto InvalidInstruction;
+
+          if (!checkSignature(o1, o2))
+            goto InvalidInstruction;
+
+          opcode.reset(uint32_t(opData.regularOp) << 10);
+          opcode.addImm(sizeOp.qs(), 30);
+          opcode.addImm(sizeOp.scalar(), 28);
+          opcode.addImm(sizeOp.size(), 22);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+        else {
+          SizeOp sizeOp = armElementTypeToSizeOp(opData.elementVecType, sop.as<Reg>().type(), sop.as<Vec>().elementType());
+          if (!sizeOp.isValid())
+            goto InvalidInstruction;
+
+          uint32_t elementIndex = o2.as<Vec>().elementIndex();
+          LMHImm lmh;
+
+          if (!encodeLMH(sizeOp.size(), elementIndex, &lmh))
+            goto InvalidElementIndex;
+
+          if (o2.as<Reg>().id() > lmh.maxRmId)
+            goto InvalidPhysId;
+
+          opcode.reset(uint32_t(opData.elementOp) << 10);
+          opcode.addImm(sizeOp.q(), 30);
+          opcode.addImm(sizeOp.size(), 22);
+          opcode.addImm(lmh.lm, 20);
+          opcode.addImm(lmh.h, 11);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdVVVI: {
+      const InstDB::EncodingData::ISimdVVVI& opData = InstDB::EncodingData::iSimdVVVI[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) {
+        const Operand_& sop = significantSimdOp(o0, o1, instFlags);
+        if (!matchSignature(o0, o1, o2, instFlags))
+          goto InvalidInstruction;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as<Reg>().type(), sop.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        uint64_t immValue = o3.as<Imm>().valueAs<uint64_t>();
+        uint32_t immSize = opData.immSize;
+
+        if (opData.imm64HasOneBitLess && !sizeOp.q())
+          immSize--;
+
+        uint32_t immMax = 1u << immSize;
+        if (immValue >= immMax)
+          goto InvalidImmediate;
+
+        opcode.reset(opData.opcode());
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        opcode.addImm(immValue, opData.immShift);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdVVVV: {
+      const InstDB::EncodingData::ISimdVVVV& opData = InstDB::EncodingData::iSimdVVVV[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
+        const Operand_& sop = significantSimdOp(o0, o1, instFlags);
+        if (!matchSignature(o0, o1, o2, o3, instFlags))
+          goto InvalidInstruction;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as<Reg>().type(), sop.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.opcode) << 10);
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5_Rm16_Ra10;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingISimdVVVVx: {
+      const InstDB::EncodingData::ISimdVVVVx& opData = InstDB::EncodingData::iSimdVVVVx[encodingIndex];
+
+      if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
+        if (o0.signature() != opData.op0Signature ||
+            o1.signature() != opData.op1Signature ||
+            o2.signature() != opData.op2Signature ||
+            o3.signature() != opData.op3Signature)
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.opcode) << 10);
+        goto EmitOp_Rd0_Rn5_Rm16_Ra10;
+      }
+
+      break;
+    }
+
+
+    case InstDB::kEncodingISimdPair: {
+      const InstDB::EncodingData::ISimdPair& opData = InstDB::EncodingData::iSimdPair[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg) && opData.opcode2) {
+        if (o0.as<Vec>().isVecD1() && o1.as<Vec>().isVecD2()) {
+          opcode.reset(uint32_t(opData.opcode2) << 10);
+          opcode.addImm(0x3, 22); // size.
+          goto EmitOp_Rd0_Rn5;
+        }
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (!matchSignature(o0, o1, o2, instFlags))
+          goto InvalidInstruction;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.opType3, o0.as<Reg>().type(), o0.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.opcode3) << 10);
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdBicOrr: {
+      const InstDB::EncodingData::SimdBicOrr& opData = InstDB::EncodingData::simdBicOrr[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (!matchSignature(o0, o1, o2, instFlags))
+          goto InvalidInstruction;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_B, o0.as<Reg>().type(), o0.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.registerOp) << 10);
+        opcode.addImm(sizeOp.q(), 30);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      if (isign4 == ENC_OPS2(Reg, Imm) || isign4 == ENC_OPS3(Reg, Imm, Imm)) {
+        SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_HS, o0.as<Reg>().type(), o0.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        if (o1.as<Imm>().valueAs<uint64_t>() > 0xFFFFFFFFu)
+          goto InvalidImmediate;
+
+        uint32_t imm = o1.as<Imm>().valueAs<uint32_t>();
+        uint32_t shift = 0;
+        uint32_t maxShift = (8u << sizeOp.size()) - 8u;
+
+        if (o2.isImm()) {
+          if (o2.as<Imm>().predicate() != uint32_t(ShiftOp::kLSL))
+            goto InvalidImmediate;
+
+          if (imm > 0xFFu || o2.as<Imm>().valueAs<uint64_t>() > maxShift)
+            goto InvalidImmediate;
+
+          shift = o2.as<Imm>().valueAs<uint32_t>();
+          if ((shift & 0x7u) != 0u)
+            goto InvalidImmediate;
+        }
+        else if (imm) {
+          shift = Support::ctz(imm) & 0x7u;
+          imm >>= shift;
+
+          if (imm > 0xFFu || shift > maxShift)
+            goto InvalidImmediate;
+        }
+
+        uint32_t cmode = 0x1u | ((shift / 8u) << 1);
+        if (sizeOp.size() == 1)
+          cmode |= B(3);
+
+        // The immediate value is split into ABC and DEFGH parts.
+        uint32_t abc = (imm >> 5) & 0x7u;
+        uint32_t defgh = imm & 0x1Fu;
+
+        opcode.reset(uint32_t(opData.immediateOp) << 10);
+        opcode.addImm(sizeOp.q(), 30);
+        opcode.addImm(abc, 16);
+        opcode.addImm(cmode, 12);
+        opcode.addImm(defgh, 5);
+        goto EmitOp_Rd0;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdCmp: {
+      const InstDB::EncodingData::SimdCmp& opData = InstDB::EncodingData::simdCmp[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg) && opData.regOp) {
+        if (!matchSignature(o0, o1, o2, instFlags))
+          goto InvalidInstruction;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as<Reg>().type(), o0.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.regOp) << 10);
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm) && opData.zeroOp) {
+        if (!matchSignature(o0, o1, instFlags))
+          goto InvalidInstruction;
+
+        if (o2.as<Imm>().value() != 0)
+          goto InvalidImmediate;
+
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as<Reg>().type(), o0.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.zeroOp) << 10);
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdDot: {
+      const InstDB::EncodingData::SimdDot& opData = InstDB::EncodingData::simdDot[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        uint32_t q = diff(o0.as<Reg>().type(), RegType::kARM_VecD);
+        uint32_t size = 2;
+
+        if (q > 1u)
+          goto InvalidInstruction;
+
+        if (!o2.as<Vec>().hasElementIndex()) {
+          if (!opData.vectorOp)
+            goto InvalidInstruction;
+
+          if (o0.as<Reg>().type() != o1.as<Reg>().type() || o1.as<Reg>().type() != o2.as<Reg>().type())
+            goto InvalidInstruction;
+
+          if (o0.as<Vec>().elementType() != opData.tA ||
+              o1.as<Vec>().elementType() != opData.tB ||
+              o2.as<Vec>().elementType() != opData.tB)
+            goto InvalidInstruction;
+
+          opcode.reset(uint32_t(opData.vectorOp) << 10);
+          opcode.addImm(q, 30);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+        else {
+          if (!opData.elementOp)
+            goto InvalidInstruction;
+
+          if (o0.as<Reg>().type() != o1.as<Reg>().type() || !o2.as<Reg>().isVecV())
+            goto InvalidInstruction;
+
+          if (o0.as<Vec>().elementType() != opData.tA ||
+              o1.as<Vec>().elementType() != opData.tB ||
+              o2.as<Vec>().elementType() != opData.tElement)
+            goto InvalidInstruction;
+
+          uint32_t elementIndex = o2.as<Vec>().elementIndex();
+          LMHImm lmh;
+
+          if (!encodeLMH(size, elementIndex, &lmh))
+            goto InvalidElementIndex;
+
+          if (o2.as<Reg>().id() > lmh.maxRmId)
+            goto InvalidPhysId;
+
+          opcode.reset(uint32_t(opData.elementOp) << 10);
+          opcode.addImm(q, 30);
+          opcode.addImm(lmh.lm, 20);
+          opcode.addImm(lmh.h, 11);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdDup: SimdDup: {
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        // Truth table of valid encodings of `Q:1|ElementType:3`
+        uint32_t kValidEncodings = B(Vec::kElementTypeB + 0) |
+                                   B(Vec::kElementTypeH + 0) |
+                                   B(Vec::kElementTypeS + 0) |
+                                   B(Vec::kElementTypeB + 8) |
+                                   B(Vec::kElementTypeH + 8) |
+                                   B(Vec::kElementTypeS + 8) |
+                                   B(Vec::kElementTypeD + 8) ;
+
+        uint32_t q = diff(o0.as<Reg>().type(), RegType::kARM_VecD);
+
+        if (o1.as<Reg>().isGp()) {
+          // DUP - Vec (scalar|vector) <- GP register.
+          //
+          // NOTE: This is only scalar for `dup d, x` case, otherwise the value
+          // would be duplicated across all vector elements (1, 2, 4, 8, or 16).
+          uint32_t elementType = o0.as<Vec>().elementType();
+          if (q > 1 || !Support::bitTest(kValidEncodings, (q << 3) | elementType))
+            goto InvalidInstruction;
+
+          uint32_t lsbIndex = elementType - 1u;
+          uint32_t imm5 = 1u << lsbIndex;
+
+          opcode.reset(0b0000111000000000000011 << 10);
+          opcode.addImm(q, 30);
+          opcode.addImm(imm5, 16);
+          goto EmitOp_Rd0_Rn5;
+        }
+
+        if (!o1.as<Reg>().isVec() || !o1.as<Vec>().hasElementIndex())
+          goto InvalidInstruction;
+
+        uint32_t dstIndex = o1.as<Vec>().elementIndex();
+        if (!o0.as<Vec>().hasElementType()) {
+          // DUP - Vec (scalar) <- Vec[N].
+          uint32_t lsbIndex = diff(o0.as<Reg>().type(), RegType::kARM_VecB);
+
+          if (lsbIndex != o1.as<Vec>().elementType() - Vec::kElementTypeB || lsbIndex > 3)
+            goto InvalidInstruction;
+
+          uint32_t imm5 = ((dstIndex << 1) | 1u) << lsbIndex;
+          if (imm5 > 31)
+            goto InvalidElementIndex;
+
+          opcode.reset(0b0101111000000000000001 << 10);
+          opcode.addImm(imm5, 16);
+          goto EmitOp_Rd0_Rn5;
+        }
+        else {
+          // DUP - Vec (all) <- Vec[N].
+          uint32_t elementType = o0.as<Vec>().elementType();
+          if (q > 1 || !Support::bitTest(kValidEncodings, (q << 3) | elementType))
+            goto InvalidInstruction;
+
+          uint32_t lsbIndex = elementType - 1u;
+          uint32_t imm5 = ((dstIndex << 1) | 1u) << lsbIndex;
+
+          if (imm5 > 31)
+            goto InvalidElementIndex;
+
+          opcode.reset(0b0000111000000000000001 << 10);
+          opcode.addImm(q, 30);
+          opcode.addImm(imm5, 16);
+          goto EmitOp_Rd0_Rn5;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdIns: SimdIns: {
+      if (isign4 == ENC_OPS2(Reg, Reg) && o0.as<Reg>().isVecV()) {
+        if (!o0.as<Vec>().hasElementIndex())
+          goto InvalidInstruction;
+
+        uint32_t elementType = o0.as<Vec>().elementType();
+        uint32_t dstIndex = o0.as<Vec>().elementIndex();
+        uint32_t lsbIndex = elementType - 1u;
+
+        uint32_t imm5 = ((dstIndex << 1) | 1u) << lsbIndex;
+        if (imm5 > 31)
+          goto InvalidElementIndex;
+
+        if (o1.as<Reg>().isGp()) {
+          // INS - Vec[N] <- GP register.
+          opcode.reset(0b0100111000000000000111 << 10);
+          opcode.addImm(imm5, 16);
+          goto EmitOp_Rd0_Rn5;
+        }
+        else if (o1.as<Reg>().isVecV() && o1.as<Vec>().hasElementIndex()) {
+          // INS - Vec[N] <- Vec[M].
+          if (o0.as<Vec>().elementType() != o1.as<Vec>().elementType())
+            goto InvalidInstruction;
+
+          uint32_t srcIndex = o1.as<Vec>().elementIndex();
+          if (o0.as<Reg>().type() != o1.as<Reg>().type())
+            goto InvalidInstruction;
+
+          uint32_t imm4 = srcIndex << lsbIndex;
+          if (imm4 > 15)
+            goto InvalidElementIndex;
+
+          opcode.reset(0b0110111000000000000001 << 10);
+          opcode.addImm(imm5, 16);
+          opcode.addImm(imm4, 11);
+          goto EmitOp_Rd0_Rn5;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdMov: {
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        if (o0.as<Reg>().isVec() && o1.as<Reg>().isVec()) {
+          // INS v.x[index], v.x[index].
+          if (o0.as<Vec>().hasElementIndex() && o1.as<Vec>().hasElementIndex())
+            goto SimdIns;
+
+          // DUP {b|h|s|d}, v.{b|h|s|d}[index].
+          if (o1.as<Vec>().hasElementIndex())
+            goto SimdDup;
+
+          if (!checkSignature(o0, o1))
+            goto InvalidInstruction;
+
+          // ORR Vd, Vn, Vm
+          uint32_t q = diff(o0.as<Reg>().type(), RegType::kARM_VecD);
+          if (q > 1)
+            goto InvalidInstruction;
+
+          opcode.reset(0b0000111010100000000111 << 10);
+          opcode.addImm(q, 30);
+          opcode.addReg(o1, 16); // Vn == Vm.
+          goto EmitOp_Rd0_Rn5;
+        }
+
+        if (o0.as<Reg>().isVec() && o1.as<Reg>().isGp()) {
+          // INS v.x[index], Rn.
+          if (o0.as<Vec>().hasElementIndex())
+            goto SimdIns;
+
+          goto InvalidInstruction;
+        }
+
+        if (o0.as<Reg>().isGp() && o1.as<Reg>().isVec()) {
+          // UMOV Rd, V.{s|d}[index].
+          encodingIndex = 1;
+          goto SimdUmov;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdMoviMvni: {
+      const InstDB::EncodingData::SimdMoviMvni& opData = InstDB::EncodingData::simdMoviMvni[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Imm) || isign4 == ENC_OPS3(Reg, Imm, Imm)) {
+        SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_Any, o0.as<Reg>().type(), o0.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        uint64_t imm64 = o1.as<Imm>().valueAs<uint64_t>();
+        uint32_t imm8 = 0;
+        uint32_t cmode = 0;
+        uint32_t inverted = opData.inverted;
+        uint32_t op = 0;
+        uint32_t shift = 0;
+        uint32_t shiftOp = uint32_t(ShiftOp::kLSL);
+
+        if (sizeOp.size() == 3u) {
+          // The second immediate should not be present, however, we accept
+          // an immediate value of zero as some user code may still pass it.
+          if (o2.isImm() && o0.as<Imm>().value() != 0)
+            goto InvalidImmediate;
+
+          if (Utils::isByteMaskImm8(imm64)) {
+            imm8 = encodeImm64ByteMaskToImm8(imm64);
+          }
+          else {
+            // Change from D to S and from 64-bit imm to 32-bit imm if this
+            // is not a byte-mask pattern.
+            if ((imm64 >> 32) == (imm64 & 0xFFFFFFFFu)) {
+              imm64 &= 0xFFFFFFFFu;
+              sizeOp.decrementSize();
+            }
+            else {
+              goto InvalidImmediate;
+            }
+          }
+        }
+
+        if (sizeOp.size() < 3u) {
+          if (imm64 > 0xFFFFFFFFu)
+            goto InvalidImmediate;
+          imm8 = uint32_t(imm64);
+
+          if (sizeOp.size() == 2) {
+            if ((imm8 >> 16) == (imm8 & 0xFFFFu)) {
+              imm8 >>= 16;
+              sizeOp.decrementSize();
+            }
+          }
+
+          if (sizeOp.size() == 1) {
+            if (imm8 > 0xFFFFu)
+              goto InvalidImmediate;
+
+            if ((imm8 >> 8) == (imm8 & 0xFFu)) {
+              imm8 >>= 8;
+              sizeOp.decrementSize();
+            }
+          }
+
+          uint32_t maxShift = (8u << sizeOp.size()) - 8u;
+          if (o2.isImm()) {
+            if (imm8 > 0xFFu || o2.as<Imm>().valueAs<uint64_t>() > maxShift)
+              goto InvalidImmediate;
+
+            shift = o2.as<Imm>().valueAs<uint32_t>();
+            shiftOp = o2.as<Imm>().predicate();
+          }
+          else if (imm8) {
+            shift = Support::ctz(imm8) & ~0x7u;
+            imm8 >>= shift;
+
+            if (imm8 > 0xFFu || shift > maxShift)
+              goto InvalidImmediate;
+          }
+
+          if ((shift & 0x7u) != 0u)
+            goto InvalidImmediate;
+        }
+
+        shift /= 8u;
+
+        switch (sizeOp.size()) {
+          case 0:
+            if (shiftOp != uint32_t(ShiftOp::kLSL))
+              goto InvalidImmediate;
+
+            if (inverted) {
+              imm8 = ~imm8 & 0xFFu;
+              inverted = 0;
+            }
+
+            cmode = B(3) | B(2) | B(1);
+            break;
+
+          case 1:
+            if (shiftOp != uint32_t(ShiftOp::kLSL))
+              goto InvalidImmediate;
+
+            cmode = B(3) | (shift << 1);
+            op = inverted;
+            break;
+
+          case 2:
+            if (shiftOp == uint32_t(ShiftOp::kLSL)) {
+              cmode = shift << 1;
+            }
+            else if (shiftOp == uint32_t(ShiftOp::kMSL)) {
+              if (shift == 0 || shift > 2)
+                goto InvalidImmediate;
+              cmode = B(3) | B(2) | (shift - 1u);
+            }
+            else {
+              goto InvalidImmediate;
+            }
+
+            op = inverted;
+            break;
+
+          case 3:
+            if (inverted) {
+              imm8 = ~imm8 & 0xFFu;
+              inverted = 0;
+            }
+
+            op = 1;
+            cmode = B(3) | B(2) | B(1);
+            break;
+        }
+
+        // The immediate value is split into ABC and DEFGH parts.
+        uint32_t abc = (imm8 >> 5) & 0x7u;
+        uint32_t defgh = imm8 & 0x1Fu;
+
+        opcode.reset(uint32_t(opData.opcode) << 10);
+        opcode.addImm(sizeOp.q(), 30);
+        opcode.addImm(op, 29);
+        opcode.addImm(abc, 16);
+        opcode.addImm(cmode, 12);
+        opcode.addImm(defgh, 5);
+        goto EmitOp_Rd0;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdShift: {
+      const InstDB::EncodingData::SimdShift& opData = InstDB::EncodingData::simdShift[encodingIndex];
+
+      const Operand_& sop = significantSimdOp(o0, o1, instFlags);
+      SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as<Reg>().type(), sop.as<Vec>().elementType());
+
+      if (!sizeOp.isValid())
+        goto InvalidInstruction;
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm) && opData.immediateOp) {
+        if (!matchSignature(o0, o1, instFlags))
+          goto InvalidInstruction;
+
+        if (o2.as<Imm>().valueAs<uint64_t>() > 63)
+          goto InvalidImmediate;
+
+        uint32_t lsbShift = sizeOp.size() + 3u;
+        uint32_t lsbMask = (1u << lsbShift) - 1u;
+        uint32_t imm = o2.as<Imm>().valueAs<uint32_t>();
+
+        // Some instructions use IMM and some X - IMM, so negate if required.
+        if (opData.invertedImm) {
+          if (imm == 0 || imm > (1u << lsbShift))
+            goto InvalidImmediate;
+          imm = Support::neg(imm) & lsbMask;
+        }
+
+        if (imm > lsbMask)
+          goto InvalidImmediate;
+        imm |= (1u << lsbShift);
+
+        opcode.reset(uint32_t(opData.immediateOp) << 10);
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(imm, 16);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg) && opData.registerOp) {
+        if (!matchSignature(o0, o1, o2, instFlags))
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.registerOp) << 10);
+        opcode.addImm(sizeOp.qs(), 30);
+        opcode.addImm(sizeOp.scalar(), 28);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5_Rm16;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdShiftES: {
+      const InstDB::EncodingData::SimdShiftES& opData = InstDB::EncodingData::simdShiftES[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Imm)) {
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as<Reg>().type(), o1.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        if (!matchSignature(o0, o1, instFlags))
+          goto InvalidInstruction;
+
+        // The immediate value must match the element size.
+        uint64_t shift = o2.as<Imm>().valueAs<uint64_t>();
+        uint32_t shiftOp = o2.as<Imm>().predicate();
+
+        if (shift != (8u << sizeOp.size()) || shiftOp != uint32_t(ShiftOp::kLSL))
+          goto InvalidImmediate;
+
+        opcode.reset(uint32_t(opData.opcode) << 10);
+        opcode.addImm(sizeOp.q(), 30);
+        opcode.addImm(sizeOp.size(), 22);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdSm3tt: {
+      const InstDB::EncodingData::SimdSm3tt& opData = InstDB::EncodingData::simdSm3tt[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg)) {
+        if (o0.as<Vec>().isVecS4() && o1.as<Vec>().isVecS4() && o2.as<Vec>().isVecS4() && o2.as<Vec>().hasElementIndex()) {
+          uint32_t imm2 = o2.as<Vec>().elementIndex();
+          if (imm2 > 3)
+            goto InvalidElementIndex;
+
+          opcode.reset(uint32_t(opData.opcode) << 10);
+          opcode.addImm(imm2, 12);
+          goto EmitOp_Rd0_Rn5_Rm16;
+        }
+      }
+
+      break;
+    }
+
+
+    case InstDB::kEncodingSimdSmovUmov: SimdUmov: {
+      const InstDB::EncodingData::SimdSmovUmov& opData = InstDB::EncodingData::simdSmovUmov[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg) && o0.as<Reg>().isGp() && o1.as<Reg>().isVec()) {
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as<Reg>().type(), o1.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        if (!o1.as<Vec>().hasElementIndex())
+          goto InvalidInstruction;
+
+        uint32_t x = o0.as<Gp>().isGpX();
+        uint32_t gpMustBeX = uint32_t(sizeOp.size() >= 3u - opData.isSigned);
+
+        if (opData.isSigned) {
+          if (gpMustBeX && !x)
+            goto InvalidInstruction;
+        }
+        else {
+          if (x != gpMustBeX)
+            goto InvalidInstruction;
+        }
+
+        uint32_t elementIndex = o1.as<Vec>().elementIndex();
+        uint32_t maxElementIndex = 15u >> sizeOp.size();
+
+        if (elementIndex > maxElementIndex)
+          goto InvalidElementIndex;
+
+        uint32_t imm5 = (1u | (elementIndex << 1)) << sizeOp.size();
+
+        opcode.reset(uint32_t(opData.opcode) << 10);
+        opcode.addImm(x, 30);
+        opcode.addImm(imm5, 16);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdSxtlUxtl: {
+      const InstDB::EncodingData::SimdSxtlUxtl& opData = InstDB::EncodingData::simdSxtlUxtl[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Reg)) {
+        SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as<Reg>().type(), o1.as<Vec>().elementType());
+        if (!sizeOp.isValid())
+          goto InvalidInstruction;
+
+        if (!matchSignature(o0, o1, instFlags))
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.opcode) << 10);
+        opcode.addImm(sizeOp.q(), 30);
+        opcode.addImm(1u, sizeOp.size() + 19);
+        goto EmitOp_Rd0_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdTblTbx: {
+      const InstDB::EncodingData::SimdTblTbx& opData = InstDB::EncodingData::simdTblTbx[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Reg) || isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) {
+        // TBL/TBX <Vd>.<Ta>, { <Vn>.16B }, <Vm>.<Ta>
+        // TBL/TBX <Vd>.<Ta>, { <Vn>.16B, <Vn+1>.16B }, <Vm>.<Ta>
+        // TBL/TBX <Vd>.<Ta>, { <Vn>.16B, <Vn+1>.16B, <Vn+2>.16B }, <Vm>.<Ta>
+        // TBL/TBX <Vd>.<Ta>, { <Vn>.16B, <Vn+1>.16B, <Vn+2>.16B, <Vn+3>.16B }, <Vm>.<Ta>
+        opcode.reset(uint32_t(opData.opcode) << 10);
+
+        const Operand_& o4 = opExt[EmitterUtils::kOp4];
+        const Operand_& o5 = opExt[EmitterUtils::kOp5];
+
+        uint32_t q = diff(o0.as<Reg>().type(), RegType::kARM_VecD);
+        if (q > 1 || o0.as<Vec>().hasElementIndex())
+          goto InvalidInstruction;
+
+        if (!o1.as<Vec>().isVecB16() || o1.as<Vec>().hasElementIndex())
+          goto InvalidInstruction;
+
+        uint32_t len = uint32_t(!o3.isNone()) + uint32_t(!o4.isNone()) + uint32_t(!o5.isNone());
+        opcode.addImm(q, 30);
+        opcode.addImm(len, 13);
+
+        switch (len) {
+          case 0:
+            if (!checkSignature(o0, o2))
+              goto InvalidInstruction;
+
+            if (o2.id() > 31)
+              goto InvalidPhysId;
+
+            opcode.addReg(o2, 16);
+            goto EmitOp_Rd0_Rn5;
+
+          case 1:
+            if (!checkSignature(o0, o3))
+              goto InvalidInstruction;
+
+            if (o3.id() > 31)
+              goto InvalidPhysId;
+
+            opcode.addReg(o3, 16);
+            goto EmitOp_Rd0_Rn5;
+
+          case 2:
+            if (!checkSignature(o0, o4))
+              goto InvalidInstruction;
+
+            if (o4.id() > 31)
+              goto InvalidPhysId;
+
+            opcode.addReg(o4, 16);
+            goto EmitOp_Rd0_Rn5;
+
+          case 3:
+            if (!checkSignature(o0, o5))
+              goto InvalidInstruction;
+
+            if (o5.id() > 31)
+              goto InvalidPhysId;
+
+            opcode.addReg(o5, 16);
+            goto EmitOp_Rd0_Rn5;
+
+          default:
+            // Should never happen.
+            goto InvalidInstruction;
+        }
+      }
+
+      break;
+    }
+
+    // ------------------------------------------------------------------------
+    // [Simd - Load / Store]
+    // ------------------------------------------------------------------------
+
+    case InstDB::kEncodingSimdLdSt: {
+      const InstDB::EncodingData::SimdLdSt& opData = InstDB::EncodingData::simdLdSt[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Mem)) {
+        const Mem& m = o1.as<Mem>();
+        rmRel = &m;
+
+        // Width  |       SZ |        XY | XSZ
+        // -------+----------+-----------+-----
+        // 8-bit  | size==00 | opc == 01 | 000
+        // 16-bit | size==01 | opc == 01 | 001
+        // 32-bit | size==10 | opc == 01 | 010
+        // 64-bit | size==11 | opc == 01 | 011
+        // 128-bit| size==00 | opc == 11 | 100
+        uint32_t xsz = diff(o0.as<Reg>().type(), RegType::kARM_VecB);
+        if (xsz > 4u || o0.as<Vec>().hasElementIndex())
+          goto InvalidRegType;
+
+        if (!checkVecId(o0))
+          goto InvalidPhysId;
+
+        if (!armCheckMemBaseIndexRel(m))
+          goto InvalidAddress;
+
+        int64_t offset = m.offset();
+        if (m.hasBaseReg()) {
+          // [Base {Offset | Index}]
+          if (m.hasIndex()) {
+            uint32_t opt = armShiftOpToLdStOptMap[m.predicate()];
+            if (opt == 0xFFu)
+              goto InvalidAddress;
+
+            uint32_t shift = m.shift();
+            uint32_t s = (shift != 0);
+
+            if (s && shift != xsz)
+              goto InvalidAddressScale;
+
+            opcode.reset(uint32_t(opData.registerOp) << 21);
+            opcode.addImm(xsz & 3u, 30);
+            opcode.addImm(xsz >> 2, 23);
+            opcode.addImm(opt, 13);
+            opcode.addImm(s, 12);
+            opcode |= B(11);
+            opcode.addReg(o0, 0);
+            goto EmitOp_MemBaseIndex_Rn5_Rm16;
+          }
+
+          // Makes it easier to work with the offset especially on 32-bit arch.
+          if (!Support::isInt32(offset))
+            goto InvalidDisplacement;
+          int32_t offset32 = int32_t(offset);
+
+          if (m.isPreOrPost()) {
+            if (!Support::isInt9(offset32))
+              goto InvalidDisplacement;
+
+            opcode.reset(uint32_t(opData.prePostOp) << 21);
+            opcode.addImm(xsz & 3u, 30);
+            opcode.addImm(xsz >> 2, 23);
+            opcode.addImm(offset32 & 0x1FF, 12);
+            opcode.addImm(m.isPreIndex(), 11);
+            opcode |= B(10);
+            opcode.addReg(o0, 0);
+            goto EmitOp_MemBase_Rn5;
+          }
+          else {
+            uint32_t imm12 = uint32_t(offset32) >> xsz;
+
+            // If this instruction is not encodable with scaled unsigned offset, try unscaled signed offset.
+            if (!Support::isUInt12(imm12) || (imm12 << xsz) != uint32_t(offset32)) {
+              instId = opData.uAltInstId;
+              instInfo = &InstDB::_instInfoTable[instId];
+              encodingIndex = instInfo->_encodingDataIndex;
+              goto Case_SimdLdurStur;
+            }
+
+            opcode.reset(uint32_t(opData.uOffsetOp) << 22);
+            opcode.addImm(xsz & 3u, 30);
+            opcode.addImm(xsz >> 2, 23);
+            opcode.addImm(imm12, 10);
+            opcode.addReg(o0, 0);
+            goto EmitOp_MemBase_Rn5;
+          }
+        }
+        else {
+          if (!opData.literalOp)
+            goto InvalidAddress;
+
+          if (xsz < 2u)
+            goto InvalidRegType;
+
+          uint32_t opc = xsz - 2u;
+          opcode.reset(uint32_t(opData.literalOp) << 24);
+          opcode.addImm(opc, 30);
+          opcode.addReg(o0, 0);
+          offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 5, 19, 2);
+          goto EmitOp_Rel;
+        }
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdLdpStp: {
+      const InstDB::EncodingData::SimdLdpStp& opData = InstDB::EncodingData::simdLdpStp[encodingIndex];
+
+      if (isign4 == ENC_OPS3(Reg, Reg, Mem)) {
+        const Mem& m = o2.as<Mem>();
+        rmRel = &m;
+
+        uint32_t opc = diff(o0.as<Reg>().type(), RegType::kARM_VecS);
+        if (opc > 2u || o0.as<Vec>().hasElementTypeOrIndex())
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1))
+          goto InvalidInstruction;
+
+        if (!checkVecId(o0, o1))
+          goto InvalidPhysId;
+
+        if (m.baseType() != RegType::kARM_GpX || m.hasIndex())
+          goto InvalidAddress;
+
+        if (m.isOffset64Bit())
+          goto InvalidDisplacement;
+
+        uint32_t offsetShift = 2u + opc;
+        int32_t offset32 = m.offsetLo32() >> offsetShift;
+
+        // Make sure we didn't lose bits by applying the mandatory offset shift.
+        if (Support::shl(offset32, offsetShift) != m.offsetLo32())
+          goto InvalidDisplacement;
+
+        // Offset is encoded as a 7-bit immediate.
+        if (!Support::isInt7(offset32))
+          goto InvalidDisplacement;
+
+        if (m.isPreOrPost() && offset32 != 0) {
+          if (!opData.prePostOp)
+            goto InvalidAddress;
+
+          opcode.reset(uint32_t(opData.prePostOp) << 22);
+          opcode.addImm(m.isPreIndex(), 24);
+        }
+        else {
+          opcode.reset(uint32_t(opData.offsetOp) << 22);
+        }
+
+        opcode.addImm(opc, 30);
+        opcode.addImm(offset32 & 0x7F, 15);
+        opcode.addReg(o1, 10);
+        opcode.addReg(o0, 0);
+        goto EmitOp_MemBase_Rn5;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdLdurStur: {
+Case_SimdLdurStur:
+      const InstDB::EncodingData::SimdLdurStur& opData = InstDB::EncodingData::simdLdurStur[encodingIndex];
+
+      if (isign4 == ENC_OPS2(Reg, Mem)) {
+        const Mem& m = o1.as<Mem>();
+        rmRel = &m;
+
+        uint32_t sz = diff(o0.as<Reg>().type(), RegType::kARM_VecB);
+        if (sz > 4 || o0.as<Vec>().hasElementTypeOrIndex())
+          goto InvalidInstruction;
+
+        if (!checkVecId(o0))
+          goto InvalidPhysId;
+
+        if (!armCheckMemBaseIndexRel(m))
+          goto InvalidAddress;
+
+        if (m.hasBaseReg() && !m.hasIndex() && !m.isPreOrPost()) {
+          if (m.isOffset64Bit())
+            goto InvalidDisplacement;
+
+          int32_t offset32 = m.offsetLo32();
+          if (!Support::isInt9(offset32))
+            goto InvalidDisplacement;
+
+          opcode.reset(uint32_t(opData.opcode) << 10);
+          opcode.addImm(sz & 3u, 30);
+          opcode.addImm(sz >> 2, 23);
+          opcode.addImm(offset32 & 0x1FF, 12);
+          opcode.addReg(o0, 0);
+          goto EmitOp_MemBase_Rn5;
+        }
+
+        goto InvalidAddress;
+      }
+
+      break;
+    }
+
+    case InstDB::kEncodingSimdLdNStN: {
+      const InstDB::EncodingData::SimdLdNStN& opData = InstDB::EncodingData::simdLdNStN[encodingIndex];
+      const Operand_& o4 = opExt[EmitterUtils::kOp4];
+
+      uint32_t n = 1;
+
+      if (isign4 == ENC_OPS2(Reg, Mem)) {
+        if (opData.n != 1)
+          goto InvalidInstruction;
+
+        rmRel = &o1;
+      }
+      else if (isign4 == ENC_OPS3(Reg, Reg, Mem)) {
+        if (opData.n != 1 && opData.n != 2)
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1) || !checkConsecutive(o0, o1))
+          goto InvalidInstruction;
+
+        n = 2;
+        rmRel = &o2;
+      }
+      else if (isign4 == ENC_OPS4(Reg, Reg, Reg, Mem) && o4.isNone()) {
+        if (opData.n != 1 && opData.n != 3)
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1, o2) || !checkConsecutive(o0, o1, o2))
+          goto InvalidInstruction;
+
+        n = 3;
+        rmRel = &o3;
+      }
+      else if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg) && o4.isMem()) {
+        if (opData.n != 1 && opData.n != 4)
+          goto InvalidInstruction;
+
+        if (!checkSignature(o0, o1, o2, o3) || !checkConsecutive(o0, o1, o2, o3))
+          goto InvalidInstruction;
+
+        n = 4;
+        rmRel = &o4;
+      }
+      else {
+        goto InvalidInstruction;
+      }
+
+      // We will use `v` and `m` from now as those are relevant for encoding.
+      const Vec& v = o0.as<Vec>();
+      const Mem& m = rmRel->as<Mem>();
+
+      uint32_t q = 0;
+      uint32_t rm = 0;
+      uint32_t rn = m.baseId();
+      uint32_t sz = v.elementType() - Vec::kElementTypeB;
+      uint32_t opcSsize = sz;
+      uint32_t offsetPossibility = 0;
+
+      if (sz > 3)
+        goto InvalidInstruction;
+
+      if (m.baseType() != RegType::kARM_GpX)
+        goto InvalidAddress;
+
+      // Rn cannot be ZR, but can be SP.
+      if (rn > 30 && rn != Gp::kIdSp)
+        goto InvalidAddress;
+
+      rn &= 31;
+
+      if (opData.replicate) {
+        if (n != opData.n)
+          goto InvalidInstruction;
+
+        // Replicates to the whole register, element index cannot be used.
+        if (v.hasElementIndex())
+          goto InvalidInstruction;
+
+        q = diff(v.type(), RegType::kARM_VecD);
+        if (q > 1)
+          goto InvalidInstruction;
+
+        opcode.reset(uint32_t(opData.singleOp) << 10);
+        offsetPossibility = (1u << sz) * n;
+      }
+      else if (v.hasElementIndex()) {
+        if (n != opData.n)
+          goto InvalidInstruction;
+
+        // LDx/STx (single structure).
+        static const uint8_t opcSsizeBySzS[] = { 0x0u << 3, 0x2u << 3, 0x4u << 3, (0x4u << 3) | 1u };
+
+        opcode.reset(uint32_t(opData.singleOp) << 10);
+        opcSsize = opcSsizeBySzS[sz];
+        offsetPossibility =  (1u << sz) * opData.n;
+
+        uint32_t elementIndex = v.elementIndex();
+        uint32_t maxElementIndex = 15 >> sz;
+
+        if (elementIndex > maxElementIndex)
+          goto InvalidElementIndex;
+
+        elementIndex <<= sz;
+        q = elementIndex >> 3;
+        opcSsize |= elementIndex & 0x7u;
+      }
+      else {
+        // LDx/STx (multiple structures).
+        static const uint8_t opcSsizeByN[] = { 0u, 0x7u << 2, 0xAu << 2, 0x6u << 2, 0x2u << 2 };
+
+        q = diff(v.type(), RegType::kARM_VecD);
+        if (q > 1)
+          goto InvalidInstruction;
+
+        if (opData.n == 1)
+          opcSsize |= opcSsizeByN[n];
+
+        opcode.reset(uint32_t(opData.multipleOp) << 10);
+        offsetPossibility = (8u << q) * n;
+      }
+
+      if (m.hasIndex()) {
+        if (m.hasOffset() || !m.isPostIndex())
+          goto InvalidAddress;
+
+        rm = m.indexId();
+        if (rm > 30)
+          goto InvalidAddress;
+
+        // Bit 23 - PostIndex.
+        opcode |= B(23);
+      }
+      else {
+        if (m.hasOffset()) {
+          if (m.offset() != int32_t(offsetPossibility) || !m.isPostIndex())
+            goto InvalidAddress;
+          rm = 31;
+
+          // Bit 23 - PostIndex.
+          opcode |= B(23);
+        }
+      }
+
+      opcode.addImm(q, 30);
+      opcode.addImm(rm, 16);
+      opcode.addImm(opcSsize, 10);
+      opcode.addImm(rn, 5);
+      goto EmitOp_Rd0;
+    }
+
+    default:
+      break;
+  }
+
+  goto InvalidInstruction;
+
+  // --------------------------------------------------------------------------
+  // [EmitGp - Single]
+  // --------------------------------------------------------------------------
+
+EmitOp_Rd0:
+  if (!checkValidRegs(o0))
+    goto InvalidPhysId;
+
+  opcode.addReg(o0, 0);
+  goto EmitOp;
+
+EmitOp_Rn5:
+  if (!checkValidRegs(o0))
+    goto InvalidPhysId;
+
+  opcode.addReg(o0, 5);
+  goto EmitOp;
+
+EmitOp_Rn5_Rm16:
+  if (!checkValidRegs(o0, o1))
+    goto InvalidPhysId;
+
+  opcode.addReg(o0, 5);
+  opcode.addReg(o1, 16);
+  goto EmitOp;
+
+EmitOp_Rd0_Rn5:
+  if (!checkValidRegs(o0, o1))
+    goto InvalidPhysId;
+
+  opcode.addReg(o0, 0);
+  opcode.addReg(o1, 5);
+  goto EmitOp;
+
+EmitOp_Rd0_Rn5_Rm16_Ra10:
+  if (!checkValidRegs(o0, o1, o2, o3))
+    goto InvalidPhysId;
+
+  opcode.addReg(o0, 0);
+  opcode.addReg(o1, 5);
+  opcode.addReg(o2, 16);
+  opcode.addReg(o3, 10);
+  goto EmitOp;
+
+EmitOp_Rd0_Rn5_Rm16:
+  if (!checkValidRegs(o0, o1, o3))
+    goto InvalidPhysId;
+
+  opcode.addReg(o0, 0);
+  opcode.addReg(o1, 5);
+  opcode.addReg(o2, 16);
+  goto EmitOp;
+
+  // --------------------------------------------------------------------------
+  // [EmitGp - Multiple]
+  // --------------------------------------------------------------------------
+
+EmitOp_Multiple:
+  {
+    ASMJIT_ASSERT(multipleOpCount > 0);
+    err = writer.ensureSpace(this, multipleOpCount * 4u);
+    if (ASMJIT_UNLIKELY(err))
+      goto Failed;
+
+    for (uint32_t i = 0; i < multipleOpCount; i++)
+      writer.emit32uLE(multipleOpData[i]);
+
+    goto EmitDone;
+  }
+
+  // --------------------------------------------------------------------------
+  // [EmitGp - Memory]
+  // --------------------------------------------------------------------------
+
+EmitOp_MemBase_Rn5:
+  if (!checkMemBase(rmRel->as<Mem>()))
+    goto InvalidAddress;
+
+  opcode.addReg(rmRel->as<Mem>().baseId(), 5);
+  goto EmitOp;
+
+EmitOp_MemBaseNoImm_Rn5:
+  if (!checkMemBase(rmRel->as<Mem>()) || rmRel->as<Mem>().hasIndex())
+    goto InvalidAddress;
+
+  if (rmRel->as<Mem>().hasOffset())
+    goto InvalidDisplacement;
+
+  opcode.addReg(rmRel->as<Mem>().baseId(), 5);
+  goto EmitOp;
+
+EmitOp_MemBaseIndex_Rn5_Rm16:
+  if (!rmRel->as<Mem>().hasBaseReg())
+    goto InvalidAddress;
+
+  if (rmRel->as<Mem>().indexId() > 30 && rmRel->as<Mem>().indexId() != Gp::kIdZr)
+    goto InvalidPhysId;
+
+  opcode.addReg(rmRel->as<Mem>().indexId(), 16);
+  opcode.addReg(rmRel->as<Mem>().baseId(), 5);
+  goto EmitOp;
+
+  // --------------------------------------------------------------------------
+  // [EmitOp - PC Relative]
+  // --------------------------------------------------------------------------
+
+EmitOp_Rel:
+  {
+    if (rmRel->isLabel() || rmRel->isMem()) {
+      uint32_t labelId;
+      int64_t labelOffset = 0;
+
+      if (rmRel->isLabel()) {
+        labelId = rmRel->as<Label>().id();
+      }
+      else {
+        labelId = rmRel->as<Mem>().baseId();
+        labelOffset = rmRel->as<Mem>().offset();
+      }
+
+      LabelEntry* label = _code->labelEntry(labelId);
+      if (ASMJIT_UNLIKELY(!label))
+        goto InvalidLabel;
+
+      if (offsetFormat.type() == OffsetType::kAArch64_ADRP) {
+        // TODO: [ARM] Always create relocation entry.
+      }
+
+      if (label->isBoundTo(_section)) {
+        // Label bound to the current section.
+        offsetValue = label->offset() - uint64_t(offset()) + uint64_t(labelOffset);
+        goto EmitOp_DispImm;
+      }
+      else {
+        // Record non-bound label.
+        size_t codeOffset = writer.offsetFrom(_bufferData);
+        LabelLink* link = _code->newLabelLink(label, _section->id(), codeOffset, intptr_t(labelOffset), offsetFormat);
+
+        if (ASMJIT_UNLIKELY(!link))
+          goto OutOfMemory;
+
+        goto EmitOp;
+      }
+    }
+  }
+
+  if (rmRel->isImm()) {
+    uint64_t baseAddress = _code->baseAddress();
+    uint64_t targetOffset = rmRel->as<Imm>().valueAs<uint64_t>();
+
+    size_t codeOffset = writer.offsetFrom(_bufferData);
+
+    if (baseAddress == Globals::kNoBaseAddress || _section->id() != 0) {
+      // Create a new RelocEntry as we cannot calculate the offset right now.
+      RelocEntry* re;
+      err = _code->newRelocEntry(&re, RelocType::kAbsToRel);
+      if (err)
+        goto Failed;
+
+      re->_sourceSectionId = _section->id();
+      re->_sourceOffset = codeOffset;
+      re->_format = offsetFormat;
+      re->_payload = rmRel->as<Imm>().valueAs<uint64_t>() + 4u;
+      goto EmitOp;
+    }
+    else {
+      uint64_t pc = baseAddress + codeOffset;
+
+      if (offsetFormat.type() == OffsetType::kAArch64_ADRP)
+        pc &= ~uint64_t(4096 - 1);
+
+      offsetValue = targetOffset - pc;
+      goto EmitOp_DispImm;
+    }
+  }
+
+  goto InvalidInstruction;
+
+EmitOp_DispImm:
+  {
+    if ((offsetValue & Support::lsbMask<uint32_t>(offsetFormat.immDiscardLsb())) != 0)
+      goto InvalidDisplacement;
+
+    int64_t dispImm64 = int64_t(offsetValue) >> offsetFormat.immDiscardLsb();
+    if (!Support::isEncodableOffset64(dispImm64, offsetFormat.immBitCount()))
+      goto InvalidDisplacement;
+
+    uint32_t dispImm32 = uint32_t(dispImm64 & Support::lsbMask<uint32_t>(offsetFormat.immBitCount()));
+    switch (offsetFormat.type()) {
+      case OffsetType::kSignedOffset: {
+        opcode.addImm(dispImm32, offsetFormat.immBitShift());
+        goto EmitOp;
+      }
+
+      case OffsetType::kAArch64_ADR:
+      case OffsetType::kAArch64_ADRP: {
+        uint32_t immLo = dispImm32 & 0x3u;
+        uint32_t immHi = dispImm32 >> 2;
+        opcode.addImm(immLo, 29);
+        opcode.addImm(immHi, 5);
+        goto EmitOp;
+      }
+
+      default:
+        goto InvalidDisplacement;
+    }
+  }
+
+  // --------------------------------------------------------------------------
+  // [EmitOp - Opcode]
+  // --------------------------------------------------------------------------
+
+EmitOp:
+  writer.emit32uLE(opcode.get());
+  goto EmitDone;
+
+  // --------------------------------------------------------------------------
+  // [Done]
+  // --------------------------------------------------------------------------
+
+EmitDone:
+  if (Support::test(options, InstOptions::kReserved)) {
+#ifndef ASMJIT_NO_LOGGING
+    if (_logger)
+      EmitterUtils::logInstructionEmitted(this, instId, options, o0, o1, o2, opExt, 0, 0, writer.cursor());
+#endif
+  }
+
+  resetExtraReg();
+  resetInstOptions();
+  resetInlineComment();
+
+  writer.done(this);
+  return kErrorOk;
+
+  // --------------------------------------------------------------------------
+  // [Error Handler]
+  // --------------------------------------------------------------------------
+
+#define ERROR_HANDLER(ERR) ERR: err = DebugUtils::errored(kError##ERR); goto Failed;
+  ERROR_HANDLER(OutOfMemory)
+  ERROR_HANDLER(InvalidAddress)
+  ERROR_HANDLER(InvalidAddressScale)
+  ERROR_HANDLER(InvalidDisplacement)
+  ERROR_HANDLER(InvalidElementIndex)
+  ERROR_HANDLER(InvalidLabel)
+  ERROR_HANDLER(InvalidImmediate)
+  ERROR_HANDLER(InvalidInstruction)
+  ERROR_HANDLER(InvalidPhysId)
+  ERROR_HANDLER(InvalidRegType)
+#undef ERROR_HANDLER
+
+Failed:
+#ifndef ASMJIT_NO_LOGGING
+  return EmitterUtils::logInstructionFailed(this, err, instId, options, o0, o1, o2, opExt);
+#else
+  resetExtraReg();
+  resetInstOptions();
+  resetInlineComment();
+  return reportError(err);
+#endif
+}
+
+#undef ENC_OPS1
+#undef ENC_OPS2
+#undef ENC_OPS3
+#undef ENC_OPS4
+
+// a64::Assembler - Align
+// ======================
+
+Error Assembler::align(AlignMode alignMode, uint32_t alignment) {
+  constexpr uint32_t kNopA64 = 0xD503201Fu; // [11010101|00000011|00100000|00011111].
+
+  if (ASMJIT_UNLIKELY(!_code))
+    return reportError(DebugUtils::errored(kErrorNotInitialized));
+
+  if (ASMJIT_UNLIKELY(uint32_t(alignMode) > uint32_t(AlignMode::kMaxValue)))
+    return reportError(DebugUtils::errored(kErrorInvalidArgument));
+
+  if (alignment <= 1)
+    return kErrorOk;
+
+  if (ASMJIT_UNLIKELY(alignment > Globals::kMaxAlignment || !Support::isPowerOf2(alignment)))
+    return reportError(DebugUtils::errored(kErrorInvalidArgument));
+
+  uint32_t i = uint32_t(Support::alignUpDiff<size_t>(offset(), alignment));
+  if (i == 0)
+    return kErrorOk;
+
+  CodeWriter writer(this);
+  ASMJIT_PROPAGATE(writer.ensureSpace(this, i));
+
+  switch (alignMode) {
+    case AlignMode::kCode: {
+      uint32_t pattern = kNopA64;
+
+      if (ASMJIT_UNLIKELY(offset() & 0x3u))
+        return DebugUtils::errored(kErrorInvalidState);
+
+      while (i >= 4) {
+        writer.emit32uLE(pattern);
+        i -= 4;
+      }
+
+      ASMJIT_ASSERT(i == 0);
+      break;
+    }
+
+    case AlignMode::kData:
+    case AlignMode::kZero:
+      writer.emitZeros(i);
+      break;
+  }
+
+  writer.done(this);
+
+#ifndef ASMJIT_NO_LOGGING
+  if (_logger) {
+    StringTmp<128> sb;
+    sb.appendChars(' ', _logger->indentation(FormatIndentationGroup::kCode));
+    sb.appendFormat("align %u\n", alignment);
+    _logger->log(sb);
+  }
+#endif
+
+  return kErrorOk;
+}
+
+// a64::Assembler - Events
+// =======================
+
+Error Assembler::onAttach(CodeHolder* code) noexcept {
+  ASMJIT_PROPAGATE(Base::onAttach(code));
+  return kErrorOk;
+}
+
+Error Assembler::onDetach(CodeHolder* code) noexcept {
+  return Base::onDetach(code);
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64

libraries/asmjit/asmjit/arm/a64assembler.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64ASSEMBLER_H_INCLUDED
+#define ASMJIT_ARM_A64ASSEMBLER_H_INCLUDED
+
+#include "../core/assembler.h"
+#include "../arm/a64emitter.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \addtogroup asmjit_a64
+//! \{
+
+//! AArch64 assembler implementation.
+class ASMJIT_VIRTAPI Assembler
+  : public BaseAssembler,
+    public EmitterExplicitT<Assembler> {
+
+public:
+  typedef BaseAssembler Base;
+
+  //! \name Construction / Destruction
+  //! \{
+
+  ASMJIT_API Assembler(CodeHolder* code = nullptr) noexcept;
+  ASMJIT_API virtual ~Assembler() noexcept;
+
+  //! \}
+
+  //! \name Accessors
+  //! \{
+
+  //! Gets whether the current ARM mode is THUMB (alternative to 32-bit ARM encoding).
+  inline bool isInThumbMode() const noexcept { return _environment.isArchThumb(); }
+
+  //! Gets the current code alignment of the current mode (ARM vs THUMB).
+  inline uint32_t codeAlignment() const noexcept { return isInThumbMode() ? 2 : 4; }
+
+  //! \}
+
+  //! \name Emit
+  //! \{
+
+  ASMJIT_API Error _emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) override;
+
+  //! \}
+
+  //! \name Align
+  //! \{
+
+  ASMJIT_API Error align(AlignMode alignMode, uint32_t alignment) override;
+
+  //! \}
+
+  //! \name Events
+  //! \{
+
+  ASMJIT_API Error onAttach(CodeHolder* code) noexcept override;
+  ASMJIT_API Error onDetach(CodeHolder* code) noexcept override;
+
+  //! \}
+};
+
+//! \}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_A64ASSEMBLER_H_INCLUDED

libraries/asmjit/asmjit/arm/a64builder.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64) && !defined(ASMJIT_NO_BUILDER)
+
+#include "../arm/a64assembler.h"
+#include "../arm/a64builder.h"
+#include "../arm/a64emithelper_p.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::Builder - Construction & Destruction
+// =========================================
+
+Builder::Builder(CodeHolder* code) noexcept : BaseBuilder() {
+  _archMask = uint64_t(1) << uint32_t(Arch::kAArch64);
+  assignEmitterFuncs(this);
+
+  if (code)
+    code->attach(this);
+}
+Builder::~Builder() noexcept {}
+
+// a64::Builder - Events
+// =====================
+
+Error Builder::onAttach(CodeHolder* code) noexcept {
+  return Base::onAttach(code);
+}
+
+Error Builder::onDetach(CodeHolder* code) noexcept {
+  return Base::onDetach(code);
+}
+
+// a64::Builder - Finalize
+// =======================
+
+Error Builder::finalize() {
+  ASMJIT_PROPAGATE(runPasses());
+  Assembler a(_code);
+  a.addEncodingOptions(encodingOptions());
+  a.addDiagnosticOptions(diagnosticOptions());
+  return serializeTo(&a);
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64 && !ASMJIT_NO_BUILDER

libraries/asmjit/asmjit/arm/a64builder.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64BUILDER_H_INCLUDED
+#define ASMJIT_ARM_A64BUILDER_H_INCLUDED
+
+#include "../core/api-config.h"
+#ifndef ASMJIT_NO_BUILDER
+
+#include "../core/builder.h"
+#include "../arm/a64emitter.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \addtogroup asmjit_a64
+//! \{
+
+//! AArch64 builder implementation.
+class ASMJIT_VIRTAPI Builder
+  : public BaseBuilder,
+    public EmitterExplicitT<Builder> {
+public:
+  ASMJIT_NONCOPYABLE(Builder)
+  typedef BaseBuilder Base;
+
+  //! \name Construction & Destruction
+  //! \{
+
+  ASMJIT_API explicit Builder(CodeHolder* code = nullptr) noexcept;
+  ASMJIT_API virtual ~Builder() noexcept;
+
+  //! \}
+
+  //! \name Events
+  //! \{
+
+  ASMJIT_API Error onAttach(CodeHolder* code) noexcept override;
+  ASMJIT_API Error onDetach(CodeHolder* code) noexcept override;
+
+  //! \}
+
+  //! \name Finalize
+  //! \{
+
+  ASMJIT_API Error finalize() override;
+
+  //! \}
+};
+
+//! \}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_BUILDER
+#endif // ASMJIT_ARM_A64BUILDER_H_INCLUDED

libraries/asmjit/asmjit/arm/a64compiler.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64) && !defined(ASMJIT_NO_COMPILER)
+
+#include "../arm/a64assembler.h"
+#include "../arm/a64compiler.h"
+#include "../arm/a64emithelper_p.h"
+#include "../arm/a64rapass_p.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::Compiler - Construction & Destruction
+// ==========================================
+
+Compiler::Compiler(CodeHolder* code) noexcept : BaseCompiler() {
+  _archMask = uint64_t(1) << uint32_t(Arch::kAArch64);
+  assignEmitterFuncs(this);
+
+  if (code)
+    code->attach(this);
+}
+Compiler::~Compiler() noexcept {}
+
+// a64::Compiler - Events
+// ======================
+
+Error Compiler::onAttach(CodeHolder* code) noexcept {
+  ASMJIT_PROPAGATE(Base::onAttach(code));
+  Error err = addPassT<ARMRAPass>();
+
+  if (ASMJIT_UNLIKELY(err)) {
+    onDetach(code);
+    return err;
+  }
+
+  return kErrorOk;
+}
+
+Error Compiler::onDetach(CodeHolder* code) noexcept {
+  return Base::onDetach(code);
+}
+
+// a64::Compiler - Finalize
+// ========================
+
+Error Compiler::finalize() {
+  ASMJIT_PROPAGATE(runPasses());
+  Assembler a(_code);
+  a.addEncodingOptions(encodingOptions());
+  a.addDiagnosticOptions(diagnosticOptions());
+  return serializeTo(&a);
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64 && !ASMJIT_NO_COMPILER

libraries/asmjit/asmjit/arm/a64compiler.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_ARMCOMPILER_H_INCLUDED
+#define ASMJIT_ARM_ARMCOMPILER_H_INCLUDED
+
+#include "../core/api-config.h"
+#ifndef ASMJIT_NO_COMPILER
+
+#include "../core/compiler.h"
+#include "../core/type.h"
+#include "../arm/a64emitter.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \addtogroup asmjit_a64
+//! \{
+
+//! AArch64 compiler implementation.
+class ASMJIT_VIRTAPI Compiler
+  : public BaseCompiler,
+    public EmitterExplicitT<Compiler> {
+public:
+  ASMJIT_NONCOPYABLE(Compiler)
+  typedef BaseCompiler Base;
+
+  //! \name Construction & Destruction
+  //! \{
+
+  ASMJIT_API explicit Compiler(CodeHolder* code = nullptr) noexcept;
+  ASMJIT_API virtual ~Compiler() noexcept;
+
+  //! \}
+
+  //! \name Virtual Registers
+  //! \{
+
+  //! \cond INTERNAL
+  template<typename RegT, typename Type>
+  inline RegT _newRegInternal(const Type& type) {
+    RegT reg(Globals::NoInit);
+    _newReg(&reg, type, nullptr);
+    return reg;
+  }
+
+  template<typename RegT, typename Type, typename... Args>
+  inline RegT _newRegInternal(const Type& type, const char* s, Args&&... args) {
+#ifndef ASMJIT_NO_LOGGING
+    RegT reg(Globals::NoInit);
+    if (sizeof...(Args) == 0)
+      _newReg(&reg, type, s);
+    else
+      _newRegFmt(&reg, type, s, std::forward<Args>(args)...);
+    return reg;
+#else
+    DebugUtils::unused(std::forward<Args>(args)...);
+    RegT reg(Globals::NoInit);
+    _newReg(&reg, type, nullptr);
+    return reg;
+#endif
+  }
+  //! \endcond
+
+  template<typename RegT, typename... Args>
+  inline RegT newSimilarReg(const RegT& ref, Args&&... args) {
+    return _newRegInternal<RegT>(ref, std::forward<Args>(args)...);
+  }
+
+  template<typename... Args>
+  inline Reg newReg(TypeId typeId, Args&&... args) { return _newRegInternal<Reg>(typeId, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newGp(TypeId typeId, Args&&... args) { return _newRegInternal<Gp>(typeId, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Vec newVec(TypeId typeId, Args&&... args) { return _newRegInternal<Vec>(typeId, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newInt32(Args&&... args) { return _newRegInternal<Gp>(TypeId::kInt32, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newUInt32(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUInt32, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newInt64(Args&&... args) { return _newRegInternal<Gp>(TypeId::kInt64, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newUInt64(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUInt64, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newIntPtr(Args&&... args) { return _newRegInternal<Gp>(TypeId::kIntPtr, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newUIntPtr(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUIntPtr, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newGpw(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUInt32, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newGpx(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUInt64, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newGpz(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUIntPtr, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Vec newVecS(Args&&... args) { return _newRegInternal<Vec>(TypeId::kFloat32, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Vec newVecD(Args&&... args) { return _newRegInternal<Vec>(TypeId::kFloat64, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Vec newVecQ(Args&&... args) { return _newRegInternal<Vec>(TypeId::kUInt8x16, std::forward<Args>(args)...); }
+
+  //! \}
+
+  //! \name Stack
+  //! \{
+
+  //! Creates a new memory chunk allocated on the current function's stack.
+  inline Mem newStack(uint32_t size, uint32_t alignment, const char* name = nullptr) {
+    Mem m(Globals::NoInit);
+    _newStack(&m, size, alignment, name);
+    return m;
+  }
+
+  //! \}
+
+  //! \name Constants
+  //! \{
+
+  //! Put data to a constant-pool and get a memory reference to it.
+  inline Mem newConst(ConstPoolScope scope, const void* data, size_t size) {
+    Mem m(Globals::NoInit);
+    _newConst(&m, scope, data, size);
+    return m;
+  }
+
+  //! Put a BYTE `val` to a constant-pool (8 bits).
+  inline Mem newByteConst(ConstPoolScope scope, uint8_t val) noexcept { return newConst(scope, &val, 1); }
+  //! Put a HWORD `val` to a constant-pool (16 bits).
+  inline Mem newHWordConst(ConstPoolScope scope, uint16_t val) noexcept { return newConst(scope, &val, 2); }
+  //! Put a WORD `val` to a constant-pool (32 bits).
+  inline Mem newWordConst(ConstPoolScope scope, uint32_t val) noexcept { return newConst(scope, &val, 4); }
+  //! Put a DWORD `val` to a constant-pool (64 bits).
+  inline Mem newDWordConst(ConstPoolScope scope, uint64_t val) noexcept { return newConst(scope, &val, 8); }
+
+  //! Put a WORD `val` to a constant-pool.
+  inline Mem newInt16Const(ConstPoolScope scope, int16_t val) noexcept { return newConst(scope, &val, 2); }
+  //! Put a WORD `val` to a constant-pool.
+  inline Mem newUInt16Const(ConstPoolScope scope, uint16_t val) noexcept { return newConst(scope, &val, 2); }
+  //! Put a DWORD `val` to a constant-pool.
+  inline Mem newInt32Const(ConstPoolScope scope, int32_t val) noexcept { return newConst(scope, &val, 4); }
+  //! Put a DWORD `val` to a constant-pool.
+  inline Mem newUInt32Const(ConstPoolScope scope, uint32_t val) noexcept { return newConst(scope, &val, 4); }
+  //! Put a QWORD `val` to a constant-pool.
+  inline Mem newInt64Const(ConstPoolScope scope, int64_t val) noexcept { return newConst(scope, &val, 8); }
+  //! Put a QWORD `val` to a constant-pool.
+  inline Mem newUInt64Const(ConstPoolScope scope, uint64_t val) noexcept { return newConst(scope, &val, 8); }
+
+  //! Put a SP-FP `val` to a constant-pool.
+  inline Mem newFloatConst(ConstPoolScope scope, float val) noexcept { return newConst(scope, &val, 4); }
+  //! Put a DP-FP `val` to a constant-pool.
+  inline Mem newDoubleConst(ConstPoolScope scope, double val) noexcept { return newConst(scope, &val, 8); }
+
+  //! \}
+
+  //! \name Instruction Options
+  //! \{
+
+  //! Force the compiler to not follow the conditional or unconditional jump.
+  inline Compiler& unfollow() noexcept { _instOptions |= InstOptions::kUnfollow; return *this; }
+
+  //! \}
+
+  //! \name Function Call & Ret Intrinsics
+  //! \{
+
+  //! Invoke a function call without `target` type enforcement.
+  inline Error invoke_(InvokeNode** out, const Operand_& target, const FuncSignature& signature) {
+    return addInvokeNode(out, Inst::kIdBlr, target, signature);
+  }
+
+  //! Invoke a function call of the given `target` and `signature` and store the added node to `out`.
+  //!
+  //! Creates a new \ref InvokeNode, initializes all the necessary members to match the given function `signature`,
+  //! adds the node to the compiler, and stores its pointer to `out`. The operation is atomic, if anything fails
+  //! nullptr is stored in `out` and error code is returned.
+  inline Error invoke(InvokeNode** out, const Gp& target, const FuncSignature& signature) { return invoke_(out, target, signature); }
+  //! \overload
+  inline Error invoke(InvokeNode** out, const Mem& target, const FuncSignature& signature) { return invoke_(out, target, signature); }
+  //! \overload
+  inline Error invoke(InvokeNode** out, const Label& target, const FuncSignature& signature) { return invoke_(out, target, signature); }
+  //! \overload
+  inline Error invoke(InvokeNode** out, const Imm& target, const FuncSignature& signature) { return invoke_(out, target, signature); }
+  //! \overload
+  inline Error invoke(InvokeNode** out, uint64_t target, const FuncSignature& signature) { return invoke_(out, Imm(int64_t(target)), signature); }
+
+  //! Return.
+  inline Error ret() { return addRet(Operand(), Operand()); }
+  //! \overload
+  inline Error ret(const BaseReg& o0) { return addRet(o0, Operand()); }
+  //! \overload
+  inline Error ret(const BaseReg& o0, const BaseReg& o1) { return addRet(o0, o1); }
+
+  //! \}
+
+  //! \name Jump Tables Support
+  //! \{
+
+  using EmitterExplicitT<Compiler>::br;
+
+  //! Adds a jump to the given `target` with the provided jump `annotation`.
+  inline Error br(const BaseReg& target, JumpAnnotation* annotation) { return emitAnnotatedJump(Inst::kIdBr, target, annotation); }
+
+  //! \}
+
+  //! \name Events
+  //! \{
+
+  ASMJIT_API Error onAttach(CodeHolder* code) noexcept override;
+  ASMJIT_API Error onDetach(CodeHolder* code) noexcept override;
+
+  //! \}
+
+  //! \name Finalize
+  //! \{
+
+  ASMJIT_API Error finalize() override;
+
+  //! \}
+};
+
+//! \}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_COMPILER
+#endif // ASMJIT_ARM_ARMCOMPILER_H_INCLUDED

libraries/asmjit/asmjit/arm/a64emithelper.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64)
+
+#include "../core/formatter.h"
+#include "../core/funcargscontext_p.h"
+#include "../core/string.h"
+#include "../core/support.h"
+#include "../core/type.h"
+#include "../arm/a64emithelper_p.h"
+#include "../arm/a64formatter_p.h"
+#include "../arm/a64instapi_p.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::EmitHelper - Emit Operations
+// =================================
+
+ASMJIT_FAVOR_SIZE Error EmitHelper::emitRegMove(
+  const Operand_& dst_,
+  const Operand_& src_, TypeId typeId, const char* comment) {
+
+  Emitter* emitter = _emitter->as<Emitter>();
+
+  // Invalid or abstract TypeIds are not allowed.
+  ASMJIT_ASSERT(TypeUtils::isValid(typeId) && !TypeUtils::isAbstract(typeId));
+
+  emitter->setInlineComment(comment);
+
+  if (dst_.isReg() && src_.isMem()) {
+    Reg dst(dst_.as<Reg>());
+    Mem src(src_.as<Mem>());
+
+    switch (typeId) {
+      case TypeId::kInt8:
+      case TypeId::kUInt8:
+        return emitter->ldrb(dst.as<Gp>(), src);
+
+      case TypeId::kInt16:
+      case TypeId::kUInt16:
+        return emitter->ldrh(dst.as<Gp>(), src);
+
+      case TypeId::kInt32:
+      case TypeId::kUInt32:
+        return emitter->ldr(dst.as<Gp>().w(), src);
+
+      case TypeId::kInt64:
+      case TypeId::kUInt64:
+        return emitter->ldr(dst.as<Gp>().x(), src);
+
+      default: {
+        if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId))
+          return emitter->ldr(dst.as<Vec>().s(), src);
+
+        if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId))
+          return emitter->ldr(dst.as<Vec>().d(), src);
+
+        if (TypeUtils::isVec128(typeId))
+          return emitter->ldr(dst.as<Vec>().q(), src);
+
+        break;
+      }
+    }
+  }
+
+  if (dst_.isMem() && src_.isReg()) {
+    Mem dst(dst_.as<Mem>());
+    Reg src(src_.as<Reg>());
+
+    switch (typeId) {
+      case TypeId::kInt8:
+      case TypeId::kUInt8:
+        return emitter->strb(src.as<Gp>(), dst);
+
+      case TypeId::kInt16:
+      case TypeId::kUInt16:
+        return emitter->strh(src.as<Gp>(), dst);
+
+      case TypeId::kInt32:
+      case TypeId::kUInt32:
+        return emitter->str(src.as<Gp>().w(), dst);
+
+      case TypeId::kInt64:
+      case TypeId::kUInt64:
+        return emitter->str(src.as<Gp>().x(), dst);
+
+      default: {
+        if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId))
+          return emitter->str(src.as<Vec>().s(), dst);
+
+        if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId))
+          return emitter->str(src.as<Vec>().d(), dst);
+
+        if (TypeUtils::isVec128(typeId))
+          return emitter->str(src.as<Vec>().q(), dst);
+
+        break;
+      }
+    }
+  }
+
+  if (dst_.isReg() && src_.isReg()) {
+    Reg dst(dst_.as<Reg>());
+    Reg src(src_.as<Reg>());
+
+    switch (typeId) {
+      case TypeId::kInt8:
+      case TypeId::kUInt8:
+      case TypeId::kInt16:
+      case TypeId::kUInt16:
+      case TypeId::kInt32:
+      case TypeId::kUInt32:
+      case TypeId::kInt64:
+      case TypeId::kUInt64:
+        return emitter->mov(src.as<Gp>().x(), dst.as<Gp>().x());
+
+      default: {
+        if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId))
+          return emitter->fmov(dst.as<Vec>().s(), src.as<Vec>().s());
+
+        if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId))
+          return emitter->mov(dst.as<Vec>().b8(), src.as<Vec>().b8());
+
+        if (TypeUtils::isVec128(typeId))
+          return emitter->mov(dst.as<Vec>().b16(), src.as<Vec>().b16());
+
+        break;
+      }
+    }
+  }
+
+  emitter->setInlineComment(nullptr);
+  return DebugUtils::errored(kErrorInvalidState);
+}
+
+Error EmitHelper::emitRegSwap(
+  const BaseReg& a,
+  const BaseReg& b, const char* comment) {
+
+  DebugUtils::unused(a, b, comment);
+  return DebugUtils::errored(kErrorInvalidState);
+}
+
+// TODO: [ARM] EmitArgMove is unfinished.
+Error EmitHelper::emitArgMove(
+  const BaseReg& dst_, TypeId dstTypeId,
+  const Operand_& src_, TypeId srcTypeId, const char* comment) {
+
+  // Deduce optional `dstTypeId`, which may be `TypeId::kVoid` in some cases.
+  if (dstTypeId == TypeId::kVoid) {
+    const ArchTraits& archTraits = ArchTraits::byArch(_emitter->arch());
+    dstTypeId = archTraits.regTypeToTypeId(dst_.type());
+  }
+
+  // Invalid or abstract TypeIds are not allowed.
+  ASMJIT_ASSERT(TypeUtils::isValid(dstTypeId) && !TypeUtils::isAbstract(dstTypeId));
+  ASMJIT_ASSERT(TypeUtils::isValid(srcTypeId) && !TypeUtils::isAbstract(srcTypeId));
+
+  Reg dst(dst_.as<Reg>());
+  Operand src(src_);
+
+  uint32_t dstSize = TypeUtils::sizeOf(dstTypeId);
+  uint32_t srcSize = TypeUtils::sizeOf(srcTypeId);
+
+  if (TypeUtils::isInt(dstTypeId)) {
+    if (TypeUtils::isInt(srcTypeId)) {
+      uint32_t x = dstSize == 8;
+
+      dst.setSignature(OperandSignature{x ? uint32_t(GpX::kSignature) : uint32_t(GpW::kSignature)});
+      _emitter->setInlineComment(comment);
+
+      if (src.isReg()) {
+        src.setSignature(dst.signature());
+        return _emitter->emit(Inst::kIdMov, dst, src);
+      }
+      else if (src.isMem()) {
+        InstId instId = Inst::kIdNone;
+          switch (srcTypeId) {
+          case TypeId::kInt8: instId = Inst::kIdLdrsb; break;
+          case TypeId::kUInt8: instId = Inst::kIdLdrb; break;
+          case TypeId::kInt16: instId = Inst::kIdLdrsh; break;
+          case TypeId::kUInt16: instId = Inst::kIdLdrh; break;
+          case TypeId::kInt32: instId = x ? Inst::kIdLdrsw : Inst::kIdLdr; break;
+          case TypeId::kUInt32: instId = Inst::kIdLdr; x = 0; break;
+          case TypeId::kInt64: instId = Inst::kIdLdr; break;
+          case TypeId::kUInt64: instId = Inst::kIdLdr; break;
+          default:
+            return DebugUtils::errored(kErrorInvalidState);
+        }
+        return _emitter->emit(instId, dst, src);
+      }
+    }
+  }
+
+  if (TypeUtils::isFloat(dstTypeId) || TypeUtils::isVec(dstTypeId)) {
+    if (TypeUtils::isFloat(srcTypeId) || TypeUtils::isVec(srcTypeId)) {
+      switch (srcSize) {
+        case 2: dst.as<Vec>().setSignature(OperandSignature{VecH::kSignature}); break;
+        case 4: dst.as<Vec>().setSignature(OperandSignature{VecS::kSignature}); break;
+        case 8: dst.as<Vec>().setSignature(OperandSignature{VecD::kSignature}); break;
+        case 16: dst.as<Vec>().setSignature(OperandSignature{VecV::kSignature}); break;
+        default:
+          return DebugUtils::errored(kErrorInvalidState);
+      }
+
+      _emitter->setInlineComment(comment);
+
+      if (src.isReg()) {
+        InstId instId = srcSize <= 4 ? Inst::kIdFmov_v : Inst::kIdMov_v;
+        src.setSignature(dst.signature());
+        return _emitter->emit(instId, dst, src);
+      }
+      else if (src.isMem()) {
+        return _emitter->emit(Inst::kIdLdr_v, dst, src);
+      }
+    }
+  }
+
+  return DebugUtils::errored(kErrorInvalidState);
+}
+
+// a64::EmitHelper - Emit Prolog & Epilog
+// ======================================
+
+struct LoadStoreInstructions {
+  InstId singleInstId;
+  InstId pairInstId;
+};
+
+struct PrologEpilogInfo {
+  struct RegPair {
+    uint8_t ids[2];
+    uint16_t offset;
+  };
+
+  struct GroupData {
+    RegPair pairs[16];
+    uint32_t pairCount;
+  };
+
+  Support::Array<GroupData, 2> groups;
+  uint32_t sizeTotal;
+
+  Error init(const FuncFrame& frame) noexcept {
+    uint32_t offset = 0;
+
+    for (RegGroup group : Support::EnumValues<RegGroup, RegGroup::kGp, RegGroup::kVec>{}) {
+      GroupData& data = groups[group];
+
+      uint32_t n = 0;
+      uint32_t pairCount = 0;
+      RegPair* pairs = data.pairs;
+
+      uint32_t slotSize = frame.saveRestoreRegSize(group);
+      uint32_t savedRegs = frame.savedRegs(group);
+
+      if (group == RegGroup::kGp && frame.hasPreservedFP()) {
+        // Must be at the beginning of the push/pop sequence.
+        ASMJIT_ASSERT(pairCount == 0);
+
+        pairs[0].offset = uint16_t(offset);
+        pairs[0].ids[0] = Gp::kIdFp;
+        pairs[0].ids[1] = Gp::kIdLr;
+        offset += slotSize * 2;
+        pairCount++;
+
+        savedRegs &= ~Support::bitMask(Gp::kIdFp, Gp::kIdLr);
+      }
+
+      Support::BitWordIterator<uint32_t> it(savedRegs);
+      while (it.hasNext()) {
+        pairs[pairCount].ids[n] = uint8_t(it.next());
+
+        if (++n == 2) {
+          pairs[pairCount].offset = uint16_t(offset);
+          offset += slotSize * 2;
+
+          n = 0;
+          pairCount++;
+        }
+      }
+
+      if (n == 1) {
+        pairs[pairCount].ids[1] = uint8_t(BaseReg::kIdBad);
+        pairs[pairCount].offset = uint16_t(offset);
+        offset += slotSize * 2;
+        pairCount++;
+      }
+
+      data.pairCount = pairCount;
+    }
+
+    sizeTotal = offset;
+    return kErrorOk;
+  }
+};
+
+ASMJIT_FAVOR_SIZE Error EmitHelper::emitProlog(const FuncFrame& frame) {
+  Emitter* emitter = _emitter->as<Emitter>();
+
+  PrologEpilogInfo pei;
+  ASMJIT_PROPAGATE(pei.init(frame));
+
+  static const Support::Array<Reg, 2> groupRegs = {{ x0, d0 }};
+  static const Support::Array<LoadStoreInstructions, 2> groupInsts = {{
+    { Inst::kIdStr  , Inst::kIdStp   },
+    { Inst::kIdStr_v, Inst::kIdStp_v }
+  }};
+
+  uint32_t adjustInitialOffset = pei.sizeTotal;
+
+  for (RegGroup group : Support::EnumValues<RegGroup, RegGroup::kGp, RegGroup::kVec>{}) {
+    const PrologEpilogInfo::GroupData& data = pei.groups[group];
+    uint32_t pairCount = data.pairCount;
+
+    Reg regs[2] = { groupRegs[group], groupRegs[group] };
+    Mem mem = ptr(sp);
+
+    const LoadStoreInstructions& insts = groupInsts[group];
+    for (uint32_t i = 0; i < pairCount; i++) {
+      const PrologEpilogInfo::RegPair& pair = data.pairs[i];
+
+      regs[0].setId(pair.ids[0]);
+      regs[1].setId(pair.ids[1]);
+      mem.setOffsetLo32(pair.offset);
+
+      if (pair.offset == 0 && adjustInitialOffset) {
+        mem.setOffset(-int(adjustInitialOffset));
+        mem.makePreIndex();
+      }
+
+      if (pair.ids[1] == BaseReg::kIdBad)
+        ASMJIT_PROPAGATE(emitter->emit(insts.singleInstId, regs[0], mem));
+      else
+        ASMJIT_PROPAGATE(emitter->emit(insts.pairInstId, regs[0], regs[1], mem));
+
+      mem.resetToFixedOffset();
+
+      if (i == 0 && frame.hasPreservedFP()) {
+        ASMJIT_PROPAGATE(emitter->mov(x29, sp));
+      }
+    }
+  }
+
+  if (frame.hasStackAdjustment()) {
+    uint32_t adj = frame.stackAdjustment();
+    if (adj <= 0xFFFu) {
+      ASMJIT_PROPAGATE(emitter->sub(sp, sp, adj));
+    }
+    else if (adj <= 0xFFFFFFu)  {
+      // TODO: [ARM] Prolog - we must touch the pages otherwise it's undefined.
+      ASMJIT_PROPAGATE(emitter->sub(sp, sp, adj & 0x000FFFu));
+      ASMJIT_PROPAGATE(emitter->sub(sp, sp, adj & 0xFFF000u));
+    }
+    else {
+      return DebugUtils::errored(kErrorInvalidState);
+    }
+  }
+
+  return kErrorOk;
+}
+
+// TODO: [ARM] Emit epilog.
+ASMJIT_FAVOR_SIZE Error EmitHelper::emitEpilog(const FuncFrame& frame) {
+  Emitter* emitter = _emitter->as<Emitter>();
+
+  PrologEpilogInfo pei;
+  ASMJIT_PROPAGATE(pei.init(frame));
+
+  static const Support::Array<Reg, 2> groupRegs = {{ x0, d0 }};
+  static const Support::Array<LoadStoreInstructions, 2> groupInsts = {{
+    { Inst::kIdLdr  , Inst::kIdLdp   },
+    { Inst::kIdLdr_v, Inst::kIdLdp_v }
+  }};
+
+  uint32_t adjustInitialOffset = pei.sizeTotal;
+
+  if (frame.hasStackAdjustment()) {
+    uint32_t adj = frame.stackAdjustment();
+    if (adj <= 0xFFFu) {
+      ASMJIT_PROPAGATE(emitter->add(sp, sp, adj));
+    }
+    else if (adj <= 0xFFFFFFu)  {
+      ASMJIT_PROPAGATE(emitter->add(sp, sp, adj & 0x000FFFu));
+      ASMJIT_PROPAGATE(emitter->add(sp, sp, adj & 0xFFF000u));
+    }
+    else {
+      return DebugUtils::errored(kErrorInvalidState);
+    }
+  }
+
+  for (int g = 1; g >= 0; g--) {
+    RegGroup group = RegGroup(g);
+    const PrologEpilogInfo::GroupData& data = pei.groups[group];
+    uint32_t pairCount = data.pairCount;
+
+    Reg regs[2] = { groupRegs[group], groupRegs[group] };
+    Mem mem = ptr(sp);
+
+    const LoadStoreInstructions& insts = groupInsts[group];
+
+    for (int i = int(pairCount) - 1; i >= 0; i--) {
+      const PrologEpilogInfo::RegPair& pair = data.pairs[i];
+
+      regs[0].setId(pair.ids[0]);
+      regs[1].setId(pair.ids[1]);
+      mem.setOffsetLo32(pair.offset);
+
+      if (pair.offset == 0 && adjustInitialOffset) {
+        mem.setOffset(int(adjustInitialOffset));
+        mem.makePostIndex();
+      }
+
+      if (pair.ids[1] == BaseReg::kIdBad)
+        ASMJIT_PROPAGATE(emitter->emit(insts.singleInstId, regs[0], mem));
+      else
+        ASMJIT_PROPAGATE(emitter->emit(insts.pairInstId, regs[0], regs[1], mem));
+
+      mem.resetToFixedOffset();
+    }
+  }
+
+  ASMJIT_PROPAGATE(emitter->ret(x30));
+
+  return kErrorOk;
+}
+
+static Error ASMJIT_CDECL Emitter_emitProlog(BaseEmitter* emitter, const FuncFrame& frame) {
+  EmitHelper emitHelper(emitter);
+  return emitHelper.emitProlog(frame);
+}
+
+static Error ASMJIT_CDECL Emitter_emitEpilog(BaseEmitter* emitter, const FuncFrame& frame) {
+  EmitHelper emitHelper(emitter);
+  return emitHelper.emitEpilog(frame);
+}
+
+static Error ASMJIT_CDECL Emitter_emitArgsAssignment(BaseEmitter* emitter, const FuncFrame& frame, const FuncArgsAssignment& args) {
+  EmitHelper emitHelper(emitter);
+  return emitHelper.emitArgsAssignment(frame, args);
+}
+
+void assignEmitterFuncs(BaseEmitter* emitter) {
+  emitter->_funcs.emitProlog = Emitter_emitProlog;
+  emitter->_funcs.emitEpilog = Emitter_emitEpilog;
+  emitter->_funcs.emitArgsAssignment = Emitter_emitArgsAssignment;
+
+#ifndef ASMJIT_NO_LOGGING
+  emitter->_funcs.formatInstruction = FormatterInternal::formatInstruction;
+#endif
+
+#ifndef ASMJIT_NO_VALIDATION
+  emitter->_funcs.validate = InstInternal::validate;
+#endif
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64

libraries/asmjit/asmjit/arm/a64emithelper_p.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_ARMEMITHELPER_P_H_INCLUDED
+#define ASMJIT_ARM_ARMEMITHELPER_P_H_INCLUDED
+
+#include "../core/api-config.h"
+
+#include "../core/emithelper_p.h"
+#include "../core/func.h"
+#include "../arm/a64emitter.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \cond INTERNAL
+//! \addtogroup asmjit_a64
+//! \{
+
+class EmitHelper : public BaseEmitHelper {
+public:
+  inline explicit EmitHelper(BaseEmitter* emitter = nullptr) noexcept
+    : BaseEmitHelper(emitter) {}
+
+  Error emitRegMove(
+    const Operand_& dst_,
+    const Operand_& src_, TypeId typeId, const char* comment = nullptr) override;
+
+  Error emitRegSwap(
+    const BaseReg& a,
+    const BaseReg& b, const char* comment = nullptr) override;
+
+  Error emitArgMove(
+    const BaseReg& dst_, TypeId dstTypeId,
+    const Operand_& src_, TypeId srcTypeId, const char* comment = nullptr) override;
+
+  Error emitProlog(const FuncFrame& frame);
+  Error emitEpilog(const FuncFrame& frame);
+};
+
+void assignEmitterFuncs(BaseEmitter* emitter);
+
+//! \}
+//! \endcond
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_ARMEMITHELPER_P_H_INCLUDED

libraries/asmjit/asmjit/arm/a64emitter.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64EMITTER_H_INCLUDED
+#define ASMJIT_ARM_A64EMITTER_H_INCLUDED
+
+#include "../core/emitter.h"
+#include "../core/support.h"
+#include "../arm/a64instdb.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+#define ASMJIT_INST_0x(NAME, ID) \
+  inline Error NAME() { return _emitter()->_emitI(Inst::kId##ID); }
+
+#define ASMJIT_INST_1x(NAME, ID, T0) \
+  inline Error NAME(const T0& o0) { return _emitter()->_emitI(Inst::kId##ID, o0); }
+
+#define ASMJIT_INST_2x(NAME, ID, T0, T1) \
+  inline Error NAME(const T0& o0, const T1& o1) { return _emitter()->_emitI(Inst::kId##ID, o0, o1); }
+
+#define ASMJIT_INST_3x(NAME, ID, T0, T1, T2) \
+  inline Error NAME(const T0& o0, const T1& o1, const T2& o2) { return _emitter()->_emitI(Inst::kId##ID, o0, o1, o2); }
+
+#define ASMJIT_INST_4x(NAME, ID, T0, T1, T2, T3) \
+  inline Error NAME(const T0& o0, const T1& o1, const T2& o2, const T3& o3) { return _emitter()->_emitI(Inst::kId##ID, o0, o1, o2, o3); }
+
+#define ASMJIT_INST_5x(NAME, ID, T0, T1, T2, T3, T4) \
+  inline Error NAME(const T0& o0, const T1& o1, const T2& o2, const T3& o3, const T4& o4) { return _emitter()->_emitI(Inst::kId##ID, o0, o1, o2, o3, o4); }
+
+#define ASMJIT_INST_6x(NAME, ID, T0, T1, T2, T3, T4, T5) \
+  inline Error NAME(const T0& o0, const T1& o1, const T2& o2, const T3& o3, const T4& o4, const T5& o5) { return _emitter()->_emitI(Inst::kId##ID, o0, o1, o2, o3, o4, o5); }
+
+#define ASMJIT_INST_1cc(NAME, ID, T0) \
+  inline Error NAME(const T0& o0) { return _emitter()->_emitI(Inst::kId##ID, o0); } \
+  \
+  inline Error NAME(CondCode cc, const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, cc), o0); } \
+  \
+  inline Error NAME##_eq(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kEQ), o0); } \
+  inline Error NAME##_ne(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kNE), o0); } \
+  inline Error NAME##_cs(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kCS), o0); } \
+  inline Error NAME##_hs(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kHS), o0); } \
+  inline Error NAME##_cc(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kCC), o0); } \
+  inline Error NAME##_lo(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kLO), o0); } \
+  inline Error NAME##_mi(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kMI), o0); } \
+  inline Error NAME##_pl(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kPL), o0); } \
+  inline Error NAME##_vs(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kVS), o0); } \
+  inline Error NAME##_vc(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kVC), o0); } \
+  inline Error NAME##_hi(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kHI), o0); } \
+  inline Error NAME##_ls(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kLS), o0); } \
+  inline Error NAME##_ge(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kGE), o0); } \
+  inline Error NAME##_lt(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kLT), o0); } \
+  inline Error NAME##_gt(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kGT), o0); } \
+  inline Error NAME##_le(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kLE), o0); } \
+  inline Error NAME##_al(const T0& o0) { return _emitter()->_emitI(BaseInst::composeARMInstId(Inst::kId##ID, CondCode::kAL), o0); }
+
+//! \addtogroup asmjit_a64
+//! \{
+
+//! ARM emitter.
+//!
+//! NOTE: This class cannot be instantiated, you can only cast to it and use it as emitter that emits to either
+//! \ref Assembler, \ref Builder, or \ref Compiler (use withcaution with \ref Compiler as it expects virtual
+//! registers to be used).
+template<typename This>
+struct EmitterExplicitT {
+  //! \cond
+
+  // These two are unfortunately reported by the sanitizer. We know what we do, however, the sanitizer doesn't.
+  // I have tried to use reinterpret_cast instead, but that would generate bad code when compiled by MSC.
+  ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF inline This* _emitter() noexcept { return static_cast<This*>(this); }
+  ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF inline const This* _emitter() const noexcept { return static_cast<const This*>(this); }
+
+  //! \endcond
+
+  // --------------------------------------------------------------------------
+  // [Options]
+  // --------------------------------------------------------------------------
+
+protected:
+  inline This& _addInstOptions(InstOptions options) noexcept {
+    static_cast<This*>(this)->addInstOptions(options);
+    return *static_cast<This*>(this);
+  }
+
+public:
+  //! \name General Purpose Instructions
+  //! \{
+
+  ASMJIT_INST_3x(adc, Adc, Gp, Gp, Gp)
+  ASMJIT_INST_3x(adcs, Adcs, Gp, Gp, Gp)
+
+  ASMJIT_INST_3x(add, Add, Gp, Gp, Gp)
+  ASMJIT_INST_4x(add, Add, Gp, Gp, Gp, Imm)
+  ASMJIT_INST_3x(add, Add, Gp, Gp, Imm)
+  ASMJIT_INST_4x(add, Add, Gp, Gp, Imm, Imm)
+  ASMJIT_INST_3x(adds, Adds, Gp, Gp, Gp)
+  ASMJIT_INST_3x(adds, Adds, Gp, Gp, Imm)
+  ASMJIT_INST_4x(adds, Adds, Gp, Gp, Gp, Imm)
+  ASMJIT_INST_4x(adds, Adds, Gp, Gp, Imm, Imm)
+
+  ASMJIT_INST_2x(adr, Adr, Gp, Imm)
+  ASMJIT_INST_2x(adr, Adr, Gp, Label)
+  ASMJIT_INST_2x(adrp, Adrp, Gp, Imm)
+  ASMJIT_INST_2x(adrp, Adrp, Gp, Label)
+
+  ASMJIT_INST_3x(and_, And, Gp, Gp, Imm)
+  ASMJIT_INST_3x(and_, And, Gp, Gp, Gp)
+  ASMJIT_INST_4x(and_, And, Gp, Gp, Gp, Imm)
+  ASMJIT_INST_3x(ands, Ands, Gp, Gp, Imm)
+  ASMJIT_INST_3x(ands, Ands, Gp, Gp, Gp)
+  ASMJIT_INST_4x(ands, Ands, Gp, Gp, Gp, Imm)
+
+  ASMJIT_INST_3x(asr, Asr, Gp, Gp, Imm)
+  ASMJIT_INST_3x(asr, Asr, Gp, Gp, Gp)
+  ASMJIT_INST_3x(asrv, Asrv, Gp, Gp, Gp)
+
+  ASMJIT_INST_2x(at, At, Imm, Gp)
+
+  ASMJIT_INST_3x(bfc, Bfc, Gp, Imm, Imm)
+  ASMJIT_INST_4x(bfi, Bfi, Gp, Gp, Imm, Imm)
+  ASMJIT_INST_4x(bfm, Bfm, Gp, Gp, Imm, Imm)
+  ASMJIT_INST_4x(bfxil, Bfxil, Gp, Gp, Imm, Imm)
+
+  ASMJIT_INST_3x(bic, Bic, Gp, Gp, Imm);
+  ASMJIT_INST_3x(bic, Bic, Gp, Gp, Gp);
+  ASMJIT_INST_4x(bic, Bic, Gp, Gp, Gp, Imm);
+  ASMJIT_INST_3x(bics, Bics, Gp, Gp, Imm);
+  ASMJIT_INST_3x(bics, Bics, Gp, Gp, Gp);
+  ASMJIT_INST_4x(bics, Bics, Gp, Gp, Gp, Imm);
+
+  ASMJIT_INST_1x(brk, Brk, Imm)
+
+  ASMJIT_INST_4x(ccmn, Ccmn, Gp, Gp, Imm, Imm);
+  ASMJIT_INST_4x(ccmn, Ccmn, Gp, Imm, Imm, Imm);
+  ASMJIT_INST_4x(ccmp, Ccmp, Gp, Gp, Imm, Imm);
+  ASMJIT_INST_4x(ccmp, Ccmp, Gp, Imm, Imm, Imm);
+
+  ASMJIT_INST_3x(cinc, Cinc, Gp, Gp, Imm);
+  ASMJIT_INST_3x(cinv, Cinv, Gp, Gp, Imm);
+
+  ASMJIT_INST_1x(clrex, Clrex, Imm)
+
+  ASMJIT_INST_2x(cls, Cls, Gp, Gp)
+  ASMJIT_INST_2x(clz, Clz, Gp, Gp)
+
+  ASMJIT_INST_2x(cmn, Cmn, Gp, Gp)
+  ASMJIT_INST_3x(cmn, Cmn, Gp, Gp, Imm)
+  ASMJIT_INST_2x(cmn, Cmn, Gp, Imm)
+  ASMJIT_INST_3x(cmn, Cmn, Gp, Imm, Imm)
+  ASMJIT_INST_2x(cmp, Cmp, Gp, Gp)
+  ASMJIT_INST_3x(cmp, Cmp, Gp, Gp, Imm)
+  ASMJIT_INST_2x(cmp, Cmp, Gp, Imm)
+  ASMJIT_INST_3x(cmp, Cmp, Gp, Imm, Imm)
+
+  ASMJIT_INST_3x(cneg, Cneg, Gp, Gp, Imm);
+
+  ASMJIT_INST_4x(csel, Csel, Gp, Gp, Gp, Imm);
+  ASMJIT_INST_2x(cset, Cset, Gp, Imm);
+  ASMJIT_INST_2x(csetm, Csetm, Gp, Imm);
+
+  ASMJIT_INST_4x(csinc, Csinc, Gp, Gp, Gp, Imm);
+  ASMJIT_INST_4x(csinv, Csinv, Gp, Gp, Gp, Imm);
+  ASMJIT_INST_4x(csneg, Csneg, Gp, Gp, Gp, Imm);
+
+  ASMJIT_INST_2x(dc, Dc, Imm, Gp)
+  ASMJIT_INST_1x(dmb, Dmb, Imm)
+  ASMJIT_INST_1x(dsb, Dsb, Imm)
+  ASMJIT_INST_0x(drps, Drps)
+
+  ASMJIT_INST_3x(eon, Eon, Gp, Gp, Gp)
+  ASMJIT_INST_4x(eon, Eon, Gp, Gp, Gp, Imm)
+
+  ASMJIT_INST_3x(eor, Eor, Gp, Gp, Imm)
+  ASMJIT_INST_3x(eor, Eor, Gp, Gp, Gp)
+  ASMJIT_INST_4x(eor, Eor, Gp, Gp, Gp, Imm)
+
+  ASMJIT_INST_0x(eret, Eret)
+  ASMJIT_INST_0x(esb, Esb)
+
+  ASMJIT_INST_4x(extr, Extr, Gp, Gp, Gp, Imm)
+
+  ASMJIT_INST_1x(hlt, Hlt, Imm)
+  ASMJIT_INST_1x(hvc, Hvc, Imm)
+  ASMJIT_INST_2x(ic, Ic, Imm, Gp)
+  ASMJIT_INST_1x(isb, Isb, Imm)
+
+  ASMJIT_INST_3x(lsl, Lsl, Gp, Gp, Imm)
+  ASMJIT_INST_3x(lsl, Lsl, Gp, Gp, Gp)
+  ASMJIT_INST_3x(lslv, Lslv, Gp, Gp, Gp)
+
+  ASMJIT_INST_3x(lsr, Lsr, Gp, Gp, Imm)
+  ASMJIT_INST_3x(lsr, Lsr, Gp, Gp, Gp)
+  ASMJIT_INST_3x(lsrv, Lsrv, Gp, Gp, Gp)
+
+  ASMJIT_INST_4x(madd, Madd, Gp, Gp, Gp, Gp)
+  ASMJIT_INST_3x(mneg, Mneg, Gp, Gp, Gp)
+
+  ASMJIT_INST_2x(mov, Mov, Gp, Gp)
+  ASMJIT_INST_2x(mov, Mov, Gp, Imm)
+  ASMJIT_INST_2x(movk, Movk, Gp, Imm)
+  ASMJIT_INST_3x(movk, Movk, Gp, Imm, Imm)
+  ASMJIT_INST_2x(movn, Movn, Gp, Imm)
+  ASMJIT_INST_3x(movn, Movn, Gp, Imm, Imm)
+  ASMJIT_INST_2x(movz, Movz, Gp, Imm)
+  ASMJIT_INST_3x(movz, Movz, Gp, Imm, Imm)
+
+  ASMJIT_INST_2x(mrs, Mrs, Gp, Imm)
+  ASMJIT_INST_2x(msr, Msr, Imm, Gp)
+  ASMJIT_INST_2x(msr, Msr, Imm, Imm)
+
+  ASMJIT_INST_4x(msub, Msub, Gp, Gp, Gp, Gp)
+  ASMJIT_INST_3x(mul, Mul, Gp, Gp, Gp)
+
+  ASMJIT_INST_2x(mvn, Mvn, Gp, Gp)
+  ASMJIT_INST_3x(mvn, Mvn, Gp, Gp, Imm)
+
+  ASMJIT_INST_2x(neg, Neg, Gp, Gp)
+  ASMJIT_INST_3x(neg, Neg, Gp, Gp, Imm)
+  ASMJIT_INST_2x(negs, Negs, Gp, Gp)
+  ASMJIT_INST_3x(negs, Negs, Gp, Gp, Imm)
+
+  ASMJIT_INST_2x(ngc, Ngc, Gp, Gp)
+  ASMJIT_INST_2x(ngcs, Ngcs, Gp, Gp)
+
+  ASMJIT_INST_3x(orn, Orn, Gp, Gp, Gp)
+  ASMJIT_INST_4x(orn, Orn, Gp, Gp, Gp, Imm)
+
+  ASMJIT_INST_3x(orr, Orr, Gp, Gp, Imm)
+  ASMJIT_INST_3x(orr, Orr, Gp, Gp, Gp)
+  ASMJIT_INST_4x(orr, Orr, Gp, Gp, Gp, Imm)
+
+  ASMJIT_INST_2x(rbit, Rbit, Gp, Gp)
+  ASMJIT_INST_1x(ret, Ret, Gp)
+
+  ASMJIT_INST_2x(rev, Rev, Gp, Gp)
+  ASMJIT_INST_2x(rev16, Rev16, Gp, Gp)
+  ASMJIT_INST_2x(rev32, Rev32, Gp, Gp)
+  ASMJIT_INST_2x(rev64, Rev64, Gp, Gp)
+
+  ASMJIT_INST_3x(ror, Ror, Gp, Gp, Imm)
+  ASMJIT_INST_3x(ror, Ror, Gp, Gp, Gp)
+  ASMJIT_INST_3x(rorv, Rorv, Gp, Gp, Gp)
+
+  ASMJIT_INST_3x(sbc, Sbc, Gp, Gp, Gp)
+  ASMJIT_INST_3x(sbcs, Sbcs, Gp, Gp, Gp)
+
+  ASMJIT_INST_4x(sbfiz, Sbfiz, Gp, Gp, Imm, Imm)
+  ASMJIT_INST_4x(sbfm, Sbfm, Gp, Gp, Imm, Imm)
+  ASMJIT_INST_4x(sbfx, Sbfx, Gp, Gp, Imm, Imm)
+
+  ASMJIT_INST_3x(sdiv, Sdiv, Gp, Gp, Gp)
+
+  ASMJIT_INST_4x(smaddl, Smaddl, Gp, Gp, Gp, Gp)
+  ASMJIT_INST_1x(smc, Smc, Imm)
+  ASMJIT_INST_3x(smnegl, Smnegl, Gp, Gp, Gp)
+  ASMJIT_INST_4x(smsubl, Smsubl, Gp, Gp, Gp, Gp)
+  ASMJIT_INST_3x(smulh, Smulh, Gp, Gp, Gp)
+  ASMJIT_INST_3x(smull, Smull, Gp, Gp, Gp)
+
+  ASMJIT_INST_3x(sub, Sub, Gp, Gp, Gp)
+  ASMJIT_INST_4x(sub, Sub, Gp, Gp, Gp, Imm)
+  ASMJIT_INST_3x(sub, Sub, Gp, Gp, Imm)
+  ASMJIT_INST_4x(sub, Sub, Gp, Gp, Imm, Imm)
+  ASMJIT_INST_3x(subs, Subs, Gp, Gp, Gp)
+  ASMJIT_INST_4x(subs, Subs, Gp, Gp, Gp, Imm)
+  ASMJIT_INST_3x(subs, Subs, Gp, Gp, Imm)
+  ASMJIT_INST_4x(subs, Subs, Gp, Gp, Imm, Imm)
+
+  ASMJIT_INST_1x(svc, Svc, Imm)
+
+  ASMJIT_INST_2x(sxtb, Sxtb, Gp, Gp)
+  ASMJIT_INST_2x(sxth, Sxth, Gp, Gp)
+  ASMJIT_INST_2x(sxtw, Sxtw, Gp, Gp)
+
+  ASMJIT_INST_4x(sys, Sys, Imm, Imm, Imm, Imm)
+  ASMJIT_INST_5x(sys, Sys, Imm, Imm, Imm, Imm, Gp)
+
+  ASMJIT_INST_2x(tlbi, Tlbi, Imm, Gp)
+  ASMJIT_INST_2x(tst, Tst, Gp, Imm)
+  ASMJIT_INST_2x(tst, Tst, Gp, Gp)
+  ASMJIT_INST_3x(tst, Tst, Gp, Gp, Imm)
+
+  ASMJIT_INST_3x(udiv, Udiv, Gp, Gp, Gp)
+
+  ASMJIT_INST_4x(ubfiz, Ubfiz, Gp, Gp, Imm, Imm)
+  ASMJIT_INST_4x(ubfm, Ubfm, Gp, Gp, Imm, Imm)
+  ASMJIT_INST_4x(ubfx, Ubfx, Gp, Gp, Imm, Imm)
+
+  ASMJIT_INST_4x(umaddl, Umaddl, Gp, Gp, Gp, Gp)
+  ASMJIT_INST_3x(umnegl, Umnegl, Gp, Gp, Gp)
+  ASMJIT_INST_4x(umsubl, Umsubl, Gp, Gp, Gp, Gp)
+  ASMJIT_INST_3x(umull, Umull, Gp, Gp, Gp)
+  ASMJIT_INST_3x(umulh, Umulh, Gp, Gp, Gp)
+
+  ASMJIT_INST_2x(uxtb, Uxtb, Gp, Gp)
+  ASMJIT_INST_2x(uxth, Uxth, Gp, Gp)
+
+  ASMJIT_INST_0x(csdb, Csdb)
+  ASMJIT_INST_1x(dcps1, Dcps1, Imm)
+  ASMJIT_INST_1x(dcps2, Dcps2, Imm)
+  ASMJIT_INST_1x(dcps3, Dcps3, Imm)
+  ASMJIT_INST_0x(dgh, Dgh)
+  ASMJIT_INST_0x(pssbb, Pssbb)
+  ASMJIT_INST_0x(ssbb, Ssbb)
+  ASMJIT_INST_1x(udf, Udf, Imm)
+  ASMJIT_INST_1x(setf8, Setf8, Gp)
+  ASMJIT_INST_1x(setf16, Setf16, Gp)
+
+  //! \}
+
+  //! \name ARMv8.4 Instructions
+  //! \{
+
+  ASMJIT_INST_0x(cfinv, Cfinv)
+
+  //! \}
+
+  //! \name ARMv8.5 Instructions
+  //! \{
+
+  ASMJIT_INST_0x(axflag, Axflag)
+  ASMJIT_INST_0x(xaflag, Xaflag)
+
+  //! \}
+
+  //! \name Branch Instructions
+  //! \{
+
+  ASMJIT_INST_1cc(b, B, Imm)
+  ASMJIT_INST_1cc(b, B, Label)
+  ASMJIT_INST_1x(bl, Bl, Imm)
+  ASMJIT_INST_1x(bl, Bl, Label)
+  ASMJIT_INST_1x(blr, Blr, Gp)
+  ASMJIT_INST_1x(br, Br, Gp)
+  ASMJIT_INST_2x(cbz, Cbz, Gp, Imm)
+  ASMJIT_INST_2x(cbz, Cbz, Gp, Label)
+  ASMJIT_INST_2x(cbnz, Cbnz, Gp, Imm)
+  ASMJIT_INST_2x(cbnz, Cbnz, Gp, Label)
+  ASMJIT_INST_3x(tbnz, Tbnz, Gp, Imm, Imm)
+  ASMJIT_INST_3x(tbnz, Tbnz, Gp, Imm, Label)
+  ASMJIT_INST_3x(tbz, Tbz, Gp, Imm, Imm)
+  ASMJIT_INST_3x(tbz, Tbz, Gp, Imm, Label)
+
+  //! \}
+
+  //! \name Load & Store Instructions
+  //! \{
+
+  ASMJIT_INST_3x(cas, Cas, Gp, Gp, Mem)
+  ASMJIT_INST_3x(casa, Casa, Gp, Gp, Mem)
+  ASMJIT_INST_3x(casab, Casab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(casah, Casah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(casal, Casal, Gp, Gp, Mem)
+  ASMJIT_INST_3x(casalb, Casalb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(casalh, Casalh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(casb, Casb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(cash, Cash, Gp, Gp, Mem)
+  ASMJIT_INST_3x(casl, Casl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(caslb, Caslb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(caslh, Caslh, Gp, Gp, Mem)
+
+  ASMJIT_INST_5x(casp, Casp, Gp, Gp, Gp, Gp, Mem)
+  ASMJIT_INST_5x(caspa, Caspa, Gp, Gp, Gp, Gp, Mem)
+  ASMJIT_INST_5x(caspal, Caspal, Gp, Gp, Gp, Gp, Mem)
+  ASMJIT_INST_5x(caspl, Caspl, Gp, Gp, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(ldadd, Ldadd, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldadda, Ldadda, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddab, Ldaddab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddah, Ldaddah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddal, Ldaddal, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddalb, Ldaddalb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddalh, Ldaddalh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddb, Ldaddb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddh, Ldaddh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddl, Ldaddl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddlb, Ldaddlb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaddlh, Ldaddlh, Gp, Gp, Mem)
+
+  ASMJIT_INST_2x(ldar, Ldar, Gp, Mem)
+  ASMJIT_INST_2x(ldarb, Ldarb, Gp, Mem)
+  ASMJIT_INST_2x(ldarh, Ldarh, Gp, Mem)
+
+  ASMJIT_INST_2x(ldaxr, Ldaxr, Gp, Mem)
+  ASMJIT_INST_2x(ldaxrb, Ldaxrb, Gp, Mem)
+  ASMJIT_INST_2x(ldaxrh, Ldaxrh, Gp, Mem)
+
+  ASMJIT_INST_3x(ldclr, Ldclr, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclra, Ldclra, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclrab, Ldclrab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclrah, Ldclrah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclral, Ldclral, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclralb, Ldclralb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclralh, Ldclralh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclrb, Ldclrb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclrh, Ldclrh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclrl, Ldclrl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclrlb, Ldclrlb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldclrlh, Ldclrlh, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(ldeor, Ldeor, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeora, Ldeora, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeorab, Ldeorab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeorah, Ldeorah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeoral, Ldeoral, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeoralb, Ldeoralb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeoralh, Ldeoralh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeorb, Ldeorb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeorh, Ldeorh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeorl, Ldeorl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeorlb, Ldeorlb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldeorlh, Ldeorlh, Gp, Gp, Mem)
+
+  ASMJIT_INST_2x(ldlar, Ldlar, Gp, Mem)
+  ASMJIT_INST_2x(ldlarb, Ldlarb, Gp, Mem)
+  ASMJIT_INST_2x(ldlarh, Ldlarh, Gp, Mem)
+
+  ASMJIT_INST_3x(ldnp, Ldnp, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(ldp, Ldp, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldpsw, Ldpsw, Gp, Gp, Mem)
+
+  ASMJIT_INST_2x(ldr, Ldr, Gp, Mem)
+  ASMJIT_INST_2x(ldrb, Ldrb, Gp, Mem)
+  ASMJIT_INST_2x(ldrh, Ldrh, Gp, Mem)
+  ASMJIT_INST_2x(ldrsb, Ldrsb, Gp, Mem)
+  ASMJIT_INST_2x(ldrsh, Ldrsh, Gp, Mem)
+  ASMJIT_INST_2x(ldrsw, Ldrsw, Gp, Mem)
+
+  ASMJIT_INST_3x(ldset, Ldset, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldseta, Ldseta, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetab, Ldsetab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetah, Ldsetah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetal, Ldsetal, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetalb, Ldsetalb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetalh, Ldsetalh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetb, Ldsetb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldseth, Ldseth, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetl, Ldsetl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetlb, Ldsetlb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsetlh, Ldsetlh, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(ldsmax, Ldsmax, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxa, Ldsmaxa, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxab, Ldsmaxab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxah, Ldsmaxah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxal, Ldsmaxal, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxalb, Ldsmaxalb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxalh, Ldsmaxalh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxb, Ldsmaxb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxh, Ldsmaxh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxl, Ldsmaxl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxlb, Ldsmaxlb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmaxlh, Ldsmaxlh, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(ldsmin, Ldsmin, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsmina, Ldsmina, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminab, Ldsminab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminah, Ldsminah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminal, Ldsminal, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminalb, Ldsminalb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminalh, Ldsminalh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminb, Ldsminb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminh, Ldsminh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminl, Ldsminl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminlb, Ldsminlb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldsminlh, Ldsminlh, Gp, Gp, Mem)
+
+  ASMJIT_INST_2x(ldtr, Ldtr, Gp, Mem)
+  ASMJIT_INST_2x(ldtrb, Ldtrb, Gp, Mem)
+  ASMJIT_INST_2x(ldtrh, Ldtrh, Gp, Mem)
+  ASMJIT_INST_2x(ldtrsb, Ldtrsb, Gp, Mem)
+  ASMJIT_INST_2x(ldtrsh, Ldtrsh, Gp, Mem)
+  ASMJIT_INST_2x(ldtrsw, Ldtrsw, Gp, Mem)
+
+  ASMJIT_INST_3x(ldumax, Ldumax, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxa, Ldumaxa, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxab, Ldumaxab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxah, Ldumaxah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxal, Ldumaxal, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxalb, Ldumaxalb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxalh, Ldumaxalh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxb, Ldumaxb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxh, Ldumaxh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxl, Ldumaxl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxlb, Ldumaxlb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumaxlh, Ldumaxlh, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(ldumin, Ldumin, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldumina, Ldumina, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminab, Lduminab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminah, Lduminah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminal, Lduminal, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminalb, Lduminalb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminalh, Lduminalh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminb, Lduminb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminh, Lduminh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminl, Lduminl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminlb, Lduminlb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(lduminlh, Lduminlh, Gp, Gp, Mem)
+
+  ASMJIT_INST_2x(ldur, Ldur, Gp, Mem)
+  ASMJIT_INST_2x(ldurb, Ldurb, Gp, Mem)
+  ASMJIT_INST_2x(ldurh, Ldurh, Gp, Mem)
+  ASMJIT_INST_2x(ldursb, Ldursb, Gp, Mem)
+  ASMJIT_INST_2x(ldursh, Ldursh, Gp, Mem)
+  ASMJIT_INST_2x(ldursw, Ldursw, Gp, Mem)
+
+  ASMJIT_INST_3x(ldxp, Ldxp, Gp, Gp, Mem)
+  ASMJIT_INST_3x(ldaxp, Ldaxp, Gp, Gp, Mem)
+
+  ASMJIT_INST_2x(ldxr, Ldxr, Gp, Mem)
+  ASMJIT_INST_2x(ldxrb, Ldxrb, Gp, Mem)
+  ASMJIT_INST_2x(ldxrh, Ldxrh, Gp, Mem)
+
+  ASMJIT_INST_2x(stadd, Stadd, Gp, Mem)
+  ASMJIT_INST_2x(staddb, Staddb, Gp, Mem)
+  ASMJIT_INST_2x(staddh, Staddh, Gp, Mem)
+  ASMJIT_INST_2x(staddl, Staddl, Gp, Mem)
+  ASMJIT_INST_2x(staddlb, Staddlb, Gp, Mem)
+  ASMJIT_INST_2x(staddlh, Staddlh, Gp, Mem)
+
+  ASMJIT_INST_2x(stclr, Stclr, Gp, Mem)
+  ASMJIT_INST_2x(stclrb, Stclrb, Gp, Mem)
+  ASMJIT_INST_2x(stclrh, Stclrh, Gp, Mem)
+  ASMJIT_INST_2x(stclrl, Stclrl, Gp, Mem)
+  ASMJIT_INST_2x(stclrlb, Stclrlb, Gp, Mem)
+  ASMJIT_INST_2x(stclrlh, Stclrlh, Gp, Mem)
+
+  ASMJIT_INST_2x(steor, Steor, Gp, Mem)
+  ASMJIT_INST_2x(steorb, Steorb, Gp, Mem)
+  ASMJIT_INST_2x(steorh, Steorh, Gp, Mem)
+  ASMJIT_INST_2x(steorl, Steorl, Gp, Mem)
+  ASMJIT_INST_2x(steorlb, Steorlb, Gp, Mem)
+  ASMJIT_INST_2x(steorlh, Steorlh, Gp, Mem)
+
+  ASMJIT_INST_2x(stllr, Stllr, Gp, Mem)
+  ASMJIT_INST_2x(stllrb, Stllrb, Gp, Mem)
+  ASMJIT_INST_2x(stllrh, Stllrh, Gp, Mem)
+
+  ASMJIT_INST_2x(stlr, Stllr, Gp, Mem)
+  ASMJIT_INST_2x(stlrb, Stllrb, Gp, Mem)
+  ASMJIT_INST_2x(stlrh, Stllrh, Gp, Mem)
+
+  ASMJIT_INST_3x(stlxr, Stlxr, Gp, Gp, Mem)
+  ASMJIT_INST_3x(stlxrb, Stlxrb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(stlxrh, Stlxrh, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(stnp, Stnp, Gp, Gp, Mem)
+  ASMJIT_INST_3x(stp, Stp, Gp, Gp, Mem)
+
+  ASMJIT_INST_2x(str, Str, Gp, Mem)
+  ASMJIT_INST_2x(strb, Strb, Gp, Mem)
+  ASMJIT_INST_2x(strh, Strh, Gp, Mem)
+
+  ASMJIT_INST_2x(stset, Stset, Gp, Mem)
+  ASMJIT_INST_2x(stsetb, Stsetb, Gp, Mem)
+  ASMJIT_INST_2x(stseth, Stseth, Gp, Mem)
+  ASMJIT_INST_2x(stsetl, Stsetl, Gp, Mem)
+  ASMJIT_INST_2x(stsetlb, Stsetlb, Gp, Mem)
+  ASMJIT_INST_2x(stsetlh, Stsetlh, Gp, Mem)
+
+  ASMJIT_INST_2x(stsmax, Stsmax, Gp, Mem)
+  ASMJIT_INST_2x(stsmaxb, Stsmaxb, Gp, Mem)
+  ASMJIT_INST_2x(stsmaxh, Stsmaxh, Gp, Mem)
+  ASMJIT_INST_2x(stsmaxl, Stsmaxl, Gp, Mem)
+  ASMJIT_INST_2x(stsmaxlb, Stsmaxlb, Gp, Mem)
+  ASMJIT_INST_2x(stsmaxlh, Stsmaxlh, Gp, Mem)
+
+  ASMJIT_INST_2x(stsmin, Stsmin, Gp, Mem)
+  ASMJIT_INST_2x(stsminb, Stsminb, Gp, Mem)
+  ASMJIT_INST_2x(stsminh, Stsminh, Gp, Mem)
+  ASMJIT_INST_2x(stsminl, Stsminl, Gp, Mem)
+  ASMJIT_INST_2x(stsminlb, Stsminlb, Gp, Mem)
+  ASMJIT_INST_2x(stsminlh, Stsminlh, Gp, Mem)
+
+  ASMJIT_INST_2x(sttr, Sttr, Gp, Mem)
+  ASMJIT_INST_2x(sttrb, Sttrb, Gp, Mem)
+  ASMJIT_INST_2x(sttrh, Sttrh, Gp, Mem)
+
+  ASMJIT_INST_2x(stumax, Stumax, Gp, Mem)
+  ASMJIT_INST_2x(stumaxb, Stumaxb, Gp, Mem)
+  ASMJIT_INST_2x(stumaxh, Stumaxh, Gp, Mem)
+  ASMJIT_INST_2x(stumaxl, Stumaxl, Gp, Mem)
+  ASMJIT_INST_2x(stumaxlb, Stumaxlb, Gp, Mem)
+  ASMJIT_INST_2x(stumaxlh, Stumaxlh, Gp, Mem)
+
+  ASMJIT_INST_2x(stumin, Stumin, Gp, Mem)
+  ASMJIT_INST_2x(stuminb, Stuminb, Gp, Mem)
+  ASMJIT_INST_2x(stuminh, Stuminh, Gp, Mem)
+  ASMJIT_INST_2x(stuminl, Stuminl, Gp, Mem)
+  ASMJIT_INST_2x(stuminlb, Stuminlb, Gp, Mem)
+  ASMJIT_INST_2x(stuminlh, Stuminlh, Gp, Mem)
+
+  ASMJIT_INST_2x(stur, Stur, Gp, Mem)
+  ASMJIT_INST_2x(sturb, Sturb, Gp, Mem)
+  ASMJIT_INST_2x(sturh, Sturh, Gp, Mem)
+
+  ASMJIT_INST_4x(stxp, Stxp, Gp, Gp, Gp, Mem)
+  ASMJIT_INST_4x(stlxp, Stlxp, Gp, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(stxr, Stxr, Gp, Gp, Mem)
+  ASMJIT_INST_3x(stxrb, Stxrb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(stxrh, Stxrh, Gp, Gp, Mem)
+
+  ASMJIT_INST_3x(swp, Swp, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swpa, Swpa, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swpab, Swpab, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swpah, Swpah, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swpal, Swpal, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swpalb, Swpalb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swpalh, Swpalh, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swpb, Swpb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swph, Swph, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swpl, Swpl, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swplb, Swplb, Gp, Gp, Mem)
+  ASMJIT_INST_3x(swplh, Swplh, Gp, Gp, Mem)
+  //! \}
+
+  //! \name CRC Instructions (ARMv8.1-A, optional in ARMv8.0-A)
+  //! \{
+
+  ASMJIT_INST_3x(crc32b, Crc32b, Gp, Gp, Gp);
+  ASMJIT_INST_3x(crc32h, Crc32h, Gp, Gp, Gp);
+  ASMJIT_INST_3x(crc32w, Crc32w, Gp, Gp, Gp);
+  ASMJIT_INST_3x(crc32x, Crc32x, Gp, Gp, Gp);
+
+  ASMJIT_INST_3x(crc32cb, Crc32cb, Gp, Gp, Gp);
+  ASMJIT_INST_3x(crc32ch, Crc32ch, Gp, Gp, Gp);
+  ASMJIT_INST_3x(crc32cw, Crc32cw, Gp, Gp, Gp);
+  ASMJIT_INST_3x(crc32cx, Crc32cx, Gp, Gp, Gp);
+
+  //! \}
+
+  //! \name MTE Instructions
+  //! \{
+
+  ASMJIT_INST_2x(autda, Autda, Gp, Gp);
+  ASMJIT_INST_2x(autdb, Autdb, Gp, Gp);
+  ASMJIT_INST_1x(autdza, Autdza, Gp);
+  ASMJIT_INST_1x(autdzb, Autdzb, Gp);
+  ASMJIT_INST_2x(autia, Autia, Gp, Gp);
+  ASMJIT_INST_0x(autia1716, Autia1716);
+  ASMJIT_INST_0x(autiasp, Autiasp);
+  ASMJIT_INST_0x(autiaz, Autiaz);
+  ASMJIT_INST_2x(autib, Autib, Gp, Gp);
+  ASMJIT_INST_0x(autib1716, Autib1716);
+  ASMJIT_INST_0x(autibsp, Autibsp);
+  ASMJIT_INST_0x(autibz, Autibz);
+  ASMJIT_INST_1x(autiza, Autiza, Gp);
+  ASMJIT_INST_1x(autizb, Autizb, Gp);
+
+  ASMJIT_INST_3x(gmi, Gmi, Gp, Gp, Gp);
+
+  ASMJIT_INST_2x(cmpp, Cmpp, Gp, Gp);
+  ASMJIT_INST_4x(addg, Addg, Gp, Gp, Imm, Imm);
+
+  ASMJIT_INST_2x(ldg, Ldg, Gp, Mem)
+  ASMJIT_INST_2x(ldgm, Ldgm, Gp, Mem)
+  ASMJIT_INST_2x(ldraa, Ldraa, Gp, Mem)
+  ASMJIT_INST_2x(ldrab, Ldrab, Gp, Mem)
+
+  ASMJIT_INST_2x(pacda, Pacda, Gp, Gp);
+  ASMJIT_INST_2x(pacdb, Pacdb, Gp, Gp);
+  ASMJIT_INST_1x(pacdza, Pacdza, Gp);
+  ASMJIT_INST_1x(pacdzb, Pacdzb, Gp);
+  ASMJIT_INST_3x(pacga, Pacga, Gp, Gp, Gp);
+
+  ASMJIT_INST_3x(subp, Subp, Gp, Gp, Gp);
+  ASMJIT_INST_3x(subps, Subps, Gp, Gp, Gp);
+  ASMJIT_INST_4x(subg, Subg, Gp, Gp, Imm, Imm);
+
+  ASMJIT_INST_2x(st2g, St2g, Gp, Mem)
+  ASMJIT_INST_2x(stg, Stg, Gp, Mem)
+  ASMJIT_INST_3x(stgp, Stgp, Gp, Gp, Mem)
+  ASMJIT_INST_2x(stgm, Stgm, Gp, Mem)
+  ASMJIT_INST_2x(stzg, Stzg, Gp, Mem)
+  ASMJIT_INST_2x(stz2g, Stz2g, Gp, Mem)
+  ASMJIT_INST_2x(stzgm, Stzgm, Gp, Mem)
+
+  ASMJIT_INST_1x(xpacd, Xpacd, Gp);
+  ASMJIT_INST_1x(xpaci, Xpaci, Gp);
+  ASMJIT_INST_0x(xpaclri, Xpaclri);
+
+  //! \}
+
+  //! \name Hint Instructions
+  //! \{
+
+  ASMJIT_INST_1x(hint, Hint, Imm)
+  ASMJIT_INST_0x(nop, Nop)
+  ASMJIT_INST_0x(sev, Sev)
+  ASMJIT_INST_0x(sevl, Sevl)
+  ASMJIT_INST_0x(wfe, Wfe)
+  ASMJIT_INST_0x(wfi, Wfi)
+  ASMJIT_INST_0x(yield, Yield)
+
+  //! \}
+
+  //! \name SIMD & FP Instructions
+  //! \{
+
+  ASMJIT_INST_2x(abs, Abs_v, Vec, Vec);
+  ASMJIT_INST_3x(add, Add_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(addhn, Addhn_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(addhn2, Addhn2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(addp, Addp_v, Vec, Vec);
+  ASMJIT_INST_3x(addp, Addp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(addv, Addv_v, Vec, Vec);
+  ASMJIT_INST_3x(and_, And_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(bic, Bic_v, Vec, Imm);
+  ASMJIT_INST_3x(bic, Bic_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(bic, Bic_v, Vec, Imm, Imm);
+  ASMJIT_INST_3x(bif, Bif_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(bit, Bit_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(bsl, Bsl_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(cls, Cls_v, Vec, Vec);
+  ASMJIT_INST_2x(clz, Clz_v, Vec, Vec);
+  ASMJIT_INST_3x(cmeq, Cmeq_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(cmeq, Cmeq_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(cmge, Cmge_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(cmge, Cmge_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(cmgt, Cmgt_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(cmgt, Cmgt_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(cmhi, Cmhi_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(cmhs, Cmhs_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(cmle, Cmle_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(cmlt, Cmlt_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(cmtst, Cmtst_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(cnt, Cnt_v, Vec, Vec);
+  ASMJIT_INST_2x(dup, Dup_v, Vec, Gp);
+  ASMJIT_INST_2x(dup, Dup_v, Vec, Vec);
+  ASMJIT_INST_3x(eor, Eor_v, Vec, Vec, Vec);
+  ASMJIT_INST_4x(ext, Ext_v, Vec, Vec, Vec, Imm);
+  ASMJIT_INST_3x(fabd, Fabd_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fabs, Fabs_v, Vec, Vec);
+  ASMJIT_INST_3x(facge, Facge_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(facgt, Facgt_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fadd, Fadd_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(faddp, Faddp_v, Vec, Vec);
+  ASMJIT_INST_3x(faddp, Faddp_v, Vec, Vec, Vec);
+  ASMJIT_INST_4x(fccmp, Fccmp_v, Vec, Vec, Imm, Imm);
+  ASMJIT_INST_4x(fccmpe, Fccmpe_v, Vec, Vec, Imm, Imm);
+  ASMJIT_INST_3x(fcmeq, Fcmeq_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fcmeq, Fcmeq_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(fcmge, Fcmge_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fcmge, Fcmge_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(fcmgt, Fcmgt_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fcmgt, Fcmgt_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(fcmle, Fcmle_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(fcmlt, Fcmlt_v, Vec, Vec, Imm);
+  ASMJIT_INST_2x(fcmp, Fcmp_v, Vec, Vec);
+  ASMJIT_INST_2x(fcmp, Fcmp_v, Vec, Imm);
+  ASMJIT_INST_2x(fcmpe, Fcmpe_v, Vec, Vec);
+  ASMJIT_INST_2x(fcmpe, Fcmpe_v, Vec, Imm);
+  ASMJIT_INST_4x(fcsel, Fcsel_v, Vec, Vec, Vec, Imm);
+  ASMJIT_INST_2x(fcvt, Fcvt_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtas, Fcvtas_v, Gp, Vec);
+  ASMJIT_INST_2x(fcvtas, Fcvtas_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtau, Fcvtau_v, Gp, Vec);
+  ASMJIT_INST_2x(fcvtau, Fcvtau_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtl, Fcvtl_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtl2, Fcvtl2_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtms, Fcvtms_v, Gp, Vec);
+  ASMJIT_INST_2x(fcvtms, Fcvtms_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtmu, Fcvtmu_v, Gp, Vec);
+  ASMJIT_INST_2x(fcvtmu, Fcvtmu_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtn, Fcvtn_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtn2, Fcvtn2_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtns, Fcvtns_v, Gp, Vec);
+  ASMJIT_INST_2x(fcvtns, Fcvtns_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtnu, Fcvtnu_v, Gp, Vec);
+  ASMJIT_INST_2x(fcvtnu, Fcvtnu_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtps, Fcvtps_v, Gp, Vec);
+  ASMJIT_INST_2x(fcvtps, Fcvtps_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtpu, Fcvtpu_v, Gp, Vec);
+  ASMJIT_INST_2x(fcvtpu, Fcvtpu_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtxn, Fcvtxn_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtxn2, Fcvtxn2_v, Vec, Vec);
+  ASMJIT_INST_2x(fcvtzs, Fcvtzs_v, Gp, Vec);
+  ASMJIT_INST_3x(fcvtzs, Fcvtzs_v, Gp, Vec, Imm);
+  ASMJIT_INST_2x(fcvtzs, Fcvtzs_v, Vec, Vec);
+  ASMJIT_INST_3x(fcvtzs, Fcvtzs_v, Vec, Vec, Imm);
+  ASMJIT_INST_2x(fcvtzu, Fcvtzu_v, Gp, Vec);
+  ASMJIT_INST_3x(fcvtzu, Fcvtzu_v, Gp, Vec, Imm);
+  ASMJIT_INST_2x(fcvtzu, Fcvtzu_v, Vec, Vec);
+  ASMJIT_INST_3x(fcvtzu, Fcvtzu_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(fdiv, Fdiv_v, Vec, Vec, Vec);
+  ASMJIT_INST_4x(fmadd, Fmadd_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmax, Fmax_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmaxnm, Fmaxnm_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmaxnmp, Fmaxnmp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fmaxnmp, Fmaxnmp_v, Vec, Vec);
+  ASMJIT_INST_2x(fmaxnmv, Fmaxnmv_v, Vec, Vec);
+  ASMJIT_INST_3x(fmaxp, Fmaxp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fmaxp, Fmaxp_v, Vec, Vec);
+  ASMJIT_INST_2x(fmaxv, Fmaxv_v, Vec, Vec);
+  ASMJIT_INST_3x(fmin, Fmin_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fminnm, Fminnm_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fminnmv, Fminnmv_v, Vec, Vec);
+  ASMJIT_INST_3x(fminnmp, Fminnmp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fminnmp, Fminnmp_v, Vec, Vec);
+  ASMJIT_INST_2x(fminp, Fminp_v, Vec, Vec);
+  ASMJIT_INST_3x(fminp, Fminp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fminv, Fminv_v, Vec, Vec);
+  ASMJIT_INST_3x(fmla, Fmla_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmls, Fmls_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fmov, Fmov_v, Gp, Vec);
+  ASMJIT_INST_2x(fmov, Fmov_v, Vec, Gp);
+  ASMJIT_INST_2x(fmov, Fmov_v, Vec, Vec);
+  ASMJIT_INST_2x(fmov, Fmov_v, Vec, Imm);
+  ASMJIT_INST_4x(fmsub, Fmsub_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmul, Fmul_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmulx, Fmulx_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fneg, Fneg_v, Vec, Vec);
+  ASMJIT_INST_4x(fnmadd, Fnmadd_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_4x(fnmsub, Fnmsub_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fnmul, Fnmul_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(frecpe, Frecpe_v, Vec, Vec);
+  ASMJIT_INST_3x(frecps, Frecps_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(frecpx, Frecpx_v, Vec, Vec);
+  ASMJIT_INST_2x(frint32x, Frint32x_v, Vec, Vec);
+  ASMJIT_INST_2x(frint32z, Frint32z_v, Vec, Vec);
+  ASMJIT_INST_2x(frint64x, Frint64x_v, Vec, Vec);
+  ASMJIT_INST_2x(frint64z, Frint64z_v, Vec, Vec);
+  ASMJIT_INST_2x(frinta, Frinta_v, Vec, Vec);
+  ASMJIT_INST_2x(frinti, Frinti_v, Vec, Vec);
+  ASMJIT_INST_2x(frintm, Frintm_v, Vec, Vec);
+  ASMJIT_INST_2x(frintn, Frintn_v, Vec, Vec);
+  ASMJIT_INST_2x(frintp, Frintp_v, Vec, Vec);
+  ASMJIT_INST_2x(frintx, Frintx_v, Vec, Vec);
+  ASMJIT_INST_2x(frintz, Frintz_v, Vec, Vec);
+  ASMJIT_INST_2x(frsqrte, Frsqrte_v, Vec, Vec);
+  ASMJIT_INST_3x(frsqrts, Frsqrts_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(fsqrt, Fsqrt_v, Vec, Vec);
+  ASMJIT_INST_3x(fsub, Fsub_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(ins, Ins_v, Vec, Gp);
+  ASMJIT_INST_2x(ins, Ins_v, Vec, Vec);
+  ASMJIT_INST_2x(ld1, Ld1_v, Vec, Mem);
+  ASMJIT_INST_3x(ld1, Ld1_v, Vec, Vec, Mem);
+  ASMJIT_INST_4x(ld1, Ld1_v, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_5x(ld1, Ld1_v, Vec, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_2x(ld1r, Ld1r_v, Vec, Mem);
+  ASMJIT_INST_3x(ld2, Ld2_v, Vec, Vec, Mem);
+  ASMJIT_INST_3x(ld2r, Ld2r_v, Vec, Vec, Mem);
+  ASMJIT_INST_4x(ld3, Ld3_v, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_4x(ld3r, Ld3r_v, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_5x(ld4, Ld4_v, Vec, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_5x(ld4r, Ld4r_v, Vec, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_3x(ldnp, Ldnp_v, Vec, Vec, Mem);
+  ASMJIT_INST_3x(ldp, Ldp_v, Vec, Vec, Mem);
+  ASMJIT_INST_2x(ldr, Ldr_v, Vec, Mem);
+  ASMJIT_INST_2x(ldur, Ldur_v, Vec, Mem);
+  ASMJIT_INST_3x(mla, Mla_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(mls, Mls_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(mov, Mov_v, Vec, Vec);
+  ASMJIT_INST_2x(mov, Mov_v, Gp, Vec);
+  ASMJIT_INST_2x(mov, Mov_v, Vec, Gp);
+  ASMJIT_INST_2x(movi, Movi_v, Vec, Imm);
+  ASMJIT_INST_3x(movi, Movi_v, Vec, Imm, Imm);
+  ASMJIT_INST_3x(mul, Mul_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(mvn, Mvn_v, Vec, Vec);
+  ASMJIT_INST_2x(mvni, Mvni_v, Vec, Imm);
+  ASMJIT_INST_3x(mvni, Mvni_v, Vec, Imm, Imm);
+  ASMJIT_INST_2x(neg, Neg_v, Vec, Vec);
+  ASMJIT_INST_2x(not_, Not_v, Vec, Vec);
+  ASMJIT_INST_3x(orn, Orn_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(orr, Orr_v, Vec, Imm);
+  ASMJIT_INST_3x(orr, Orr_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(orr, Orr_v, Vec, Imm, Imm);
+  ASMJIT_INST_3x(pmul, Pmul_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(pmull, Pmull_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(pmull2, Pmull2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(raddhn, Raddhn_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(raddhn2, Raddhn2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(rbit, Rbit_v, Vec, Vec);
+  ASMJIT_INST_2x(rev16, Rev16_v, Vec, Vec);
+  ASMJIT_INST_2x(rev32, Rev32_v, Vec, Vec);
+  ASMJIT_INST_2x(rev64, Rev64_v, Vec, Vec);
+  ASMJIT_INST_3x(rshrn, Rshrn_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(rshrn2, Rshrn2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(rsubhn, Rsubhn_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(rsubhn2, Rsubhn2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(saba, Saba_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sabal, Sabal_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sabal2, Sabal2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sabd, Sabd_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sabdl, Sabdl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sabdl2, Sabdl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sadalp, Sadalp_v, Vec, Vec);
+  ASMJIT_INST_3x(saddl, Saddl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(saddl2, Saddl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(saddlp, Saddlp_v, Vec, Vec);
+  ASMJIT_INST_2x(saddlv, Saddlv_v, Vec, Vec);
+  ASMJIT_INST_3x(saddw, Saddw_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(saddw2, Saddw2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(scvtf, Scvtf_v, Vec, Gp);
+  ASMJIT_INST_3x(scvtf, Scvtf_v, Vec, Gp, Imm);
+  ASMJIT_INST_2x(scvtf, Scvtf_v, Vec, Vec);
+  ASMJIT_INST_3x(scvtf, Scvtf_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(shadd, Shadd_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(shl, Shl_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(shll, Shll_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(shll2, Shll2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(shrn, Shrn_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(shrn2, Shrn2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(shsub, Shsub_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sli, Sli_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(smax, Smax_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(smaxp, Smaxp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(smaxv, Smaxv_v, Vec, Vec);
+  ASMJIT_INST_3x(smin, Smin_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sminp, Sminp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sminv, Sminv_v, Vec, Vec);
+  ASMJIT_INST_3x(smlal, Smlal_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(smlal2, Smlal2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(smlsl, Smlsl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(smlsl2, Smlsl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(smov, Smov_v, Gp, Vec);
+  ASMJIT_INST_3x(smull, Smull_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(smull2, Smull2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sqabs, Sqabs_v, Vec, Vec);
+  ASMJIT_INST_3x(sqadd, Sqadd_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqdmlal, Sqdmlal_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqdmlal2, Sqdmlal2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqdmlsl, Sqdmlsl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqdmlsl2, Sqdmlsl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqdmulh, Sqdmulh_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqdmull, Sqdmull_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqdmull2, Sqdmull2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sqneg, Sqneg_v, Vec, Vec);
+  ASMJIT_INST_3x(sqrdmulh, Sqrdmulh_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqrshl, Sqrshl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqrshrn, Sqrshrn_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqrshrn2, Sqrshrn2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqrshrun, Sqrshrun_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqrshrun2, Sqrshrun2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqshl, Sqshl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqshl, Sqshl_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqshlu, Sqshlu_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqshrn, Sqshrn_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqshrn2, Sqshrn2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqshrun, Sqshrun_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqshrun2, Sqshrun2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sqsub, Sqsub_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sqxtn, Sqxtn_v, Vec, Vec);
+  ASMJIT_INST_2x(sqxtn2, Sqxtn2_v, Vec, Vec);
+  ASMJIT_INST_2x(sqxtun, Sqxtun_v, Vec, Vec);
+  ASMJIT_INST_2x(sqxtun2, Sqxtun2_v, Vec, Vec);
+  ASMJIT_INST_3x(srhadd, Srhadd_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sri, Sri_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(srshl, Srshl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(srshr, Srshr_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(srsra, Srsra_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sshl, Sshl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sshll, Sshll_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sshll2, Sshll2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(sshr, Sshr_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(ssra, Ssra_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(ssubl, Ssubl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(ssubl2, Ssubl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(ssubw, Ssubw_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(ssubw2, Ssubw2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(st1, St1_v, Vec, Mem);
+  ASMJIT_INST_3x(st1, St1_v, Vec, Vec, Mem);
+  ASMJIT_INST_4x(st1, St1_v, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_5x(st1, St1_v, Vec, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_3x(st2, St2_v, Vec, Vec, Mem);
+  ASMJIT_INST_4x(st3, St3_v, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_5x(st4, St4_v, Vec, Vec, Vec, Vec, Mem);
+  ASMJIT_INST_3x(stnp, Stnp_v, Vec, Vec, Mem);
+  ASMJIT_INST_3x(stp, Stp_v, Vec, Vec, Mem);
+  ASMJIT_INST_2x(str, Str_v, Vec, Mem);
+  ASMJIT_INST_2x(stur, Stur_v, Vec, Mem);
+  ASMJIT_INST_3x(sub, Sub_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(subhn, Subhn_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(subhn2, Subhn2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(suqadd, Suqadd_v, Vec, Vec);
+  ASMJIT_INST_2x(sxtl, Sxtl_v, Vec, Vec);
+  ASMJIT_INST_2x(sxtl2, Sxtl2_v, Vec, Vec);
+  ASMJIT_INST_3x(tbl, Tbl_v, Vec, Vec, Vec);
+  ASMJIT_INST_4x(tbl, Tbl_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_5x(tbl, Tbl_v, Vec, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_6x(tbl, Tbl_v, Vec, Vec, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_3x(tbx, Tbx_v, Vec, Vec, Vec);
+  ASMJIT_INST_4x(tbx, Tbx_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_5x(tbx, Tbx_v, Vec, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_6x(tbx, Tbx_v, Vec, Vec, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_3x(trn1, Trn1_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(trn2, Trn2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uaba, Uaba_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uabal, Uabal_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uabal2, Uabal2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uabd, Uabd_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uabdl, Uabdl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uabdl2, Uabdl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(uadalp, Uadalp_v, Vec, Vec);
+  ASMJIT_INST_3x(uaddl, Uaddl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uaddl2, Uaddl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(uaddlp, Uaddlp_v, Vec, Vec);
+  ASMJIT_INST_2x(uaddlv, Uaddlv_v, Vec, Vec);
+  ASMJIT_INST_3x(uaddw, Uaddw_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uaddw2, Uaddw2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(ucvtf, Ucvtf_v, Vec, Gp);
+  ASMJIT_INST_3x(ucvtf, Ucvtf_v, Vec, Gp, Imm);
+  ASMJIT_INST_2x(ucvtf, Ucvtf_v, Vec, Vec);
+  ASMJIT_INST_3x(ucvtf, Ucvtf_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(uhadd, Uhadd_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uhsub, Uhsub_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(umax, Umax_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(umaxp, Umaxp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(umaxv, Umaxv_v, Vec, Vec);
+  ASMJIT_INST_3x(umin, Umin_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uminp, Uminp_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(uminv, Uminv_v, Vec, Vec);
+  ASMJIT_INST_3x(umlal, Umlal_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(umlal2, Umlal2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(umlsl, Umlsl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(umlsl2, Umlsl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(umov, Umov_v, Gp, Vec);
+  ASMJIT_INST_3x(umull, Umull_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(umull2, Umull2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uqadd, Uqadd_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uqrshl, Uqrshl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uqrshl, Uqrshl_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(uqrshrn, Uqrshrn_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(uqrshrn2, Uqrshrn2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(uqshl, Uqshl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uqshl, Uqshl_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(uqshrn, Uqshrn_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(uqshrn2, Uqshrn2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(uqsub, Uqsub_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(uqxtn, Uqxtn_v, Vec, Vec);
+  ASMJIT_INST_2x(uqxtn2, Uqxtn2_v, Vec, Vec);
+  ASMJIT_INST_2x(urecpe, Urecpe_v, Vec, Vec);
+  ASMJIT_INST_3x(urhadd, Urhadd_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(urshl, Urshl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(urshr, Urshr_v, Vec, Vec, Imm);
+  ASMJIT_INST_2x(ursqrte, Ursqrte_v, Vec, Vec);
+  ASMJIT_INST_3x(ursra, Ursra_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(ushl, Ushl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(ushll, Ushll_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(ushll2, Ushll2_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(ushr, Ushr_v, Vec, Vec, Imm);
+  ASMJIT_INST_2x(usqadd, Usqadd_v, Vec, Vec);
+  ASMJIT_INST_3x(usra, Usra_v, Vec, Vec, Imm);
+  ASMJIT_INST_3x(usubl, Usubl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(usubl2, Usubl2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(usubw, Usubw_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(usubw2, Usubw2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(uxtl, Uxtl_v, Vec, Vec);
+  ASMJIT_INST_2x(uxtl2, Uxtl2_v, Vec, Vec);
+  ASMJIT_INST_3x(uzp1, Uzp1_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(uzp2, Uzp2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(xtn, Xtn_v, Vec, Vec);
+  ASMJIT_INST_2x(xtn2, Xtn2_v, Vec, Vec);
+  ASMJIT_INST_3x(zip1, Zip1_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(zip2, Zip2_v, Vec, Vec, Vec);
+
+  //! \}
+
+  //! \name AES Instructions
+  //! \{
+
+  ASMJIT_INST_2x(aesd, Aesd_v, Vec, Vec);
+  ASMJIT_INST_2x(aese, Aese_v, Vec, Vec);
+  ASMJIT_INST_2x(aesimc, Aesimc_v, Vec, Vec);
+  ASMJIT_INST_2x(aesmc, Aesmc_v, Vec, Vec);
+
+  //! \}
+
+  //! \name SHA1 Instructions
+  //! \{
+
+  ASMJIT_INST_3x(sha1c, Sha1c_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sha1h, Sha1h_v, Vec, Vec);
+  ASMJIT_INST_3x(sha1m, Sha1m_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sha1p, Sha1p_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sha1su0, Sha1su0_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sha1su1, Sha1su1_v, Vec, Vec);
+
+  //! \}
+
+  //! \name SHA2 Instructions
+  //! \{
+
+  ASMJIT_INST_3x(sha256h, Sha256h_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sha256h2, Sha256h2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sha256su0, Sha256su0_v, Vec, Vec);
+  ASMJIT_INST_3x(sha256su1, Sha256su1_v, Vec, Vec, Vec);
+
+  //! \}
+
+  //! \name RDMA Instructions (ARMv8.1-A)
+  //! \{
+
+  ASMJIT_INST_3x(sqrdmlah, Sqrdmlah_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sqrdmlsh, Sqrdmlsh_v, Vec, Vec, Vec);
+
+  //! \}
+
+  //! \name FCMA Instruction (ARMv8.3-A)
+  //! \{
+
+  ASMJIT_INST_4x(fcadd, Fcadd_v, Vec, Vec, Vec, Imm);
+  ASMJIT_INST_4x(fcmla, Fcmla_v, Vec, Vec, Vec, Imm);
+
+  //! \}
+
+  //! \name FJCVTZS Instruction (ARMv8.3-A)
+  //! \{
+
+  ASMJIT_INST_2x(fjcvtzs, Fjcvtzs_v, Gp, Vec);
+
+  //! \}
+
+  //! \name FP16FML Instructions (ARMv8.4-A, optional in ARMv8.2-A)
+  //! \{
+
+  ASMJIT_INST_3x(fmlal, Fmlal_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmlal2, Fmlal2_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmlsl, Fmlsl_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(fmlsl2, Fmlsl2_v, Vec, Vec, Vec);
+
+
+  //! \}
+
+  //! \name SHA3 Instructions (ARMv8.4-A, optional in ARMv8.2-A)
+  //! \{
+
+  ASMJIT_INST_4x(bcax, Bcax_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_4x(eor3, Eor3_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_3x(rax1, Rax1_v, Vec, Vec, Vec);
+  ASMJIT_INST_4x(xar, Xar_v, Vec, Vec, Vec, Imm);
+
+  //! \}
+
+  //! \name SHA512 Instructions (ARMv8.4-A)
+  //! \{
+
+  ASMJIT_INST_3x(sha512h, Sha512h_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sha512h2, Sha512h2_v, Vec, Vec, Vec);
+  ASMJIT_INST_2x(sha512su0, Sha512su0_v, Vec, Vec);
+  ASMJIT_INST_3x(sha512su1, Sha512su1_v, Vec, Vec, Vec);
+
+  //! \}
+
+  //! \name SM3 Instructions (ARMv8.4-A)
+  //! \{
+
+  ASMJIT_INST_3x(sm3partw1, Sm3partw1_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sm3partw2, Sm3partw2_v, Vec, Vec, Vec);
+  ASMJIT_INST_4x(sm3ss1, Sm3ss1_v, Vec, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sm3tt1a, Sm3tt1a_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sm3tt1b, Sm3tt1b_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sm3tt2a, Sm3tt2a_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sm3tt2b, Sm3tt2b_v, Vec, Vec, Vec);
+
+  //! \}
+
+  //! \name SM4 Instructions (ARMv8.4-A)
+  //! \{
+
+  ASMJIT_INST_2x(sm4e, Sm4e_v, Vec, Vec);
+  ASMJIT_INST_3x(sm4ekey, Sm4ekey_v, Vec, Vec, Vec);
+
+  //! \}
+
+  //! \name DOTPROD Instructions (ARMv8.4-A, optional in ARMv8.2-A)
+  //! \{
+
+  ASMJIT_INST_3x(sdot, Sdot_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(udot, Udot_v, Vec, Vec, Vec);
+
+  //! \}
+
+  //! \name BF16 Instructions (ARMv8.6-A)
+  //! \{
+
+  ASMJIT_INST_2x(bfcvt, Bfcvt_v, Vec, Vec);
+  ASMJIT_INST_2x(bfcvtn, Bfcvtn_v, Vec, Vec);
+  ASMJIT_INST_2x(bfcvtn2, Bfcvtn2_v, Vec, Vec);
+  ASMJIT_INST_3x(bfmlalb, Bfmlalb_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(bfmlalt, Bfmlalt_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(bfmmla, Bfmmla_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(bfdot, Bfdot_v, Vec, Vec, Vec);
+
+  //! \}
+
+  //! \name I8MM Instructions (ARMv8.6-A)
+  //! \{
+
+  ASMJIT_INST_3x(smmla, Smmla_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(sudot, Sudot_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(ummla, Ummla_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(usdot, Usdot_v, Vec, Vec, Vec);
+  ASMJIT_INST_3x(usmmla, Usmmla_v, Vec, Vec, Vec);
+
+  //! \}
+};
+
+//! Emitter (ARM).
+//!
+//! \note This class cannot be instantiated, you can only cast to it and use it as emitter that emits to either
+//! `a64::Assembler`, `a64::Builder`, or `a64::Compiler` (use with caution with `a64::Compiler` as it requires
+//! virtual registers).
+class Emitter : public BaseEmitter, public EmitterExplicitT<Emitter> {
+  ASMJIT_NONCONSTRUCTIBLE(Emitter)
+};
+
+//! \}
+
+#undef ASMJIT_INST_0x
+#undef ASMJIT_INST_1x
+#undef ASMJIT_INST_2x
+#undef ASMJIT_INST_3x
+#undef ASMJIT_INST_4x
+#undef ASMJIT_INST_5x
+#undef ASMJIT_INST_6x
+#undef ASMJIT_INST_1cc
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_A64EMITTER_H_INCLUDED

libraries/asmjit/asmjit/arm/a64formatter.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#ifndef ASMJIT_NO_LOGGING
+
+#include "../core/misc_p.h"
+#include "../core/support.h"
+#include "../arm/a64formatter_p.h"
+#include "../arm/a64instapi_p.h"
+#include "../arm/a64instdb_p.h"
+#include "../arm/a64operand.h"
+
+#ifndef ASMJIT_NO_COMPILER
+  #include "../core/compiler.h"
+#endif
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::FormatterInternal - Format Register
+// ========================================
+
+ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  RegType regType,
+  uint32_t rId,
+  uint32_t elementType,
+  uint32_t elementIndex) noexcept {
+
+  DebugUtils::unused(flags);
+  DebugUtils::unused(arch);
+
+  static const char bhsdq[] = "bhsdq";
+
+  bool virtRegFormatted = false;
+
+#ifndef ASMJIT_NO_COMPILER
+  if (Operand::isVirtId(rId)) {
+    if (emitter && emitter->isCompiler()) {
+      const BaseCompiler* cc = static_cast<const BaseCompiler*>(emitter);
+      if (cc->isVirtIdValid(rId)) {
+        VirtReg* vReg = cc->virtRegById(rId);
+        ASMJIT_ASSERT(vReg != nullptr);
+
+        const char* name = vReg->name();
+        if (name && name[0] != '\0')
+          ASMJIT_PROPAGATE(sb.append(name));
+        else
+          ASMJIT_PROPAGATE(sb.appendFormat("%%%u", unsigned(Operand::virtIdToIndex(rId))));
+
+        virtRegFormatted = true;
+      }
+    }
+  }
+#else
+  DebugUtils::unused(emitter, flags);
+#endif
+
+  if (!virtRegFormatted) {
+    char letter = '\0';
+    switch (regType) {
+      case RegType::kARM_GpW:
+        if (rId == Gp::kIdZr)
+          return sb.append("wzr");
+        if (rId == Gp::kIdSp)
+          return sb.append("wsp");
+
+        letter = 'w';
+        break;
+
+      case RegType::kARM_GpX:
+        if (rId == Gp::kIdZr)
+          return sb.append("xzr");
+        if (rId == Gp::kIdSp)
+          return sb.append("sp");
+
+        letter = 'x';
+        break;
+
+      case RegType::kARM_VecB:
+      case RegType::kARM_VecH:
+      case RegType::kARM_VecS:
+      case RegType::kARM_VecD:
+      case RegType::kARM_VecV:
+        letter = bhsdq[uint32_t(regType) - uint32_t(RegType::kARM_VecB)];
+        if (elementType)
+          letter = 'v';
+        break;
+
+      default:
+        ASMJIT_PROPAGATE(sb.appendFormat("<Reg-%u>?$u", uint32_t(regType), rId));
+        break;
+    }
+
+    if (letter)
+      ASMJIT_PROPAGATE(sb.appendFormat("%c%u", letter, rId));
+  }
+
+  if (elementType) {
+    char elementLetter = '\0';
+    uint32_t elementCount = 0;
+
+    switch (elementType) {
+      case Vec::kElementTypeB:
+        elementLetter = 'b';
+        elementCount = 16;
+        break;
+
+      case Vec::kElementTypeH:
+        elementLetter = 'h';
+        elementCount = 8;
+        break;
+
+      case Vec::kElementTypeS:
+        elementLetter = 's';
+        elementCount = 4;
+        break;
+
+      case Vec::kElementTypeD:
+        elementLetter = 'd';
+        elementCount = 2;
+        break;
+
+      default:
+        return sb.append(".<Unknown>");
+    }
+
+    if (elementLetter) {
+      if (elementIndex == 0xFFFFFFFFu) {
+        if (regType == RegType::kARM_VecD)
+          elementCount /= 2u;
+        ASMJIT_PROPAGATE(sb.appendFormat(".%u%c", elementCount, elementLetter));
+      }
+      else {
+        ASMJIT_PROPAGATE(sb.appendFormat(".%c[%u]", elementLetter, elementIndex));
+      }
+    }
+  }
+
+  return kErrorOk;
+}
+
+// a64::FormatterInternal - Format Operand
+// =======================================
+
+ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  const Operand_& op) noexcept {
+
+  if (op.isReg()) {
+    const BaseReg& reg = op.as<BaseReg>();
+
+    uint32_t elementType = op.as<Vec>().elementType();
+    uint32_t elementIndex = op.as<Vec>().elementIndex();
+
+    if (!op.as<Vec>().hasElementIndex())
+      elementIndex = 0xFFFFFFFFu;
+
+    return formatRegister(sb, flags, emitter, arch, reg.type(), reg.id(), elementType, elementIndex);
+  }
+
+  if (op.isMem()) {
+    const Mem& m = op.as<Mem>();
+    ASMJIT_PROPAGATE(sb.append('['));
+
+    if (m.hasBase()) {
+      if (m.hasBaseLabel()) {
+        ASMJIT_PROPAGATE(Formatter::formatLabel(sb, flags, emitter, m.baseId()));
+      }
+      else {
+        FormatFlags modifiedFlags = flags;
+        if (m.isRegHome()) {
+          ASMJIT_PROPAGATE(sb.append('&'));
+          modifiedFlags &= ~FormatFlags::kRegCasts;
+        }
+        ASMJIT_PROPAGATE(formatRegister(sb, modifiedFlags, emitter, arch, m.baseType(), m.baseId()));
+      }
+    }
+    else {
+      // ARM really requires base.
+      if (m.hasIndex() || m.hasOffset()) {
+        ASMJIT_PROPAGATE(sb.append("<None>"));
+      }
+    }
+
+    // The post index makes it look like there was another operand, but it's
+    // still the part of AsmJit's `arm::Mem` operand so it's consistent with
+    // other architectures.
+    if (m.isPostIndex())
+      ASMJIT_PROPAGATE(sb.append(']'));
+
+    if (m.hasIndex()) {
+      ASMJIT_PROPAGATE(sb.append(", "));
+      ASMJIT_PROPAGATE(formatRegister(sb, flags, emitter, arch, m.indexType(), m.indexId()));
+    }
+
+    if (m.hasOffset()) {
+      ASMJIT_PROPAGATE(sb.append(", "));
+
+      int64_t off = int64_t(m.offset());
+      uint32_t base = 10;
+
+      if (Support::test(flags, FormatFlags::kHexOffsets) && uint64_t(off) > 9)
+        base = 16;
+
+      if (base == 10) {
+        ASMJIT_PROPAGATE(sb.appendInt(off, base));
+      }
+      else {
+        ASMJIT_PROPAGATE(sb.append("0x"));
+        ASMJIT_PROPAGATE(sb.appendUInt(uint64_t(off), base));
+      }
+    }
+
+    if (m.hasShift()) {
+      ASMJIT_PROPAGATE(sb.append(' '));
+      if (!m.isPreOrPost())
+        ASMJIT_PROPAGATE(formatShiftOp(sb, (ShiftOp)m.predicate()));
+      ASMJIT_PROPAGATE(sb.appendFormat(" %u", m.shift()));
+    }
+
+    if (!m.isPostIndex())
+      ASMJIT_PROPAGATE(sb.append(']'));
+
+    if (m.isPreIndex())
+      ASMJIT_PROPAGATE(sb.append('!'));
+
+    return kErrorOk;
+  }
+
+  if (op.isImm()) {
+    const Imm& i = op.as<Imm>();
+    int64_t val = i.value();
+
+    if (Support::test(flags, FormatFlags::kHexImms) && uint64_t(val) > 9) {
+      ASMJIT_PROPAGATE(sb.append("0x"));
+      return sb.appendUInt(uint64_t(val), 16);
+    }
+    else {
+      return sb.appendInt(val, 10);
+    }
+  }
+
+  if (op.isLabel()) {
+    return Formatter::formatLabel(sb, flags, emitter, op.id());
+  }
+
+  return sb.append("<None>");
+}
+
+// a64::FormatterInternal - Format Instruction
+// ===========================================
+
+ASMJIT_FAVOR_SIZE Error FormatterInternal::formatInstruction(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept {
+
+  DebugUtils::unused(arch);
+
+  // Format instruction options and instruction mnemonic.
+  InstId instId = inst.realId();
+  if (instId < Inst::_kIdCount)
+    ASMJIT_PROPAGATE(InstInternal::instIdToString(arch, instId, sb));
+  else
+    ASMJIT_PROPAGATE(sb.appendFormat("[InstId=#%u]", unsigned(instId)));
+
+  CondCode cc = inst.armCondCode();
+  if (cc != CondCode::kAL) {
+    ASMJIT_PROPAGATE(sb.append('.'));
+    ASMJIT_PROPAGATE(formatCondCode(sb, cc));
+  }
+
+  for (uint32_t i = 0; i < opCount; i++) {
+    const Operand_& op = operands[i];
+    if (op.isNone())
+      break;
+
+    ASMJIT_PROPAGATE(sb.append(i == 0 ? " " : ", "));
+    ASMJIT_PROPAGATE(formatOperand(sb, flags, emitter, arch, op));
+  }
+
+  return kErrorOk;
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_LOGGING

libraries/asmjit/asmjit/arm/a64formatter_p.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64FORMATTER_P_H_INCLUDED
+#define ASMJIT_ARM_A64FORMATTER_P_H_INCLUDED
+
+#include "../core/api-config.h"
+#ifndef ASMJIT_NO_LOGGING
+
+#include "../core/formatter.h"
+#include "../core/string.h"
+#include "../arm/armformatter_p.h"
+#include "../arm/a64globals.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \cond INTERNAL
+//! \addtogroup asmjit_a64
+//! \{
+
+namespace FormatterInternal {
+
+using namespace arm::FormatterInternal;
+
+Error ASMJIT_CDECL formatRegister(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  RegType regType,
+  uint32_t regId,
+  uint32_t elementType = 0,
+  uint32_t elementIndex = 0xFFFFFFFFu) noexcept;
+
+Error ASMJIT_CDECL formatOperand(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  const Operand_& op) noexcept;
+
+Error ASMJIT_CDECL formatInstruction(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept;
+
+} // {FormatterInternal}
+
+//! \}
+//! \endcond
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_LOGGING
+#endif // ASMJIT_ARM_A64FORMATTER_P_H_INCLUDED

libraries/asmjit/asmjit/arm/a64func.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64)
+
+#include "../arm/a64func_p.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+namespace FuncInternal {
+
+static inline bool shouldThreatAsCDecl(CallConvId ccId) noexcept {
+  return ccId == CallConvId::kCDecl ||
+         ccId == CallConvId::kStdCall ||
+         ccId == CallConvId::kFastCall ||
+         ccId == CallConvId::kVectorCall ||
+         ccId == CallConvId::kThisCall ||
+         ccId == CallConvId::kRegParm1 ||
+         ccId == CallConvId::kRegParm2 ||
+         ccId == CallConvId::kRegParm3;
+}
+
+static RegType regTypeFromFpOrVecTypeId(TypeId typeId) noexcept {
+  if (typeId == TypeId::kFloat32)
+    return RegType::kARM_VecS;
+  else if (typeId == TypeId::kFloat64)
+    return RegType::kARM_VecD;
+  else if (TypeUtils::isVec32(typeId))
+    return RegType::kARM_VecS;
+  else if (TypeUtils::isVec64(typeId))
+    return RegType::kARM_VecD;
+  else if (TypeUtils::isVec128(typeId))
+    return RegType::kARM_VecV;
+  else
+    return RegType::kNone;
+}
+
+ASMJIT_FAVOR_SIZE Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept {
+  cc.setArch(environment.arch());
+
+  cc.setSaveRestoreRegSize(RegGroup::kGp, 8);
+  cc.setSaveRestoreRegSize(RegGroup::kVec, 8);
+  cc.setSaveRestoreAlignment(RegGroup::kGp, 16);
+  cc.setSaveRestoreAlignment(RegGroup::kVec, 16);
+  cc.setSaveRestoreAlignment(RegGroup::kExtraVirt2, 1);
+  cc.setSaveRestoreAlignment(RegGroup::kExtraVirt3, 1);
+  cc.setPassedOrder(RegGroup::kGp, 0, 1, 2, 3, 4, 5, 6, 7);
+  cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3, 4, 5, 6, 7);
+  cc.setNaturalStackAlignment(16);
+
+  if (shouldThreatAsCDecl(ccId)) {
+    // ARM doesn't have that many calling conventions as we can find in X86 world, treat most conventions as __cdecl.
+    cc.setId(CallConvId::kCDecl);
+    cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(Gp::kIdOs, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30));
+    cc.setPreservedRegs(RegGroup::kVec, Support::bitMask(8, 9, 10, 11, 12, 13, 14, 15));
+  }
+  else {
+    cc.setId(ccId);
+    cc.setSaveRestoreRegSize(RegGroup::kVec, 16);
+    cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30));
+    cc.setPreservedRegs(RegGroup::kVec, Support::bitMask(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31));
+  }
+
+  return kErrorOk;
+}
+
+ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept {
+  DebugUtils::unused(signature);
+
+  const CallConv& cc = func.callConv();
+  uint32_t stackOffset = 0;
+
+  uint32_t i;
+  uint32_t argCount = func.argCount();
+
+  if (func.hasRet()) {
+    for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
+      TypeId typeId = func._rets[valueIndex].typeId();
+
+      // Terminate at the first void type (end of the pack).
+      if (typeId == TypeId::kVoid)
+        break;
+
+      switch (typeId) {
+        case TypeId::kInt8:
+        case TypeId::kInt16:
+        case TypeId::kInt32: {
+          func._rets[valueIndex].initReg(RegType::kARM_GpW, valueIndex, TypeId::kInt32);
+          break;
+        }
+
+        case TypeId::kUInt8:
+        case TypeId::kUInt16:
+        case TypeId::kUInt32: {
+          func._rets[valueIndex].initReg(RegType::kARM_GpW, valueIndex, TypeId::kUInt32);
+          break;
+        }
+
+        case TypeId::kInt64:
+        case TypeId::kUInt64: {
+          func._rets[valueIndex].initReg(RegType::kARM_GpX, valueIndex, typeId);
+          break;
+        }
+
+        default: {
+          RegType regType = regTypeFromFpOrVecTypeId(typeId);
+          if (regType == RegType::kNone)
+            return DebugUtils::errored(kErrorInvalidRegType);
+
+          func._rets[valueIndex].initReg(regType, valueIndex, typeId);
+          break;
+        }
+      }
+    }
+  }
+
+  switch (cc.strategy()) {
+    case CallConvStrategy::kDefault: {
+      uint32_t gpzPos = 0;
+      uint32_t vecPos = 0;
+
+      for (i = 0; i < argCount; i++) {
+        FuncValue& arg = func._args[i][0];
+        TypeId typeId = arg.typeId();
+
+        if (TypeUtils::isInt(typeId)) {
+          uint32_t regId = BaseReg::kIdBad;
+
+          if (gpzPos < CallConv::kMaxRegArgsPerGroup)
+            regId = cc._passedOrder[RegGroup::kGp].id[gpzPos];
+
+          if (regId != BaseReg::kIdBad) {
+            RegType regType = typeId <= TypeId::kUInt32 ? RegType::kARM_GpW : RegType::kARM_GpX;
+            arg.assignRegData(regType, regId);
+            func.addUsedRegs(RegGroup::kGp, Support::bitMask(regId));
+            gpzPos++;
+          }
+          else {
+            uint32_t size = Support::max<uint32_t>(TypeUtils::sizeOf(typeId), registerSize);
+            arg.assignStackOffset(int32_t(stackOffset));
+            stackOffset += size;
+          }
+          continue;
+        }
+
+        if (TypeUtils::isFloat(typeId) || TypeUtils::isVec(typeId)) {
+          uint32_t regId = BaseReg::kIdBad;
+
+          if (vecPos < CallConv::kMaxRegArgsPerGroup)
+            regId = cc._passedOrder[RegGroup::kVec].id[vecPos];
+
+          if (regId != BaseReg::kIdBad) {
+            RegType regType = regTypeFromFpOrVecTypeId(typeId);
+            if (regType == RegType::kNone)
+              return DebugUtils::errored(kErrorInvalidRegType);
+
+            arg.initTypeId(typeId);
+            arg.assignRegData(regType, regId);
+            func.addUsedRegs(RegGroup::kVec, Support::bitMask(regId));
+            vecPos++;
+          }
+          else {
+            uint32_t size = TypeUtils::sizeOf(typeId);
+            arg.assignStackOffset(int32_t(stackOffset));
+            stackOffset += size;
+          }
+          continue;
+        }
+      }
+      break;
+    }
+
+    default:
+      return DebugUtils::errored(kErrorInvalidState);
+  }
+
+  func._argStackSize = stackOffset;
+  return kErrorOk;
+}
+
+} // {FuncInternal}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64

libraries/asmjit/asmjit/arm/a64func_p.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64FUNC_P_H_INCLUDED
+#define ASMJIT_ARM_A64FUNC_P_H_INCLUDED
+
+#include "../core/func.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \cond INTERNAL
+//! \addtogroup asmjit_a64
+//! \{
+
+//! AArch64-specific function API (calling conventions and other utilities).
+namespace FuncInternal {
+
+//! Initialize `CallConv` structure (AArch64 specific).
+Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept;
+
+//! Initialize `FuncDetail` (AArch64 specific).
+Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept;
+
+} // {FuncInternal}
+
+//! \}
+//! \endcond
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_A64FUNC_P_H_INCLUDED