JIT compilation on ARM processors (#3517)

* Upgraded to new version of asmjit

* JIT compilation for ARM

* Updated CMake script

CMakeLists.txt

@@ -247,8 +247,8 @@ IF(X86)
 ELSE()
     SET_SOURCE_FILES_PROPERTIES(${CMAKE_SOURCE_DIR}/libraries/sfmt/src/SFMT.cpp PROPERTIES COMPILE_FLAGS "-UHAVE_SSE2")
 ENDIF()
-IF(X86 AND NOT (WIN32 AND OPENMM_BUILD_STATIC_LIB))
-    FILE(GLOB src_files ${CMAKE_CURRENT_SOURCE_DIR}/libraries/asmjit/*/*.cpp)
+IF((X86 OR ARM) AND NOT (WIN32 AND OPENMM_BUILD_STATIC_LIB))
+    FILE(GLOB src_files ${CMAKE_CURRENT_SOURCE_DIR}/libraries/asmjit/asmjit/*/*.cpp)
     FILE(GLOB incl_files ${CMAKE_CURRENT_SOURCE_DIR}/libraries/asmjit/*.h)
     SET(SOURCE_FILES ${SOURCE_FILES} ${src_files})
     SET(SOURCE_INCLUDE_FILES ${SOURCE_INCLUDE_FILES} ${incl_files})

libraries/asmjit/arm.h

-// [AsmJit]
-// Complete x86/x64 JIT and Remote Assembler for C++.
-//
-// [License]
-// Zlib - See LICENSE.md file in the package.
-
-// [Guard]
-#ifndef _ASMJIT_ARM_H
-#define _ASMJIT_ARM_H
-
-// [Dependencies]
-#include "./base.h"
-
-#include "./arm/armassembler.h"
-#include "./arm/armbuilder.h"
-#include "./arm/armcompiler.h"
-#include "./arm/arminst.h"
-#include "./arm/armoperand.h"
-
-// [Guard]
-#endif // _ASMJIT_ARM_H

libraries/asmjit/asmjit.h

-// [AsmJit]
-// Complete x86/x64 JIT and Remote Assembler for C++.
-//
-// [License]
-// Zlib - See LICENSE.md file in the package.
-
-// [Guard]
-#ifndef _ASMJIT_ASMJIT_H
-#define _ASMJIT_ASMJIT_H
-
-// ============================================================================
-// [asmjit_mainpage]
-// ============================================================================
-
-//! \mainpage
-//!
-//! AsmJit - Complete x86/x64 JIT and Remote Assembler for C++.
-//!
-//! Introduction provided by the project page at https://github.com/asmjit/asmjit.
-
-//! \defgroup asmjit_base AsmJit Base API (architecture independent)
-//!
-//! \brief Backend Neutral API.
-
-//! \defgroup asmjit_x86 AsmJit X86/X64 API
-//!
-//! \brief X86/X64 Backend API.
-
-//! \defgroup asmjit_arm AsmJit ARM32/ARM64 API
-//!
-//! \brief ARM32/ARM64 Backend API.
-
-// [Dependencies]
-#include "./base.h"
-
-// [X86/X64]
-#if defined(ASMJIT_BUILD_X86)
-#include "./x86.h"
-#endif // ASMJIT_BUILD_X86
-
-// [ARM32/ARM64]
-#if defined(ASMJIT_BUILD_ARM)
-#include "./arm.h"
-#endif // ASMJIT_BUILD_ARM
-
-// [Guard]
-#endif // _ASMJIT_ASMJIT_H

libraries/asmjit/LICENSE.md → libraries/asmjit/asmjit/LICENSE.md

-Copyright (c) 2008-2017, Petr Kobalicek
+Copyright (c) 2008-2020 The AsmJit Authors
 
 This software is provided 'as-is', without any express or implied
 warranty. In no event will the authors be held liable for any damages

libraries/asmjit/asmjit/a64.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_A64_H_INCLUDED
+#define ASMJIT_A64_H_INCLUDED
+
+//! \addtogroup asmjit_a64
+//!
+//! ### Emitters
+//!
+//!   - \ref a64::Assembler - AArch64 assembler (must read, provides examples).
+//!   - \ref a64::Builder - AArch64 builder.
+//!   - \ref a64::Compiler - AArch64 compiler.
+//!   - \ref a64::Emitter - AArch64 emitter (abstract).
+//!
+//! ### Supported Instructions
+//!
+//!   - Emitters:
+//!     - \ref a64::EmitterExplicitT - Provides all instructions that use explicit
+//!       operands, provides also utility functions. The member functions provided
+//!       are part of all ARM/AArch64 emitters.
+//!
+//!   - Instruction representation:
+//!     - \ref a64::Inst::Id - instruction identifiers.
+//!
+//! ### Register Operands
+//!
+//!   - \ref arm::Reg - Base class for any AArch32/AArch64 register.
+//!     - \ref arm::Gp - General purpose register:
+//!       - \ref arm::GpW - 32-bit register.
+//!       - \ref arm::GpX - 64-bit register.
+//!     - \ref arm::Vec - Vector (SIMD) register:
+//!       - \ref arm::VecB - 8-bit SIMD register (AArch64 only).
+//!       - \ref arm::VecH - 16-bit SIMD register (AArch64 only).
+//!       - \ref arm::VecS - 32-bit SIMD register.
+//!       - \ref arm::VecD - 64-bit SIMD register.
+//!       - \ref arm::VecV - 128-bit SIMD register.
+//!
+//! ### Memory Operands
+//!
+//!   - \ref arm::Mem - AArch32/AArch64 memory operand that provides support for all ARM addressing features
+//!     including base, index, pre/post increment, and ARM-specific shift addressing and index extending.
+//!
+//! ### Other
+//!
+//!   - \ref arm::Shift - Shift operation and value.
+//!   - \ref a64::Utils - Utilities that can help during code generation for AArch64.
+
+#include "./arm.h"
+#include "./arm/a64assembler.h"
+#include "./arm/a64builder.h"
+#include "./arm/a64compiler.h"
+#include "./arm/a64emitter.h"
+#include "./arm/a64globals.h"
+#include "./arm/a64instdb.h"
+#include "./arm/a64operand.h"
+#include "./arm/a64utils.h"
+
+#endif // ASMJIT_A64_H_INCLUDED
+

libraries/asmjit/asmjit/arm.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_H_INCLUDED
+#define ASMJIT_ARM_H_INCLUDED
+
+//! \addtogroup asmjit_arm
+//!
+//! ### Namespaces
+//!
+//!   - \ref arm - arm namespace provides common functionality for both AArch32 and AArch64 backends.
+//!   - \ref a64 - a64 namespace provides support for AArch64 architecture. In addition it includes
+//!     \ref arm namespace, so you can only use a single namespace when targeting AArch64 architecture.
+//!
+//! ### Emitters
+//!
+//!   - AArch64
+//!     - \ref a64::Assembler - AArch64 assembler (must read, provides examples).
+//!     - \ref a64::Builder - AArch64 builder.
+//!     - \ref a64::Compiler - AArch64 compiler.
+//!     - \ref a64::Emitter - AArch64 emitter (abstract).
+//!
+//! ### Supported Instructions
+//!
+//!   - AArch64:
+//!     - Emitters:
+//!       - \ref a64::EmitterExplicitT - Provides all instructions that use explicit operands, provides also
+//!         utility functions. The member functions provided are part of all AArch64 emitters.
+//!     - Instruction representation:
+//!       - \ref a64::Inst::Id - instruction identifiers.
+//!
+//! ### Register Operands
+//!
+//!   - \ref arm::Reg - Base class for any AArch32/AArch64 register.
+//!     - \ref arm::Gp - General purpose register:
+//!       - \ref arm::GpW - 32-bit register.
+//!       - \ref arm::GpX - 64-bit register.
+//!     - \ref arm::Vec - Vector (SIMD) register:
+//!       - \ref arm::VecB - 8-bit SIMD register (AArch64 only).
+//!       - \ref arm::VecH - 16-bit SIMD register (AArch64 only).
+//!       - \ref arm::VecS - 32-bit SIMD register.
+//!       - \ref arm::VecD - 64-bit SIMD register.
+//!       - \ref arm::VecV - 128-bit SIMD register.
+//!
+//! ### Memory Operands
+//!
+//!   - \ref arm::Mem - AArch32/AArch64 memory operand that provides support for all ARM addressing features
+//!     including base, index, pre/post increment, and ARM-specific shift addressing and index extending.
+//!
+//! ### Other
+//!
+//!   - \ref arm::Shift - Shift operation and value (both AArch32 and AArch64).
+//!   - \ref arm::DataType - Data type that is part of an instruction in AArch32 mode.
+//!   - \ref a64::Utils - Utilities that can help during code generation for AArch64.
+
+#include "./core.h"
+#include "./arm/armglobals.h"
+#include "./arm/armoperand.h"
+
+#endif // ASMJIT_ARM_H_INCLUDED

libraries/asmjit/asmjit/arm/a64archtraits_p.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64ARCHTRAITS_P_H_INCLUDED
+#define ASMJIT_ARM_A64ARCHTRAITS_P_H_INCLUDED
+
+#include "../core/archtraits.h"
+#include "../core/misc_p.h"
+#include "../core/type.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \cond INTERNAL
+//! \addtogroup asmjit_a64
+//! \{
+
+static const constexpr ArchTraits a64ArchTraits = {
+  // SP/FP/LR/PC.
+  Gp::kIdSp, Gp::kIdFp, Gp::kIdLr, 0xFF,
+
+  // Reserved.
+  { 0, 0, 0 },
+
+  // HW stack alignment (AArch64 requires stack aligned to 64 bytes).
+  16,
+
+  // Min/max stack offset - byte addressing is the worst, VecQ addressing the best.
+  4095, 65520,
+
+  // Instruction hints [Gp, Vec, ExtraVirt2, ExtraVirt3].
+  {{
+    InstHints::kPushPop,
+    InstHints::kPushPop,
+    InstHints::kNoHints,
+    InstHints::kNoHints
+  }},
+
+  // RegInfo.
+  #define V(index) OperandSignature{arm::RegTraits<RegType(index)>::kSignature}
+  {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }},
+  #undef V
+
+  // RegTypeToTypeId.
+  #define V(index) TypeId(arm::RegTraits<RegType(index)>::kTypeId)
+  {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }},
+  #undef V
+
+  // TypeIdToRegType.
+  #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8)    ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8)   ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16)   ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16)  ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32)   ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32)  ? RegType::kARM_GpW   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt64)   ? RegType::kARM_GpX   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt64)  ? RegType::kARM_GpX   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr)  ? RegType::kARM_GpX   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kARM_GpX   : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kARM_VecS  : \
+                    index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kARM_VecD  : RegType::kNone)
+  {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }},
+  #undef V
+
+  // Word names of 8-bit, 16-bit, 32-bit, and 64-bit quantities.
+  {
+    ArchTypeNameId::kByte,
+    ArchTypeNameId::kHWord,
+    ArchTypeNameId::kWord,
+    ArchTypeNameId::kXWord
+  }
+};
+
+//! \}
+//! \endcond
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_A64ARCHTRAITS_P_H_INCLUDED

libraries/asmjit/asmjit/arm/a64assembler.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64ASSEMBLER_H_INCLUDED
+#define ASMJIT_ARM_A64ASSEMBLER_H_INCLUDED
+
+#include "../core/assembler.h"
+#include "../arm/a64emitter.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \addtogroup asmjit_a64
+//! \{
+
+//! AArch64 assembler implementation.
+class ASMJIT_VIRTAPI Assembler
+  : public BaseAssembler,
+    public EmitterExplicitT<Assembler> {
+
+public:
+  typedef BaseAssembler Base;
+
+  //! \name Construction / Destruction
+  //! \{
+
+  ASMJIT_API Assembler(CodeHolder* code = nullptr) noexcept;
+  ASMJIT_API virtual ~Assembler() noexcept;
+
+  //! \}
+
+  //! \name Accessors
+  //! \{
+
+  //! Gets whether the current ARM mode is THUMB (alternative to 32-bit ARM encoding).
+  inline bool isInThumbMode() const noexcept { return _environment.isArchThumb(); }
+
+  //! Gets the current code alignment of the current mode (ARM vs THUMB).
+  inline uint32_t codeAlignment() const noexcept { return isInThumbMode() ? 2 : 4; }
+
+  //! \}
+
+  //! \name Emit
+  //! \{
+
+  ASMJIT_API Error _emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) override;
+
+  //! \}
+
+  //! \name Align
+  //! \{
+
+  ASMJIT_API Error align(AlignMode alignMode, uint32_t alignment) override;
+
+  //! \}
+
+  //! \name Events
+  //! \{
+
+  ASMJIT_API Error onAttach(CodeHolder* code) noexcept override;
+  ASMJIT_API Error onDetach(CodeHolder* code) noexcept override;
+
+  //! \}
+};
+
+//! \}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_A64ASSEMBLER_H_INCLUDED

libraries/asmjit/asmjit/arm/a64builder.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64) && !defined(ASMJIT_NO_BUILDER)
+
+#include "../arm/a64assembler.h"
+#include "../arm/a64builder.h"
+#include "../arm/a64emithelper_p.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::Builder - Construction & Destruction
+// =========================================
+
+Builder::Builder(CodeHolder* code) noexcept : BaseBuilder() {
+  _archMask = uint64_t(1) << uint32_t(Arch::kAArch64);
+  assignEmitterFuncs(this);
+
+  if (code)
+    code->attach(this);
+}
+Builder::~Builder() noexcept {}
+
+// a64::Builder - Events
+// =====================
+
+Error Builder::onAttach(CodeHolder* code) noexcept {
+  return Base::onAttach(code);
+}
+
+Error Builder::onDetach(CodeHolder* code) noexcept {
+  return Base::onDetach(code);
+}
+
+// a64::Builder - Finalize
+// =======================
+
+Error Builder::finalize() {
+  ASMJIT_PROPAGATE(runPasses());
+  Assembler a(_code);
+  a.addEncodingOptions(encodingOptions());
+  a.addDiagnosticOptions(diagnosticOptions());
+  return serializeTo(&a);
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64 && !ASMJIT_NO_BUILDER

libraries/asmjit/asmjit/arm/a64builder.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64BUILDER_H_INCLUDED
+#define ASMJIT_ARM_A64BUILDER_H_INCLUDED
+
+#include "../core/api-config.h"
+#ifndef ASMJIT_NO_BUILDER
+
+#include "../core/builder.h"
+#include "../arm/a64emitter.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \addtogroup asmjit_a64
+//! \{
+
+//! AArch64 builder implementation.
+class ASMJIT_VIRTAPI Builder
+  : public BaseBuilder,
+    public EmitterExplicitT<Builder> {
+public:
+  ASMJIT_NONCOPYABLE(Builder)
+  typedef BaseBuilder Base;
+
+  //! \name Construction & Destruction
+  //! \{
+
+  ASMJIT_API explicit Builder(CodeHolder* code = nullptr) noexcept;
+  ASMJIT_API virtual ~Builder() noexcept;
+
+  //! \}
+
+  //! \name Events
+  //! \{
+
+  ASMJIT_API Error onAttach(CodeHolder* code) noexcept override;
+  ASMJIT_API Error onDetach(CodeHolder* code) noexcept override;
+
+  //! \}
+
+  //! \name Finalize
+  //! \{
+
+  ASMJIT_API Error finalize() override;
+
+  //! \}
+};
+
+//! \}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_BUILDER
+#endif // ASMJIT_ARM_A64BUILDER_H_INCLUDED

libraries/asmjit/asmjit/arm/a64compiler.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64) && !defined(ASMJIT_NO_COMPILER)
+
+#include "../arm/a64assembler.h"
+#include "../arm/a64compiler.h"
+#include "../arm/a64emithelper_p.h"
+#include "../arm/a64rapass_p.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::Compiler - Construction & Destruction
+// ==========================================
+
+Compiler::Compiler(CodeHolder* code) noexcept : BaseCompiler() {
+  _archMask = uint64_t(1) << uint32_t(Arch::kAArch64);
+  assignEmitterFuncs(this);
+
+  if (code)
+    code->attach(this);
+}
+Compiler::~Compiler() noexcept {}
+
+// a64::Compiler - Events
+// ======================
+
+Error Compiler::onAttach(CodeHolder* code) noexcept {
+  ASMJIT_PROPAGATE(Base::onAttach(code));
+  Error err = addPassT<ARMRAPass>();
+
+  if (ASMJIT_UNLIKELY(err)) {
+    onDetach(code);
+    return err;
+  }
+
+  return kErrorOk;
+}
+
+Error Compiler::onDetach(CodeHolder* code) noexcept {
+  return Base::onDetach(code);
+}
+
+// a64::Compiler - Finalize
+// ========================
+
+Error Compiler::finalize() {
+  ASMJIT_PROPAGATE(runPasses());
+  Assembler a(_code);
+  a.addEncodingOptions(encodingOptions());
+  a.addDiagnosticOptions(diagnosticOptions());
+  return serializeTo(&a);
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64 && !ASMJIT_NO_COMPILER

libraries/asmjit/asmjit/arm/a64compiler.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_ARMCOMPILER_H_INCLUDED
+#define ASMJIT_ARM_ARMCOMPILER_H_INCLUDED
+
+#include "../core/api-config.h"
+#ifndef ASMJIT_NO_COMPILER
+
+#include "../core/compiler.h"
+#include "../core/type.h"
+#include "../arm/a64emitter.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \addtogroup asmjit_a64
+//! \{
+
+//! AArch64 compiler implementation.
+class ASMJIT_VIRTAPI Compiler
+  : public BaseCompiler,
+    public EmitterExplicitT<Compiler> {
+public:
+  ASMJIT_NONCOPYABLE(Compiler)
+  typedef BaseCompiler Base;
+
+  //! \name Construction & Destruction
+  //! \{
+
+  ASMJIT_API explicit Compiler(CodeHolder* code = nullptr) noexcept;
+  ASMJIT_API virtual ~Compiler() noexcept;
+
+  //! \}
+
+  //! \name Virtual Registers
+  //! \{
+
+  //! \cond INTERNAL
+  template<typename RegT, typename Type>
+  inline RegT _newRegInternal(const Type& type) {
+    RegT reg(Globals::NoInit);
+    _newReg(&reg, type, nullptr);
+    return reg;
+  }
+
+  template<typename RegT, typename Type, typename... Args>
+  inline RegT _newRegInternal(const Type& type, const char* s, Args&&... args) {
+#ifndef ASMJIT_NO_LOGGING
+    RegT reg(Globals::NoInit);
+    if (sizeof...(Args) == 0)
+      _newReg(&reg, type, s);
+    else
+      _newRegFmt(&reg, type, s, std::forward<Args>(args)...);
+    return reg;
+#else
+    DebugUtils::unused(std::forward<Args>(args)...);
+    RegT reg(Globals::NoInit);
+    _newReg(&reg, type, nullptr);
+    return reg;
+#endif
+  }
+  //! \endcond
+
+  template<typename RegT, typename... Args>
+  inline RegT newSimilarReg(const RegT& ref, Args&&... args) {
+    return _newRegInternal<RegT>(ref, std::forward<Args>(args)...);
+  }
+
+  template<typename... Args>
+  inline Reg newReg(TypeId typeId, Args&&... args) { return _newRegInternal<Reg>(typeId, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newGp(TypeId typeId, Args&&... args) { return _newRegInternal<Gp>(typeId, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Vec newVec(TypeId typeId, Args&&... args) { return _newRegInternal<Vec>(typeId, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newInt32(Args&&... args) { return _newRegInternal<Gp>(TypeId::kInt32, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newUInt32(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUInt32, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newInt64(Args&&... args) { return _newRegInternal<Gp>(TypeId::kInt64, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newUInt64(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUInt64, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newIntPtr(Args&&... args) { return _newRegInternal<Gp>(TypeId::kIntPtr, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newUIntPtr(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUIntPtr, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Gp newGpw(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUInt32, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newGpx(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUInt64, std::forward<Args>(args)...); }
+  template<typename... Args>
+  inline Gp newGpz(Args&&... args) { return _newRegInternal<Gp>(TypeId::kUIntPtr, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Vec newVecS(Args&&... args) { return _newRegInternal<Vec>(TypeId::kFloat32, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Vec newVecD(Args&&... args) { return _newRegInternal<Vec>(TypeId::kFloat64, std::forward<Args>(args)...); }
+
+  template<typename... Args>
+  inline Vec newVecQ(Args&&... args) { return _newRegInternal<Vec>(TypeId::kUInt8x16, std::forward<Args>(args)...); }
+
+  //! \}
+
+  //! \name Stack
+  //! \{
+
+  //! Creates a new memory chunk allocated on the current function's stack.
+  inline Mem newStack(uint32_t size, uint32_t alignment, const char* name = nullptr) {
+    Mem m(Globals::NoInit);
+    _newStack(&m, size, alignment, name);
+    return m;
+  }
+
+  //! \}
+
+  //! \name Constants
+  //! \{
+
+  //! Put data to a constant-pool and get a memory reference to it.
+  inline Mem newConst(ConstPoolScope scope, const void* data, size_t size) {
+    Mem m(Globals::NoInit);
+    _newConst(&m, scope, data, size);
+    return m;
+  }
+
+  //! Put a BYTE `val` to a constant-pool (8 bits).
+  inline Mem newByteConst(ConstPoolScope scope, uint8_t val) noexcept { return newConst(scope, &val, 1); }
+  //! Put a HWORD `val` to a constant-pool (16 bits).
+  inline Mem newHWordConst(ConstPoolScope scope, uint16_t val) noexcept { return newConst(scope, &val, 2); }
+  //! Put a WORD `val` to a constant-pool (32 bits).
+  inline Mem newWordConst(ConstPoolScope scope, uint32_t val) noexcept { return newConst(scope, &val, 4); }
+  //! Put a DWORD `val` to a constant-pool (64 bits).
+  inline Mem newDWordConst(ConstPoolScope scope, uint64_t val) noexcept { return newConst(scope, &val, 8); }
+
+  //! Put a WORD `val` to a constant-pool.
+  inline Mem newInt16Const(ConstPoolScope scope, int16_t val) noexcept { return newConst(scope, &val, 2); }
+  //! Put a WORD `val` to a constant-pool.
+  inline Mem newUInt16Const(ConstPoolScope scope, uint16_t val) noexcept { return newConst(scope, &val, 2); }
+  //! Put a DWORD `val` to a constant-pool.
+  inline Mem newInt32Const(ConstPoolScope scope, int32_t val) noexcept { return newConst(scope, &val, 4); }
+  //! Put a DWORD `val` to a constant-pool.
+  inline Mem newUInt32Const(ConstPoolScope scope, uint32_t val) noexcept { return newConst(scope, &val, 4); }
+  //! Put a QWORD `val` to a constant-pool.
+  inline Mem newInt64Const(ConstPoolScope scope, int64_t val) noexcept { return newConst(scope, &val, 8); }
+  //! Put a QWORD `val` to a constant-pool.
+  inline Mem newUInt64Const(ConstPoolScope scope, uint64_t val) noexcept { return newConst(scope, &val, 8); }
+
+  //! Put a SP-FP `val` to a constant-pool.
+  inline Mem newFloatConst(ConstPoolScope scope, float val) noexcept { return newConst(scope, &val, 4); }
+  //! Put a DP-FP `val` to a constant-pool.
+  inline Mem newDoubleConst(ConstPoolScope scope, double val) noexcept { return newConst(scope, &val, 8); }
+
+  //! \}
+
+  //! \name Instruction Options
+  //! \{
+
+  //! Force the compiler to not follow the conditional or unconditional jump.
+  inline Compiler& unfollow() noexcept { _instOptions |= InstOptions::kUnfollow; return *this; }
+
+  //! \}
+
+  //! \name Function Call & Ret Intrinsics
+  //! \{
+
+  //! Invoke a function call without `target` type enforcement.
+  inline Error invoke_(InvokeNode** out, const Operand_& target, const FuncSignature& signature) {
+    return addInvokeNode(out, Inst::kIdBlr, target, signature);
+  }
+
+  //! Invoke a function call of the given `target` and `signature` and store the added node to `out`.
+  //!
+  //! Creates a new \ref InvokeNode, initializes all the necessary members to match the given function `signature`,
+  //! adds the node to the compiler, and stores its pointer to `out`. The operation is atomic, if anything fails
+  //! nullptr is stored in `out` and error code is returned.
+  inline Error invoke(InvokeNode** out, const Gp& target, const FuncSignature& signature) { return invoke_(out, target, signature); }
+  //! \overload
+  inline Error invoke(InvokeNode** out, const Mem& target, const FuncSignature& signature) { return invoke_(out, target, signature); }
+  //! \overload
+  inline Error invoke(InvokeNode** out, const Label& target, const FuncSignature& signature) { return invoke_(out, target, signature); }
+  //! \overload
+  inline Error invoke(InvokeNode** out, const Imm& target, const FuncSignature& signature) { return invoke_(out, target, signature); }
+  //! \overload
+  inline Error invoke(InvokeNode** out, uint64_t target, const FuncSignature& signature) { return invoke_(out, Imm(int64_t(target)), signature); }
+
+  //! Return.
+  inline Error ret() { return addRet(Operand(), Operand()); }
+  //! \overload
+  inline Error ret(const BaseReg& o0) { return addRet(o0, Operand()); }
+  //! \overload
+  inline Error ret(const BaseReg& o0, const BaseReg& o1) { return addRet(o0, o1); }
+
+  //! \}
+
+  //! \name Jump Tables Support
+  //! \{
+
+  using EmitterExplicitT<Compiler>::br;
+
+  //! Adds a jump to the given `target` with the provided jump `annotation`.
+  inline Error br(const BaseReg& target, JumpAnnotation* annotation) { return emitAnnotatedJump(Inst::kIdBr, target, annotation); }
+
+  //! \}
+
+  //! \name Events
+  //! \{
+
+  ASMJIT_API Error onAttach(CodeHolder* code) noexcept override;
+  ASMJIT_API Error onDetach(CodeHolder* code) noexcept override;
+
+  //! \}
+
+  //! \name Finalize
+  //! \{
+
+  ASMJIT_API Error finalize() override;
+
+  //! \}
+};
+
+//! \}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_COMPILER
+#endif // ASMJIT_ARM_ARMCOMPILER_H_INCLUDED

libraries/asmjit/asmjit/arm/a64emithelper.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64)
+
+#include "../core/formatter.h"
+#include "../core/funcargscontext_p.h"
+#include "../core/string.h"
+#include "../core/support.h"
+#include "../core/type.h"
+#include "../arm/a64emithelper_p.h"
+#include "../arm/a64formatter_p.h"
+#include "../arm/a64instapi_p.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::EmitHelper - Emit Operations
+// =================================
+
+ASMJIT_FAVOR_SIZE Error EmitHelper::emitRegMove(
+  const Operand_& dst_,
+  const Operand_& src_, TypeId typeId, const char* comment) {
+
+  Emitter* emitter = _emitter->as<Emitter>();
+
+  // Invalid or abstract TypeIds are not allowed.
+  ASMJIT_ASSERT(TypeUtils::isValid(typeId) && !TypeUtils::isAbstract(typeId));
+
+  emitter->setInlineComment(comment);
+
+  if (dst_.isReg() && src_.isMem()) {
+    Reg dst(dst_.as<Reg>());
+    Mem src(src_.as<Mem>());
+
+    switch (typeId) {
+      case TypeId::kInt8:
+      case TypeId::kUInt8:
+        return emitter->ldrb(dst.as<Gp>(), src);
+
+      case TypeId::kInt16:
+      case TypeId::kUInt16:
+        return emitter->ldrh(dst.as<Gp>(), src);
+
+      case TypeId::kInt32:
+      case TypeId::kUInt32:
+        return emitter->ldr(dst.as<Gp>().w(), src);
+
+      case TypeId::kInt64:
+      case TypeId::kUInt64:
+        return emitter->ldr(dst.as<Gp>().x(), src);
+
+      default: {
+        if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId))
+          return emitter->ldr(dst.as<Vec>().s(), src);
+
+        if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId))
+          return emitter->ldr(dst.as<Vec>().d(), src);
+
+        if (TypeUtils::isVec128(typeId))
+          return emitter->ldr(dst.as<Vec>().q(), src);
+
+        break;
+      }
+    }
+  }
+
+  if (dst_.isMem() && src_.isReg()) {
+    Mem dst(dst_.as<Mem>());
+    Reg src(src_.as<Reg>());
+
+    switch (typeId) {
+      case TypeId::kInt8:
+      case TypeId::kUInt8:
+        return emitter->strb(src.as<Gp>(), dst);
+
+      case TypeId::kInt16:
+      case TypeId::kUInt16:
+        return emitter->strh(src.as<Gp>(), dst);
+
+      case TypeId::kInt32:
+      case TypeId::kUInt32:
+        return emitter->str(src.as<Gp>().w(), dst);
+
+      case TypeId::kInt64:
+      case TypeId::kUInt64:
+        return emitter->str(src.as<Gp>().x(), dst);
+
+      default: {
+        if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId))
+          return emitter->str(src.as<Vec>().s(), dst);
+
+        if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId))
+          return emitter->str(src.as<Vec>().d(), dst);
+
+        if (TypeUtils::isVec128(typeId))
+          return emitter->str(src.as<Vec>().q(), dst);
+
+        break;
+      }
+    }
+  }
+
+  if (dst_.isReg() && src_.isReg()) {
+    Reg dst(dst_.as<Reg>());
+    Reg src(src_.as<Reg>());
+
+    switch (typeId) {
+      case TypeId::kInt8:
+      case TypeId::kUInt8:
+      case TypeId::kInt16:
+      case TypeId::kUInt16:
+      case TypeId::kInt32:
+      case TypeId::kUInt32:
+      case TypeId::kInt64:
+      case TypeId::kUInt64:
+        return emitter->mov(src.as<Gp>().x(), dst.as<Gp>().x());
+
+      default: {
+        if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId))
+          return emitter->fmov(dst.as<Vec>().s(), src.as<Vec>().s());
+
+        if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId))
+          return emitter->mov(dst.as<Vec>().b8(), src.as<Vec>().b8());
+
+        if (TypeUtils::isVec128(typeId))
+          return emitter->mov(dst.as<Vec>().b16(), src.as<Vec>().b16());
+
+        break;
+      }
+    }
+  }
+
+  emitter->setInlineComment(nullptr);
+  return DebugUtils::errored(kErrorInvalidState);
+}
+
+Error EmitHelper::emitRegSwap(
+  const BaseReg& a,
+  const BaseReg& b, const char* comment) {
+
+  DebugUtils::unused(a, b, comment);
+  return DebugUtils::errored(kErrorInvalidState);
+}
+
+// TODO: [ARM] EmitArgMove is unfinished.
+Error EmitHelper::emitArgMove(
+  const BaseReg& dst_, TypeId dstTypeId,
+  const Operand_& src_, TypeId srcTypeId, const char* comment) {
+
+  // Deduce optional `dstTypeId`, which may be `TypeId::kVoid` in some cases.
+  if (dstTypeId == TypeId::kVoid) {
+    const ArchTraits& archTraits = ArchTraits::byArch(_emitter->arch());
+    dstTypeId = archTraits.regTypeToTypeId(dst_.type());
+  }
+
+  // Invalid or abstract TypeIds are not allowed.
+  ASMJIT_ASSERT(TypeUtils::isValid(dstTypeId) && !TypeUtils::isAbstract(dstTypeId));
+  ASMJIT_ASSERT(TypeUtils::isValid(srcTypeId) && !TypeUtils::isAbstract(srcTypeId));
+
+  Reg dst(dst_.as<Reg>());
+  Operand src(src_);
+
+  uint32_t dstSize = TypeUtils::sizeOf(dstTypeId);
+  uint32_t srcSize = TypeUtils::sizeOf(srcTypeId);
+
+  if (TypeUtils::isInt(dstTypeId)) {
+    if (TypeUtils::isInt(srcTypeId)) {
+      uint32_t x = dstSize == 8;
+
+      dst.setSignature(OperandSignature{x ? uint32_t(GpX::kSignature) : uint32_t(GpW::kSignature)});
+      _emitter->setInlineComment(comment);
+
+      if (src.isReg()) {
+        src.setSignature(dst.signature());
+        return _emitter->emit(Inst::kIdMov, dst, src);
+      }
+      else if (src.isMem()) {
+        InstId instId = Inst::kIdNone;
+          switch (srcTypeId) {
+          case TypeId::kInt8: instId = Inst::kIdLdrsb; break;
+          case TypeId::kUInt8: instId = Inst::kIdLdrb; break;
+          case TypeId::kInt16: instId = Inst::kIdLdrsh; break;
+          case TypeId::kUInt16: instId = Inst::kIdLdrh; break;
+          case TypeId::kInt32: instId = x ? Inst::kIdLdrsw : Inst::kIdLdr; break;
+          case TypeId::kUInt32: instId = Inst::kIdLdr; x = 0; break;
+          case TypeId::kInt64: instId = Inst::kIdLdr; break;
+          case TypeId::kUInt64: instId = Inst::kIdLdr; break;
+          default:
+            return DebugUtils::errored(kErrorInvalidState);
+        }
+        return _emitter->emit(instId, dst, src);
+      }
+    }
+  }
+
+  if (TypeUtils::isFloat(dstTypeId) || TypeUtils::isVec(dstTypeId)) {
+    if (TypeUtils::isFloat(srcTypeId) || TypeUtils::isVec(srcTypeId)) {
+      switch (srcSize) {
+        case 2: dst.as<Vec>().setSignature(OperandSignature{VecH::kSignature}); break;
+        case 4: dst.as<Vec>().setSignature(OperandSignature{VecS::kSignature}); break;
+        case 8: dst.as<Vec>().setSignature(OperandSignature{VecD::kSignature}); break;
+        case 16: dst.as<Vec>().setSignature(OperandSignature{VecV::kSignature}); break;
+        default:
+          return DebugUtils::errored(kErrorInvalidState);
+      }
+
+      _emitter->setInlineComment(comment);
+
+      if (src.isReg()) {
+        InstId instId = srcSize <= 4 ? Inst::kIdFmov_v : Inst::kIdMov_v;
+        src.setSignature(dst.signature());
+        return _emitter->emit(instId, dst, src);
+      }
+      else if (src.isMem()) {
+        return _emitter->emit(Inst::kIdLdr_v, dst, src);
+      }
+    }
+  }
+
+  return DebugUtils::errored(kErrorInvalidState);
+}
+
+// a64::EmitHelper - Emit Prolog & Epilog
+// ======================================
+
+struct LoadStoreInstructions {
+  InstId singleInstId;
+  InstId pairInstId;
+};
+
+struct PrologEpilogInfo {
+  struct RegPair {
+    uint8_t ids[2];
+    uint16_t offset;
+  };
+
+  struct GroupData {
+    RegPair pairs[16];
+    uint32_t pairCount;
+  };
+
+  Support::Array<GroupData, 2> groups;
+  uint32_t sizeTotal;
+
+  Error init(const FuncFrame& frame) noexcept {
+    uint32_t offset = 0;
+
+    for (RegGroup group : Support::EnumValues<RegGroup, RegGroup::kGp, RegGroup::kVec>{}) {
+      GroupData& data = groups[group];
+
+      uint32_t n = 0;
+      uint32_t pairCount = 0;
+      RegPair* pairs = data.pairs;
+
+      uint32_t slotSize = frame.saveRestoreRegSize(group);
+      uint32_t savedRegs = frame.savedRegs(group);
+
+      if (group == RegGroup::kGp && frame.hasPreservedFP()) {
+        // Must be at the beginning of the push/pop sequence.
+        ASMJIT_ASSERT(pairCount == 0);
+
+        pairs[0].offset = uint16_t(offset);
+        pairs[0].ids[0] = Gp::kIdFp;
+        pairs[0].ids[1] = Gp::kIdLr;
+        offset += slotSize * 2;
+        pairCount++;
+
+        savedRegs &= ~Support::bitMask(Gp::kIdFp, Gp::kIdLr);
+      }
+
+      Support::BitWordIterator<uint32_t> it(savedRegs);
+      while (it.hasNext()) {
+        pairs[pairCount].ids[n] = uint8_t(it.next());
+
+        if (++n == 2) {
+          pairs[pairCount].offset = uint16_t(offset);
+          offset += slotSize * 2;
+
+          n = 0;
+          pairCount++;
+        }
+      }
+
+      if (n == 1) {
+        pairs[pairCount].ids[1] = uint8_t(BaseReg::kIdBad);
+        pairs[pairCount].offset = uint16_t(offset);
+        offset += slotSize * 2;
+        pairCount++;
+      }
+
+      data.pairCount = pairCount;
+    }
+
+    sizeTotal = offset;
+    return kErrorOk;
+  }
+};
+
+ASMJIT_FAVOR_SIZE Error EmitHelper::emitProlog(const FuncFrame& frame) {
+  Emitter* emitter = _emitter->as<Emitter>();
+
+  PrologEpilogInfo pei;
+  ASMJIT_PROPAGATE(pei.init(frame));
+
+  static const Support::Array<Reg, 2> groupRegs = {{ x0, d0 }};
+  static const Support::Array<LoadStoreInstructions, 2> groupInsts = {{
+    { Inst::kIdStr  , Inst::kIdStp   },
+    { Inst::kIdStr_v, Inst::kIdStp_v }
+  }};
+
+  uint32_t adjustInitialOffset = pei.sizeTotal;
+
+  for (RegGroup group : Support::EnumValues<RegGroup, RegGroup::kGp, RegGroup::kVec>{}) {
+    const PrologEpilogInfo::GroupData& data = pei.groups[group];
+    uint32_t pairCount = data.pairCount;
+
+    Reg regs[2] = { groupRegs[group], groupRegs[group] };
+    Mem mem = ptr(sp);
+
+    const LoadStoreInstructions& insts = groupInsts[group];
+    for (uint32_t i = 0; i < pairCount; i++) {
+      const PrologEpilogInfo::RegPair& pair = data.pairs[i];
+
+      regs[0].setId(pair.ids[0]);
+      regs[1].setId(pair.ids[1]);
+      mem.setOffsetLo32(pair.offset);
+
+      if (pair.offset == 0 && adjustInitialOffset) {
+        mem.setOffset(-int(adjustInitialOffset));
+        mem.makePreIndex();
+      }
+
+      if (pair.ids[1] == BaseReg::kIdBad)
+        ASMJIT_PROPAGATE(emitter->emit(insts.singleInstId, regs[0], mem));
+      else
+        ASMJIT_PROPAGATE(emitter->emit(insts.pairInstId, regs[0], regs[1], mem));
+
+      mem.resetToFixedOffset();
+
+      if (i == 0 && frame.hasPreservedFP()) {
+        ASMJIT_PROPAGATE(emitter->mov(x29, sp));
+      }
+    }
+  }
+
+  if (frame.hasStackAdjustment()) {
+    uint32_t adj = frame.stackAdjustment();
+    if (adj <= 0xFFFu) {
+      ASMJIT_PROPAGATE(emitter->sub(sp, sp, adj));
+    }
+    else if (adj <= 0xFFFFFFu)  {
+      // TODO: [ARM] Prolog - we must touch the pages otherwise it's undefined.
+      ASMJIT_PROPAGATE(emitter->sub(sp, sp, adj & 0x000FFFu));
+      ASMJIT_PROPAGATE(emitter->sub(sp, sp, adj & 0xFFF000u));
+    }
+    else {
+      return DebugUtils::errored(kErrorInvalidState);
+    }
+  }
+
+  return kErrorOk;
+}
+
+// TODO: [ARM] Emit epilog.
+ASMJIT_FAVOR_SIZE Error EmitHelper::emitEpilog(const FuncFrame& frame) {
+  Emitter* emitter = _emitter->as<Emitter>();
+
+  PrologEpilogInfo pei;
+  ASMJIT_PROPAGATE(pei.init(frame));
+
+  static const Support::Array<Reg, 2> groupRegs = {{ x0, d0 }};
+  static const Support::Array<LoadStoreInstructions, 2> groupInsts = {{
+    { Inst::kIdLdr  , Inst::kIdLdp   },
+    { Inst::kIdLdr_v, Inst::kIdLdp_v }
+  }};
+
+  uint32_t adjustInitialOffset = pei.sizeTotal;
+
+  if (frame.hasStackAdjustment()) {
+    uint32_t adj = frame.stackAdjustment();
+    if (adj <= 0xFFFu) {
+      ASMJIT_PROPAGATE(emitter->add(sp, sp, adj));
+    }
+    else if (adj <= 0xFFFFFFu)  {
+      ASMJIT_PROPAGATE(emitter->add(sp, sp, adj & 0x000FFFu));
+      ASMJIT_PROPAGATE(emitter->add(sp, sp, adj & 0xFFF000u));
+    }
+    else {
+      return DebugUtils::errored(kErrorInvalidState);
+    }
+  }
+
+  for (int g = 1; g >= 0; g--) {
+    RegGroup group = RegGroup(g);
+    const PrologEpilogInfo::GroupData& data = pei.groups[group];
+    uint32_t pairCount = data.pairCount;
+
+    Reg regs[2] = { groupRegs[group], groupRegs[group] };
+    Mem mem = ptr(sp);
+
+    const LoadStoreInstructions& insts = groupInsts[group];
+
+    for (int i = int(pairCount) - 1; i >= 0; i--) {
+      const PrologEpilogInfo::RegPair& pair = data.pairs[i];
+
+      regs[0].setId(pair.ids[0]);
+      regs[1].setId(pair.ids[1]);
+      mem.setOffsetLo32(pair.offset);
+
+      if (pair.offset == 0 && adjustInitialOffset) {
+        mem.setOffset(int(adjustInitialOffset));
+        mem.makePostIndex();
+      }
+
+      if (pair.ids[1] == BaseReg::kIdBad)
+        ASMJIT_PROPAGATE(emitter->emit(insts.singleInstId, regs[0], mem));
+      else
+        ASMJIT_PROPAGATE(emitter->emit(insts.pairInstId, regs[0], regs[1], mem));
+
+      mem.resetToFixedOffset();
+    }
+  }
+
+  ASMJIT_PROPAGATE(emitter->ret(x30));
+
+  return kErrorOk;
+}
+
+static Error ASMJIT_CDECL Emitter_emitProlog(BaseEmitter* emitter, const FuncFrame& frame) {
+  EmitHelper emitHelper(emitter);
+  return emitHelper.emitProlog(frame);
+}
+
+static Error ASMJIT_CDECL Emitter_emitEpilog(BaseEmitter* emitter, const FuncFrame& frame) {
+  EmitHelper emitHelper(emitter);
+  return emitHelper.emitEpilog(frame);
+}
+
+static Error ASMJIT_CDECL Emitter_emitArgsAssignment(BaseEmitter* emitter, const FuncFrame& frame, const FuncArgsAssignment& args) {
+  EmitHelper emitHelper(emitter);
+  return emitHelper.emitArgsAssignment(frame, args);
+}
+
+void assignEmitterFuncs(BaseEmitter* emitter) {
+  emitter->_funcs.emitProlog = Emitter_emitProlog;
+  emitter->_funcs.emitEpilog = Emitter_emitEpilog;
+  emitter->_funcs.emitArgsAssignment = Emitter_emitArgsAssignment;
+
+#ifndef ASMJIT_NO_LOGGING
+  emitter->_funcs.formatInstruction = FormatterInternal::formatInstruction;
+#endif
+
+#ifndef ASMJIT_NO_VALIDATION
+  emitter->_funcs.validate = InstInternal::validate;
+#endif
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64

libraries/asmjit/asmjit/arm/a64emithelper_p.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_ARMEMITHELPER_P_H_INCLUDED
+#define ASMJIT_ARM_ARMEMITHELPER_P_H_INCLUDED
+
+#include "../core/api-config.h"
+
+#include "../core/emithelper_p.h"
+#include "../core/func.h"
+#include "../arm/a64emitter.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \cond INTERNAL
+//! \addtogroup asmjit_a64
+//! \{
+
+class EmitHelper : public BaseEmitHelper {
+public:
+  inline explicit EmitHelper(BaseEmitter* emitter = nullptr) noexcept
+    : BaseEmitHelper(emitter) {}
+
+  Error emitRegMove(
+    const Operand_& dst_,
+    const Operand_& src_, TypeId typeId, const char* comment = nullptr) override;
+
+  Error emitRegSwap(
+    const BaseReg& a,
+    const BaseReg& b, const char* comment = nullptr) override;
+
+  Error emitArgMove(
+    const BaseReg& dst_, TypeId dstTypeId,
+    const Operand_& src_, TypeId srcTypeId, const char* comment = nullptr) override;
+
+  Error emitProlog(const FuncFrame& frame);
+  Error emitEpilog(const FuncFrame& frame);
+};
+
+void assignEmitterFuncs(BaseEmitter* emitter);
+
+//! \}
+//! \endcond
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_ARMEMITHELPER_P_H_INCLUDED

libraries/asmjit/asmjit/arm/a64formatter.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#ifndef ASMJIT_NO_LOGGING
+
+#include "../core/misc_p.h"
+#include "../core/support.h"
+#include "../arm/a64formatter_p.h"
+#include "../arm/a64instapi_p.h"
+#include "../arm/a64instdb_p.h"
+#include "../arm/a64operand.h"
+
+#ifndef ASMJIT_NO_COMPILER
+  #include "../core/compiler.h"
+#endif
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+// a64::FormatterInternal - Format Register
+// ========================================
+
+ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  RegType regType,
+  uint32_t rId,
+  uint32_t elementType,
+  uint32_t elementIndex) noexcept {
+
+  DebugUtils::unused(flags);
+  DebugUtils::unused(arch);
+
+  static const char bhsdq[] = "bhsdq";
+
+  bool virtRegFormatted = false;
+
+#ifndef ASMJIT_NO_COMPILER
+  if (Operand::isVirtId(rId)) {
+    if (emitter && emitter->isCompiler()) {
+      const BaseCompiler* cc = static_cast<const BaseCompiler*>(emitter);
+      if (cc->isVirtIdValid(rId)) {
+        VirtReg* vReg = cc->virtRegById(rId);
+        ASMJIT_ASSERT(vReg != nullptr);
+
+        const char* name = vReg->name();
+        if (name && name[0] != '\0')
+          ASMJIT_PROPAGATE(sb.append(name));
+        else
+          ASMJIT_PROPAGATE(sb.appendFormat("%%%u", unsigned(Operand::virtIdToIndex(rId))));
+
+        virtRegFormatted = true;
+      }
+    }
+  }
+#else
+  DebugUtils::unused(emitter, flags);
+#endif
+
+  if (!virtRegFormatted) {
+    char letter = '\0';
+    switch (regType) {
+      case RegType::kARM_GpW:
+        if (rId == Gp::kIdZr)
+          return sb.append("wzr");
+        if (rId == Gp::kIdSp)
+          return sb.append("wsp");
+
+        letter = 'w';
+        break;
+
+      case RegType::kARM_GpX:
+        if (rId == Gp::kIdZr)
+          return sb.append("xzr");
+        if (rId == Gp::kIdSp)
+          return sb.append("sp");
+
+        letter = 'x';
+        break;
+
+      case RegType::kARM_VecB:
+      case RegType::kARM_VecH:
+      case RegType::kARM_VecS:
+      case RegType::kARM_VecD:
+      case RegType::kARM_VecV:
+        letter = bhsdq[uint32_t(regType) - uint32_t(RegType::kARM_VecB)];
+        if (elementType)
+          letter = 'v';
+        break;
+
+      default:
+        ASMJIT_PROPAGATE(sb.appendFormat("<Reg-%u>?$u", uint32_t(regType), rId));
+        break;
+    }
+
+    if (letter)
+      ASMJIT_PROPAGATE(sb.appendFormat("%c%u", letter, rId));
+  }
+
+  if (elementType) {
+    char elementLetter = '\0';
+    uint32_t elementCount = 0;
+
+    switch (elementType) {
+      case Vec::kElementTypeB:
+        elementLetter = 'b';
+        elementCount = 16;
+        break;
+
+      case Vec::kElementTypeH:
+        elementLetter = 'h';
+        elementCount = 8;
+        break;
+
+      case Vec::kElementTypeS:
+        elementLetter = 's';
+        elementCount = 4;
+        break;
+
+      case Vec::kElementTypeD:
+        elementLetter = 'd';
+        elementCount = 2;
+        break;
+
+      default:
+        return sb.append(".<Unknown>");
+    }
+
+    if (elementLetter) {
+      if (elementIndex == 0xFFFFFFFFu) {
+        if (regType == RegType::kARM_VecD)
+          elementCount /= 2u;
+        ASMJIT_PROPAGATE(sb.appendFormat(".%u%c", elementCount, elementLetter));
+      }
+      else {
+        ASMJIT_PROPAGATE(sb.appendFormat(".%c[%u]", elementLetter, elementIndex));
+      }
+    }
+  }
+
+  return kErrorOk;
+}
+
+// a64::FormatterInternal - Format Operand
+// =======================================
+
+ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  const Operand_& op) noexcept {
+
+  if (op.isReg()) {
+    const BaseReg& reg = op.as<BaseReg>();
+
+    uint32_t elementType = op.as<Vec>().elementType();
+    uint32_t elementIndex = op.as<Vec>().elementIndex();
+
+    if (!op.as<Vec>().hasElementIndex())
+      elementIndex = 0xFFFFFFFFu;
+
+    return formatRegister(sb, flags, emitter, arch, reg.type(), reg.id(), elementType, elementIndex);
+  }
+
+  if (op.isMem()) {
+    const Mem& m = op.as<Mem>();
+    ASMJIT_PROPAGATE(sb.append('['));
+
+    if (m.hasBase()) {
+      if (m.hasBaseLabel()) {
+        ASMJIT_PROPAGATE(Formatter::formatLabel(sb, flags, emitter, m.baseId()));
+      }
+      else {
+        FormatFlags modifiedFlags = flags;
+        if (m.isRegHome()) {
+          ASMJIT_PROPAGATE(sb.append('&'));
+          modifiedFlags &= ~FormatFlags::kRegCasts;
+        }
+        ASMJIT_PROPAGATE(formatRegister(sb, modifiedFlags, emitter, arch, m.baseType(), m.baseId()));
+      }
+    }
+    else {
+      // ARM really requires base.
+      if (m.hasIndex() || m.hasOffset()) {
+        ASMJIT_PROPAGATE(sb.append("<None>"));
+      }
+    }
+
+    // The post index makes it look like there was another operand, but it's
+    // still the part of AsmJit's `arm::Mem` operand so it's consistent with
+    // other architectures.
+    if (m.isPostIndex())
+      ASMJIT_PROPAGATE(sb.append(']'));
+
+    if (m.hasIndex()) {
+      ASMJIT_PROPAGATE(sb.append(", "));
+      ASMJIT_PROPAGATE(formatRegister(sb, flags, emitter, arch, m.indexType(), m.indexId()));
+    }
+
+    if (m.hasOffset()) {
+      ASMJIT_PROPAGATE(sb.append(", "));
+
+      int64_t off = int64_t(m.offset());
+      uint32_t base = 10;
+
+      if (Support::test(flags, FormatFlags::kHexOffsets) && uint64_t(off) > 9)
+        base = 16;
+
+      if (base == 10) {
+        ASMJIT_PROPAGATE(sb.appendInt(off, base));
+      }
+      else {
+        ASMJIT_PROPAGATE(sb.append("0x"));
+        ASMJIT_PROPAGATE(sb.appendUInt(uint64_t(off), base));
+      }
+    }
+
+    if (m.hasShift()) {
+      ASMJIT_PROPAGATE(sb.append(' '));
+      if (!m.isPreOrPost())
+        ASMJIT_PROPAGATE(formatShiftOp(sb, (ShiftOp)m.predicate()));
+      ASMJIT_PROPAGATE(sb.appendFormat(" %u", m.shift()));
+    }
+
+    if (!m.isPostIndex())
+      ASMJIT_PROPAGATE(sb.append(']'));
+
+    if (m.isPreIndex())
+      ASMJIT_PROPAGATE(sb.append('!'));
+
+    return kErrorOk;
+  }
+
+  if (op.isImm()) {
+    const Imm& i = op.as<Imm>();
+    int64_t val = i.value();
+
+    if (Support::test(flags, FormatFlags::kHexImms) && uint64_t(val) > 9) {
+      ASMJIT_PROPAGATE(sb.append("0x"));
+      return sb.appendUInt(uint64_t(val), 16);
+    }
+    else {
+      return sb.appendInt(val, 10);
+    }
+  }
+
+  if (op.isLabel()) {
+    return Formatter::formatLabel(sb, flags, emitter, op.id());
+  }
+
+  return sb.append("<None>");
+}
+
+// a64::FormatterInternal - Format Instruction
+// ===========================================
+
+ASMJIT_FAVOR_SIZE Error FormatterInternal::formatInstruction(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept {
+
+  DebugUtils::unused(arch);
+
+  // Format instruction options and instruction mnemonic.
+  InstId instId = inst.realId();
+  if (instId < Inst::_kIdCount)
+    ASMJIT_PROPAGATE(InstInternal::instIdToString(arch, instId, sb));
+  else
+    ASMJIT_PROPAGATE(sb.appendFormat("[InstId=#%u]", unsigned(instId)));
+
+  CondCode cc = inst.armCondCode();
+  if (cc != CondCode::kAL) {
+    ASMJIT_PROPAGATE(sb.append('.'));
+    ASMJIT_PROPAGATE(formatCondCode(sb, cc));
+  }
+
+  for (uint32_t i = 0; i < opCount; i++) {
+    const Operand_& op = operands[i];
+    if (op.isNone())
+      break;
+
+    ASMJIT_PROPAGATE(sb.append(i == 0 ? " " : ", "));
+    ASMJIT_PROPAGATE(formatOperand(sb, flags, emitter, arch, op));
+  }
+
+  return kErrorOk;
+}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_LOGGING

libraries/asmjit/asmjit/arm/a64formatter_p.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64FORMATTER_P_H_INCLUDED
+#define ASMJIT_ARM_A64FORMATTER_P_H_INCLUDED
+
+#include "../core/api-config.h"
+#ifndef ASMJIT_NO_LOGGING
+
+#include "../core/formatter.h"
+#include "../core/string.h"
+#include "../arm/armformatter_p.h"
+#include "../arm/a64globals.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \cond INTERNAL
+//! \addtogroup asmjit_a64
+//! \{
+
+namespace FormatterInternal {
+
+using namespace arm::FormatterInternal;
+
+Error ASMJIT_CDECL formatRegister(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  RegType regType,
+  uint32_t regId,
+  uint32_t elementType = 0,
+  uint32_t elementIndex = 0xFFFFFFFFu) noexcept;
+
+Error ASMJIT_CDECL formatOperand(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  const Operand_& op) noexcept;
+
+Error ASMJIT_CDECL formatInstruction(
+  String& sb,
+  FormatFlags flags,
+  const BaseEmitter* emitter,
+  Arch arch,
+  const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept;
+
+} // {FormatterInternal}
+
+//! \}
+//! \endcond
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_LOGGING
+#endif // ASMJIT_ARM_A64FORMATTER_P_H_INCLUDED

libraries/asmjit/asmjit/arm/a64func.cpp

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#include "../core/api-build_p.h"
+#if !defined(ASMJIT_NO_AARCH64)
+
+#include "../arm/a64func_p.h"
+#include "../arm/a64operand.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+namespace FuncInternal {
+
+static inline bool shouldThreatAsCDecl(CallConvId ccId) noexcept {
+  return ccId == CallConvId::kCDecl ||
+         ccId == CallConvId::kStdCall ||
+         ccId == CallConvId::kFastCall ||
+         ccId == CallConvId::kVectorCall ||
+         ccId == CallConvId::kThisCall ||
+         ccId == CallConvId::kRegParm1 ||
+         ccId == CallConvId::kRegParm2 ||
+         ccId == CallConvId::kRegParm3;
+}
+
+static RegType regTypeFromFpOrVecTypeId(TypeId typeId) noexcept {
+  if (typeId == TypeId::kFloat32)
+    return RegType::kARM_VecS;
+  else if (typeId == TypeId::kFloat64)
+    return RegType::kARM_VecD;
+  else if (TypeUtils::isVec32(typeId))
+    return RegType::kARM_VecS;
+  else if (TypeUtils::isVec64(typeId))
+    return RegType::kARM_VecD;
+  else if (TypeUtils::isVec128(typeId))
+    return RegType::kARM_VecV;
+  else
+    return RegType::kNone;
+}
+
+ASMJIT_FAVOR_SIZE Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept {
+  cc.setArch(environment.arch());
+
+  cc.setSaveRestoreRegSize(RegGroup::kGp, 8);
+  cc.setSaveRestoreRegSize(RegGroup::kVec, 8);
+  cc.setSaveRestoreAlignment(RegGroup::kGp, 16);
+  cc.setSaveRestoreAlignment(RegGroup::kVec, 16);
+  cc.setSaveRestoreAlignment(RegGroup::kExtraVirt2, 1);
+  cc.setSaveRestoreAlignment(RegGroup::kExtraVirt3, 1);
+  cc.setPassedOrder(RegGroup::kGp, 0, 1, 2, 3, 4, 5, 6, 7);
+  cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3, 4, 5, 6, 7);
+  cc.setNaturalStackAlignment(16);
+
+  if (shouldThreatAsCDecl(ccId)) {
+    // ARM doesn't have that many calling conventions as we can find in X86 world, treat most conventions as __cdecl.
+    cc.setId(CallConvId::kCDecl);
+    cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(Gp::kIdOs, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30));
+    cc.setPreservedRegs(RegGroup::kVec, Support::bitMask(8, 9, 10, 11, 12, 13, 14, 15));
+  }
+  else {
+    cc.setId(ccId);
+    cc.setSaveRestoreRegSize(RegGroup::kVec, 16);
+    cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30));
+    cc.setPreservedRegs(RegGroup::kVec, Support::bitMask(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31));
+  }
+
+  return kErrorOk;
+}
+
+ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept {
+  DebugUtils::unused(signature);
+
+  const CallConv& cc = func.callConv();
+  uint32_t stackOffset = 0;
+
+  uint32_t i;
+  uint32_t argCount = func.argCount();
+
+  if (func.hasRet()) {
+    for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) {
+      TypeId typeId = func._rets[valueIndex].typeId();
+
+      // Terminate at the first void type (end of the pack).
+      if (typeId == TypeId::kVoid)
+        break;
+
+      switch (typeId) {
+        case TypeId::kInt8:
+        case TypeId::kInt16:
+        case TypeId::kInt32: {
+          func._rets[valueIndex].initReg(RegType::kARM_GpW, valueIndex, TypeId::kInt32);
+          break;
+        }
+
+        case TypeId::kUInt8:
+        case TypeId::kUInt16:
+        case TypeId::kUInt32: {
+          func._rets[valueIndex].initReg(RegType::kARM_GpW, valueIndex, TypeId::kUInt32);
+          break;
+        }
+
+        case TypeId::kInt64:
+        case TypeId::kUInt64: {
+          func._rets[valueIndex].initReg(RegType::kARM_GpX, valueIndex, typeId);
+          break;
+        }
+
+        default: {
+          RegType regType = regTypeFromFpOrVecTypeId(typeId);
+          if (regType == RegType::kNone)
+            return DebugUtils::errored(kErrorInvalidRegType);
+
+          func._rets[valueIndex].initReg(regType, valueIndex, typeId);
+          break;
+        }
+      }
+    }
+  }
+
+  switch (cc.strategy()) {
+    case CallConvStrategy::kDefault: {
+      uint32_t gpzPos = 0;
+      uint32_t vecPos = 0;
+
+      for (i = 0; i < argCount; i++) {
+        FuncValue& arg = func._args[i][0];
+        TypeId typeId = arg.typeId();
+
+        if (TypeUtils::isInt(typeId)) {
+          uint32_t regId = BaseReg::kIdBad;
+
+          if (gpzPos < CallConv::kMaxRegArgsPerGroup)
+            regId = cc._passedOrder[RegGroup::kGp].id[gpzPos];
+
+          if (regId != BaseReg::kIdBad) {
+            RegType regType = typeId <= TypeId::kUInt32 ? RegType::kARM_GpW : RegType::kARM_GpX;
+            arg.assignRegData(regType, regId);
+            func.addUsedRegs(RegGroup::kGp, Support::bitMask(regId));
+            gpzPos++;
+          }
+          else {
+            uint32_t size = Support::max<uint32_t>(TypeUtils::sizeOf(typeId), registerSize);
+            arg.assignStackOffset(int32_t(stackOffset));
+            stackOffset += size;
+          }
+          continue;
+        }
+
+        if (TypeUtils::isFloat(typeId) || TypeUtils::isVec(typeId)) {
+          uint32_t regId = BaseReg::kIdBad;
+
+          if (vecPos < CallConv::kMaxRegArgsPerGroup)
+            regId = cc._passedOrder[RegGroup::kVec].id[vecPos];
+
+          if (regId != BaseReg::kIdBad) {
+            RegType regType = regTypeFromFpOrVecTypeId(typeId);
+            if (regType == RegType::kNone)
+              return DebugUtils::errored(kErrorInvalidRegType);
+
+            arg.initTypeId(typeId);
+            arg.assignRegData(regType, regId);
+            func.addUsedRegs(RegGroup::kVec, Support::bitMask(regId));
+            vecPos++;
+          }
+          else {
+            uint32_t size = TypeUtils::sizeOf(typeId);
+            arg.assignStackOffset(int32_t(stackOffset));
+            stackOffset += size;
+          }
+          continue;
+        }
+      }
+      break;
+    }
+
+    default:
+      return DebugUtils::errored(kErrorInvalidState);
+  }
+
+  func._argStackSize = stackOffset;
+  return kErrorOk;
+}
+
+} // {FuncInternal}
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // !ASMJIT_NO_AARCH64

libraries/asmjit/asmjit/arm/a64func_p.h

+// This file is part of AsmJit project <https://asmjit.com>
+//
+// See asmjit.h or LICENSE.md for license and copyright information
+// SPDX-License-Identifier: Zlib
+
+#ifndef ASMJIT_ARM_A64FUNC_P_H_INCLUDED
+#define ASMJIT_ARM_A64FUNC_P_H_INCLUDED
+
+#include "../core/func.h"
+
+ASMJIT_BEGIN_SUB_NAMESPACE(a64)
+
+//! \cond INTERNAL
+//! \addtogroup asmjit_a64
+//! \{
+
+//! AArch64-specific function API (calling conventions and other utilities).
+namespace FuncInternal {
+
+//! Initialize `CallConv` structure (AArch64 specific).
+Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept;
+
+//! Initialize `FuncDetail` (AArch64 specific).
+Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept;
+
+} // {FuncInternal}
+
+//! \}
+//! \endcond
+
+ASMJIT_END_SUB_NAMESPACE
+
+#endif // ASMJIT_ARM_A64FUNC_P_H_INCLUDED