Skip to content

GitLab

Unverified Commit cc7018ea authored by Peter Eastman's avatar Peter Eastman Committed by GitHub
Browse files

JIT compilation on ARM processors (#3517) 

* Upgraded to new version of asmjit

* JIT compilation for ARM

* Updated CMake script
parent 0644f054
Hide whitespace changes
Inline Side-by-side
Showing with 5430 additions and 0 deletions
libraries/asmjit/asmjit/core/rastack.cpp 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
#ifndef ASMJIT_NO_COMPILER
#include "../core/rastack_p.h"
#include "../core/support.h"
ASMJIT_BEGIN_NAMESPACE
// RAStackAllocator - Slots
// ========================
RAStackSlot* RAStackAllocator::newSlot(uint32_t baseRegId, uint32_t size, uint32_t alignment, uint32_t flags) noexcept {
if (ASMJIT_UNLIKELY(_slots.willGrow(allocator(), 1) != kErrorOk))
return nullptr;
RAStackSlot* slot = allocator()->allocT<RAStackSlot>();
if (ASMJIT_UNLIKELY(!slot))
return nullptr;
slot->_baseRegId = uint8_t(baseRegId);
slot->_alignment = uint8_t(Support::max<uint32_t>(alignment, 1));
slot->_flags = uint16_t(flags);
slot->_useCount = 0;
slot->_size = size;
slot->_weight = 0;
slot->_offset = 0;
_alignment = Support::max<uint32_t>(_alignment, alignment);
_slots.appendUnsafe(slot);
return slot;
}
// RAStackAllocator - Utilities
// ============================
struct RAStackGap {
inline RAStackGap() noexcept
: offset(0),
size(0) {}
inline RAStackGap(uint32_t offset, uint32_t size) noexcept
: offset(offset),
size(size) {}
inline RAStackGap(const RAStackGap& other) noexcept
: offset(other.offset),
size(other.size) {}
uint32_t offset;
uint32_t size;
};
Error RAStackAllocator::calculateStackFrame() noexcept {
// Base weight added to all registers regardless of their size and alignment.
uint32_t kBaseRegWeight = 16;
// STEP 1:
//
// Update usage based on the size of the slot. We boost smaller slots in a way that 32-bit register has higher
// priority than a 128-bit register, however, if one 128-bit register is used 4 times more than some other 32-bit
// register it will overweight it.
for (RAStackSlot* slot : _slots) {
uint32_t alignment = slot->alignment();
ASMJIT_ASSERT(alignment > 0);
uint32_t power = Support::min<uint32_t>(Support::ctz(alignment), 6);
uint64_t weight;
if (slot->isRegHome())
weight = kBaseRegWeight + (uint64_t(slot->useCount()) * (7 - power));
else
weight = power;
// If overflown, which has less chance of winning a lottery, just use max possible weight. In such case it
// probably doesn't matter at all.
if (weight > 0xFFFFFFFFu)
weight = 0xFFFFFFFFu;
slot->setWeight(uint32_t(weight));
}
// STEP 2:
//
// Sort stack slots based on their newly calculated weight (in descending order).
_slots.sort([](const RAStackSlot* a, const RAStackSlot* b) noexcept {
return a->weight() > b->weight() ? 1 :
a->weight() == b->weight() ? 0 : -1;
});
// STEP 3:
//
// Calculate offset of each slot. We start from the slot that has the highest weight and advance to slots with
// lower weight. It could look that offsets start from the first slot in our list and then simply increase, but
// it's not always the case as we also try to fill all gaps introduced by the fact that slots are sorted by
// weight and not by size & alignment, so when we need to align some slot we distribute the gap caused by the
// alignment to `gaps`.
uint32_t offset = 0;
ZoneVector<RAStackGap> gaps[kSizeCount - 1];
for (RAStackSlot* slot : _slots) {
if (slot->isStackArg())
continue;
uint32_t slotAlignment = slot->alignment();
uint32_t alignedOffset = Support::alignUp(offset, slotAlignment);
// Try to find a slot within gaps first, before advancing the `offset`.
bool foundGap = false;
uint32_t gapSize = 0;
uint32_t gapOffset = 0;
{
uint32_t slotSize = slot->size();
if (slotSize < (1u << uint32_t(ASMJIT_ARRAY_SIZE(gaps)))) {
// Iterate from the lowest to the highest possible.
uint32_t index = Support::ctz(slotSize);
do {
if (!gaps[index].empty()) {
RAStackGap gap = gaps[index].pop();
ASMJIT_ASSERT(Support::isAligned(gap.offset, slotAlignment));
slot->setOffset(int32_t(gap.offset));
gapSize = gap.size - slotSize;
gapOffset = gap.offset - slotSize;
foundGap = true;
break;
}
} while (++index < uint32_t(ASMJIT_ARRAY_SIZE(gaps)));
}
}
// No gap found, we may create a new one(s) if the current offset is not aligned.
if (!foundGap && offset != alignedOffset) {
gapSize = alignedOffset - offset;
gapOffset = alignedOffset;
offset = alignedOffset;
}
// True if we have found a gap and not filled all of it or we aligned the current offset.
if (gapSize) {
uint32_t gapEnd = gapSize + gapOffset;
while (gapOffset < gapEnd) {
uint32_t index = Support::ctz(gapOffset);
uint32_t slotSize = 1u << index;
// Weird case, better to bail...
if (gapEnd - gapOffset < slotSize)
break;
ASMJIT_PROPAGATE(gaps[index].append(allocator(), RAStackGap(gapOffset, slotSize)));
gapOffset += slotSize;
}
}
if (!foundGap) {
ASMJIT_ASSERT(Support::isAligned(offset, slotAlignment));
slot->setOffset(int32_t(offset));
offset += slot->size();
}
}
_stackSize = Support::alignUp(offset, _alignment);
return kErrorOk;
}
Error RAStackAllocator::adjustSlotOffsets(int32_t offset) noexcept {
for (RAStackSlot* slot : _slots)
if (!slot->isStackArg())
slot->_offset += offset;
return kErrorOk;
}
ASMJIT_END_NAMESPACE
#endif // !ASMJIT_NO_COMPILER
libraries/asmjit/asmjit/core/rastack_p.h 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#ifndef ASMJIT_CORE_RASTACK_P_H_INCLUDED
#define ASMJIT_CORE_RASTACK_P_H_INCLUDED
#include "../core/api-config.h"
#ifndef ASMJIT_NO_COMPILER
#include "../core/radefs_p.h"
ASMJIT_BEGIN_NAMESPACE
//! \cond INTERNAL
//! \addtogroup asmjit_ra
//! \{
//! Stack slot.
struct RAStackSlot {
//! Stack slot flags.
//!
//! TODO: kFlagStackArg is not used by the current implementation, do we need to keep it?
enum Flags : uint16_t {
//! Stack slot is register home slot.
kFlagRegHome = 0x0001u,
//! Stack slot position matches argument passed via stack.
kFlagStackArg = 0x0002u
};
enum ArgIndex : uint32_t {
kNoArgIndex = 0xFF
};
//! \name Members
//! \{
//! Base register used to address the stack.
uint8_t _baseRegId;
//! Minimum alignment required by the slot.
uint8_t _alignment;
//! Reserved for future use.
uint16_t _flags;
//! Size of memory required by the slot.
uint32_t _size;
//! Usage counter (one unit equals one memory access).
uint32_t _useCount;
//! Weight of the slot, calculated by \ref RAStackAllocator::calculateStackFrame().
uint32_t _weight;
//! Stack offset, calculated by \ref RAStackAllocator::calculateStackFrame().
int32_t _offset;
//! \}
//! \name Accessors
//! \{
inline uint32_t baseRegId() const noexcept { return _baseRegId; }
inline void setBaseRegId(uint32_t id) noexcept { _baseRegId = uint8_t(id); }
inline uint32_t size() const noexcept { return _size; }
inline uint32_t alignment() const noexcept { return _alignment; }
inline uint32_t flags() const noexcept { return _flags; }
inline bool hasFlag(uint32_t flag) const noexcept { return (_flags & flag) != 0; }
inline void addFlags(uint32_t flags) noexcept { _flags = uint16_t(_flags | flags); }
inline bool isRegHome() const noexcept { return hasFlag(kFlagRegHome); }
inline bool isStackArg() const noexcept { return hasFlag(kFlagStackArg); }
inline uint32_t useCount() const noexcept { return _useCount; }
inline void addUseCount(uint32_t n = 1) noexcept { _useCount += n; }
inline uint32_t weight() const noexcept { return _weight; }
inline void setWeight(uint32_t weight) noexcept { _weight = weight; }
inline int32_t offset() const noexcept { return _offset; }
inline void setOffset(int32_t offset) noexcept { _offset = offset; }
//! \}
};
typedef ZoneVector<RAStackSlot*> RAStackSlots;
//! Stack allocator.
class RAStackAllocator {
public:
ASMJIT_NONCOPYABLE(RAStackAllocator)
enum Size : uint32_t {
kSize1 = 0,
kSize2 = 1,
kSize4 = 2,
kSize8 = 3,
kSize16 = 4,
kSize32 = 5,
kSize64 = 6,
kSizeCount = 7
};
//! \name Members
//! \{
//! Allocator used to allocate internal data.
ZoneAllocator* _allocator;
//! Count of bytes used by all slots.
uint32_t _bytesUsed;
//! Calculated stack size (can be a bit greater than `_bytesUsed`).
uint32_t _stackSize;
//! Minimum stack alignment.
uint32_t _alignment;
//! Stack slots vector.
RAStackSlots _slots;
//! \}
//! \name Construction & Destruction
//! \{
inline RAStackAllocator() noexcept
: _allocator(nullptr),
_bytesUsed(0),
_stackSize(0),
_alignment(1),
_slots() {}
inline void reset(ZoneAllocator* allocator) noexcept {
_allocator = allocator;
_bytesUsed = 0;
_stackSize = 0;
_alignment = 1;
_slots.reset();
}
//! \}
//! \name Accessors
//! \{
inline ZoneAllocator* allocator() const noexcept { return _allocator; }
inline uint32_t bytesUsed() const noexcept { return _bytesUsed; }
inline uint32_t stackSize() const noexcept { return _stackSize; }
inline uint32_t alignment() const noexcept { return _alignment; }
inline RAStackSlots& slots() noexcept { return _slots; }
inline const RAStackSlots& slots() const noexcept { return _slots; }
inline uint32_t slotCount() const noexcept { return _slots.size(); }
//! \}
//! \name Utilities
//! \{
RAStackSlot* newSlot(uint32_t baseRegId, uint32_t size, uint32_t alignment, uint32_t flags = 0) noexcept;
Error calculateStackFrame() noexcept;
Error adjustSlotOffsets(int32_t offset) noexcept;
//! \}
};
//! \}
//! \endcond
ASMJIT_END_NAMESPACE
#endif // !ASMJIT_NO_COMPILER
#endif // ASMJIT_CORE_RASTACK_P_H_INCLUDED
libraries/asmjit/asmjit/core/string.cpp 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
#include "../core/string.h"
#include "../core/support.h"
ASMJIT_BEGIN_NAMESPACE
// String - Globals
// ================
static const char String_baseN[] = "0123456789ABCDEF";
constexpr size_t kMinAllocSize = 64;
constexpr size_t kMaxAllocSize = SIZE_MAX - Globals::kGrowThreshold;
// String - Clear & Reset
// ======================
Error String::reset() noexcept {
if (_type == kTypeLarge)
::free(_large.data);
_resetInternal();
return kErrorOk;
}
Error String::clear() noexcept {
if (isLargeOrExternal()) {
_large.size = 0;
_large.data[0] = '\0';
}
else {
_raw.uptr[0] = 0;
}
return kErrorOk;
}
// String - Prepare
// ================
char* String::prepare(ModifyOp op, size_t size) noexcept {
char* curData;
size_t curSize;
size_t curCapacity;
if (isLargeOrExternal()) {
curData = this->_large.data;
curSize = this->_large.size;
curCapacity = this->_large.capacity;
}
else {
curData = this->_small.data;
curSize = this->_small.type;
curCapacity = kSSOCapacity;
}
if (op == ModifyOp::kAssign) {
if (size > curCapacity) {
// Prevent arithmetic overflow.
if (ASMJIT_UNLIKELY(size >= kMaxAllocSize))
return nullptr;
size_t newCapacity = Support::alignUp<size_t>(size + 1, kMinAllocSize);
char* newData = static_cast<char*>(::malloc(newCapacity));
if (ASMJIT_UNLIKELY(!newData))
return nullptr;
if (_type == kTypeLarge)
::free(curData);
_large.type = kTypeLarge;
_large.size = size;
_large.capacity = newCapacity - 1;
_large.data = newData;
newData[size] = '\0';
return newData;
}
else {
_setSize(size);
curData[size] = '\0';
return curData;
}
}
else {
// Prevent arithmetic overflow.
if (ASMJIT_UNLIKELY(size >= kMaxAllocSize - curSize))
return nullptr;
size_t newSize = size + curSize;
size_t newSizePlusOne = newSize + 1;
if (newSizePlusOne > curCapacity) {
size_t newCapacity = Support::max<size_t>(curCapacity + 1, kMinAllocSize);
if (newCapacity < newSizePlusOne && newCapacity < Globals::kGrowThreshold)
newCapacity = Support::alignUpPowerOf2(newCapacity);
if (newCapacity < newSizePlusOne)
newCapacity = Support::alignUp(newSizePlusOne, Globals::kGrowThreshold);
if (ASMJIT_UNLIKELY(newCapacity < newSizePlusOne))
return nullptr;
char* newData = static_cast<char*>(::malloc(newCapacity));
if (ASMJIT_UNLIKELY(!newData))
return nullptr;
memcpy(newData, curData, curSize);
if (_type == kTypeLarge)
::free(curData);
_large.type = kTypeLarge;
_large.size = newSize;
_large.capacity = newCapacity - 1;
_large.data = newData;
newData[newSize] = '\0';
return newData + curSize;
}
else {
_setSize(newSize);
curData[newSize] = '\0';
return curData + curSize;
}
}
}
// String - Assign
// ===============
Error String::assign(const char* data, size_t size) noexcept {
char* dst = nullptr;
// Null terminated string without `size` specified.
if (size == SIZE_MAX)
size = data ? strlen(data) : size_t(0);
if (isLargeOrExternal()) {
if (size <= _large.capacity) {
dst = _large.data;
_large.size = size;
}
else {
size_t capacityPlusOne = Support::alignUp(size + 1, 32);
if (ASMJIT_UNLIKELY(capacityPlusOne < size))
return DebugUtils::errored(kErrorOutOfMemory);
dst = static_cast<char*>(::malloc(capacityPlusOne));
if (ASMJIT_UNLIKELY(!dst))
return DebugUtils::errored(kErrorOutOfMemory);
if (_type == kTypeLarge)
::free(_large.data);
_large.type = kTypeLarge;
_large.data = dst;
_large.size = size;
_large.capacity = capacityPlusOne - 1;
}
}
else {
if (size <= kSSOCapacity) {
ASMJIT_ASSERT(size < 0xFFu);
dst = _small.data;
_small.type = uint8_t(size);
}
else {
dst = static_cast<char*>(::malloc(size + 1));
if (ASMJIT_UNLIKELY(!dst))
return DebugUtils::errored(kErrorOutOfMemory);
_large.type = kTypeLarge;
_large.data = dst;
_large.size = size;
_large.capacity = size;
}
}
// Optionally copy data from `data` and null-terminate.
if (data && size) {
// NOTE: It's better to use `memmove()`. If, for any reason, somebody uses
// this function to substring the same string it would work as expected.
::memmove(dst, data, size);
}
dst[size] = '\0';
return kErrorOk;
}
// String - Operations
// ===================
Error String::_opString(ModifyOp op, const char* str, size_t size) noexcept {
if (size == SIZE_MAX)
size = str ? strlen(str) : size_t(0);
if (!size)
return kErrorOk;
char* p = prepare(op, size);
if (!p)
return DebugUtils::errored(kErrorOutOfMemory);
memcpy(p, str, size);
return kErrorOk;
}
Error String::_opChar(ModifyOp op, char c) noexcept {
char* p = prepare(op, 1);
if (!p)
return DebugUtils::errored(kErrorOutOfMemory);
*p = c;
return kErrorOk;
}
Error String::_opChars(ModifyOp op, char c, size_t n) noexcept {
if (!n)
return kErrorOk;
char* p = prepare(op, n);
if (!p)
return DebugUtils::errored(kErrorOutOfMemory);
memset(p, c, n);
return kErrorOk;
}
Error String::padEnd(size_t n, char c) noexcept {
size_t size = this->size();
return n > size ? appendChars(c, n - size) : kErrorOk;
}
Error String::_opNumber(ModifyOp op, uint64_t i, uint32_t base, size_t width, StringFormatFlags flags) noexcept {
if (base == 0)
base = 10;
char buf[128];
char* p = buf + ASMJIT_ARRAY_SIZE(buf);
uint64_t orig = i;
char sign = '\0';
// Format Sign
// -----------
if (Support::test(flags, StringFormatFlags::kSigned) && int64_t(i) < 0) {
i = uint64_t(-int64_t(i));
sign = '-';
}
else if (Support::test(flags, StringFormatFlags::kShowSign)) {
sign = '+';
}
else if (Support::test(flags, StringFormatFlags::kShowSpace)) {
sign = ' ';
}
// Format Number
// -------------
switch (base) {
case 2:
case 8:
case 16: {
uint32_t shift = Support::ctz(base);
uint32_t mask = base - 1;
do {
uint64_t d = i >> shift;
size_t r = size_t(i & mask);
*--p = String_baseN[r];
i = d;
} while (i);
break;
}
case 10: {
do {
uint64_t d = i / 10;
uint64_t r = i % 10;
*--p = char(uint32_t('0') + uint32_t(r));
i = d;
} while (i);
break;
}
default:
return DebugUtils::errored(kErrorInvalidArgument);
}
size_t numberSize = (size_t)(buf + ASMJIT_ARRAY_SIZE(buf) - p);
// Alternate Form
// --------------
if (Support::test(flags, StringFormatFlags::kAlternate)) {
if (base == 8) {
if (orig != 0)
*--p = '0';
}
if (base == 16) {
*--p = 'x';
*--p = '0';
}
}
// String Width
// ------------
if (sign != 0)
*--p = sign;
if (width > 256)
width = 256;
if (width <= numberSize)
width = 0;
else
width -= numberSize;
// Finalize
// --------
size_t prefixSize = (size_t)(buf + ASMJIT_ARRAY_SIZE(buf) - p) - numberSize;
char* data = prepare(op, prefixSize + width + numberSize);
if (!data)
return DebugUtils::errored(kErrorOutOfMemory);
memcpy(data, p, prefixSize);
data += prefixSize;
memset(data, '0', width);
data += width;
memcpy(data, p + prefixSize, numberSize);
return kErrorOk;
}
Error String::_opHex(ModifyOp op, const void* data, size_t size, char separator) noexcept {
char* dst;
const uint8_t* src = static_cast<const uint8_t*>(data);
if (!size)
return kErrorOk;
if (separator) {
if (ASMJIT_UNLIKELY(size >= SIZE_MAX / 3))
return DebugUtils::errored(kErrorOutOfMemory);
dst = prepare(op, size * 3 - 1);
if (ASMJIT_UNLIKELY(!dst))
return DebugUtils::errored(kErrorOutOfMemory);
size_t i = 0;
for (;;) {
dst[0] = String_baseN[(src[0] >> 4) & 0xF];
dst[1] = String_baseN[(src[0] ) & 0xF];
if (++i == size)
break;
// This makes sure that the separator is only put between two hexadecimal bytes.
dst[2] = separator;
dst += 3;
src++;
}
}
else {
if (ASMJIT_UNLIKELY(size >= SIZE_MAX / 2))
return DebugUtils::errored(kErrorOutOfMemory);
dst = prepare(op, size * 2);
if (ASMJIT_UNLIKELY(!dst))
return DebugUtils::errored(kErrorOutOfMemory);
for (size_t i = 0; i < size; i++, dst += 2, src++) {
dst[0] = String_baseN[(src[0] >> 4) & 0xF];
dst[1] = String_baseN[(src[0] ) & 0xF];
}
}
return kErrorOk;
}
Error String::_opFormat(ModifyOp op, const char* fmt, ...) noexcept {
Error err;
va_list ap;
va_start(ap, fmt);
err = _opVFormat(op, fmt, ap);
va_end(ap);
return err;
}
Error String::_opVFormat(ModifyOp op, const char* fmt, va_list ap) noexcept {
size_t startAt = (op == ModifyOp::kAssign) ? size_t(0) : size();
size_t remainingCapacity = capacity() - startAt;
char buf[1024];
int fmtResult;
size_t outputSize;
va_list apCopy;
va_copy(apCopy, ap);
if (remainingCapacity >= 128) {
fmtResult = vsnprintf(data() + startAt, remainingCapacity, fmt, ap);
outputSize = size_t(fmtResult);
if (ASMJIT_LIKELY(outputSize <= remainingCapacity)) {
_setSize(startAt + outputSize);
return kErrorOk;
}
}
else {
fmtResult = vsnprintf(buf, ASMJIT_ARRAY_SIZE(buf), fmt, ap);
outputSize = size_t(fmtResult);
if (ASMJIT_LIKELY(outputSize < ASMJIT_ARRAY_SIZE(buf)))
return _opString(op, buf, outputSize);
}
if (ASMJIT_UNLIKELY(fmtResult < 0))
return DebugUtils::errored(kErrorInvalidState);
char* p = prepare(op, outputSize);
if (ASMJIT_UNLIKELY(!p))
return DebugUtils::errored(kErrorOutOfMemory);
fmtResult = vsnprintf(p, outputSize + 1, fmt, apCopy);
ASMJIT_ASSERT(size_t(fmtResult) == outputSize);
return kErrorOk;
}
Error String::truncate(size_t newSize) noexcept {
if (isLargeOrExternal()) {
if (newSize < _large.size) {
_large.data[newSize] = '\0';
_large.size = newSize;
}
}
else {
if (newSize < _type) {
_small.data[newSize] = '\0';
_small.type = uint8_t(newSize);
}
}
return kErrorOk;
}
bool String::eq(const char* other, size_t size) const noexcept {
const char* aData = data();
const char* bData = other;
size_t aSize = this->size();
size_t bSize = size;
if (bSize == SIZE_MAX) {
size_t i;
for (i = 0; i < aSize; i++)
if (aData[i] != bData[i] || bData[i] == 0)
return false;
return bData[i] == 0;
}
else {
if (aSize != bSize)
return false;
return ::memcmp(aData, bData, aSize) == 0;
}
}
// String - Tests
// ==============
#if defined(ASMJIT_TEST)
UNIT(core_string) {
String s;
EXPECT(s.isLargeOrExternal() == false);
EXPECT(s.isExternal() == false);
EXPECT(s.assign('a') == kErrorOk);
EXPECT(s.size() == 1);
EXPECT(s.capacity() == String::kSSOCapacity);
EXPECT(s.data()[0] == 'a');
EXPECT(s.data()[1] == '\0');
EXPECT(s.eq("a") == true);
EXPECT(s.eq("a", 1) == true);
EXPECT(s.assignChars('b', 4) == kErrorOk);
EXPECT(s.size() == 4);
EXPECT(s.capacity() == String::kSSOCapacity);
EXPECT(s.data()[0] == 'b');
EXPECT(s.data()[1] == 'b');
EXPECT(s.data()[2] == 'b');
EXPECT(s.data()[3] == 'b');
EXPECT(s.data()[4] == '\0');
EXPECT(s.eq("bbbb") == true);
EXPECT(s.eq("bbbb", 4) == true);
EXPECT(s.assign("abc") == kErrorOk);
EXPECT(s.size() == 3);
EXPECT(s.capacity() == String::kSSOCapacity);
EXPECT(s.data()[0] == 'a');
EXPECT(s.data()[1] == 'b');
EXPECT(s.data()[2] == 'c');
EXPECT(s.data()[3] == '\0');
EXPECT(s.eq("abc") == true);
EXPECT(s.eq("abc", 3) == true);
const char* large = "Large string that will not fit into SSO buffer";
EXPECT(s.assign(large) == kErrorOk);
EXPECT(s.isLargeOrExternal() == true);
EXPECT(s.size() == strlen(large));
EXPECT(s.capacity() > String::kSSOCapacity);
EXPECT(s.eq(large) == true);
EXPECT(s.eq(large, strlen(large)) == true);
const char* additional = " (additional content)";
EXPECT(s.isLargeOrExternal() == true);
EXPECT(s.append(additional) == kErrorOk);
EXPECT(s.size() == strlen(large) + strlen(additional));
EXPECT(s.clear() == kErrorOk);
EXPECT(s.size() == 0);
EXPECT(s.empty() == true);
EXPECT(s.data()[0] == '\0');
EXPECT(s.isLargeOrExternal() == true); // Clear should never release the memory.
EXPECT(s.appendUInt(1234) == kErrorOk);
EXPECT(s.eq("1234") == true);
EXPECT(s.assignUInt(0xFFFF, 16, 0, StringFormatFlags::kAlternate) == kErrorOk);
EXPECT(s.eq("0xFFFF"));
StringTmp<64> sTmp;
EXPECT(sTmp.isLargeOrExternal());
EXPECT(sTmp.isExternal());
EXPECT(sTmp.appendChars(' ', 1000) == kErrorOk);
EXPECT(!sTmp.isExternal());
}
#endif
ASMJIT_END_NAMESPACE
libraries/asmjit/asmjit/core/string.h 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#ifndef ASMJIT_CORE_STRING_H_INCLUDED
#define ASMJIT_CORE_STRING_H_INCLUDED
#include "../core/support.h"
#include "../core/zone.h"
ASMJIT_BEGIN_NAMESPACE
//! \addtogroup asmjit_utilities
//! \{
//! Format flags used by \ref String API.
enum class StringFormatFlags : uint32_t {
//! No flags.
kNone = 0x00000000u,
//! Show sign.
kShowSign = 0x00000001u,
//! Show space.
kShowSpace = 0x00000002u,
//! Alternate form (use 0x when formatting HEX number).
kAlternate = 0x00000004u,
//! The input is signed.
kSigned = 0x80000000u
};
ASMJIT_DEFINE_ENUM_FLAGS(StringFormatFlags)
//! Fixed string - only useful for strings that would never exceed `N - 1` characters; always null-terminated.
template<size_t N>
union FixedString {
//! \name Constants
//! \{
// This cannot be constexpr as GCC 4.8 refuses constexpr members of unions.
enum : uint32_t {
kNumUInt32Words = uint32_t((N + sizeof(uint32_t) - 1) / sizeof(uint32_t))
};
//! \}
//! \name Members
//! \{
char str[kNumUInt32Words * sizeof(uint32_t)];
uint32_t u32[kNumUInt32Words];
//! \}
//! \name Utilities
//! \{
inline bool eq(const char* other) const noexcept {
return strcmp(str, other) == 0;
}
//! \}
};
//! A simple non-reference counted string that uses small string optimization (SSO).
//!
//! This string has 3 allocation possibilities:
//!
//! 1. Small - embedded buffer is used for up to `kSSOCapacity` characters. This should handle most small
//! strings and thus avoid dynamic memory allocation for most use-cases.
//!
//! 2. Large - string that doesn't fit into an embedded buffer (or string that was truncated from a larger
//! buffer) and is owned by AsmJit. When you destroy the string AsmJit would automatically
//! release the large buffer.
//!
//! 3. External - like Large (2), however, the large buffer is not owned by AsmJit and won't be released when
//! the string is destroyed or reallocated. This is mostly useful for working with larger temporary
//! strings allocated on stack or with immutable strings.
class String {
public:
ASMJIT_NONCOPYABLE(String)
//! String operation.
enum class ModifyOp : uint32_t {
//! Assignment - a new content replaces the current one.
kAssign = 0,
//! Append - a new content is appended to the string.
kAppend = 1
};
//! \cond INTERNAL
enum : uint32_t {
kLayoutSize = 32,
kSSOCapacity = kLayoutSize - 2
};
//! String type.
enum Type : uint8_t {
//! Large string (owned by String).
kTypeLarge = 0x1Fu,
//! External string (zone allocated or not owned by String).
kTypeExternal = 0x20u
};
union Raw {
uint8_t u8[kLayoutSize];
uint64_t u64[kLayoutSize / sizeof(uint64_t)];
uintptr_t uptr[kLayoutSize / sizeof(uintptr_t)];
};
struct Small {
uint8_t type;
char data[kSSOCapacity + 1u];
};
struct Large {
uint8_t type;
uint8_t reserved[sizeof(uintptr_t) - 1];
size_t size;
size_t capacity;
char* data;
};
union {
uint8_t _type;
Raw _raw;
Small _small;
Large _large;
};
//! \endcond
//! \name Construction & Destruction
//! \{
//! Creates a default-initialized string if zero length.
inline String() noexcept
: _small {} {}
//! Creates a string that takes ownership of the content of the `other` string.
inline String(String&& other) noexcept {
_raw = other._raw;
other._resetInternal();
}
inline ~String() noexcept {
reset();
}
//! Reset the string into a construction state.
ASMJIT_API Error reset() noexcept;
//! \}
//! \name Overloaded Operators
//! \{
inline String& operator=(String&& other) noexcept {
swap(other);
other.reset();
return *this;
}
inline bool operator==(const char* other) const noexcept { return eq(other); }
inline bool operator!=(const char* other) const noexcept { return !eq(other); }
inline bool operator==(const String& other) const noexcept { return eq(other); }
inline bool operator!=(const String& other) const noexcept { return !eq(other); }
//! \}
//! \name Accessors
//! \{
inline bool isExternal() const noexcept { return _type == kTypeExternal; }
inline bool isLargeOrExternal() const noexcept { return _type >= kTypeLarge; }
//! Tests whether the string is empty.
inline bool empty() const noexcept { return size() == 0; }
//! Returns the size of the string.
inline size_t size() const noexcept { return isLargeOrExternal() ? size_t(_large.size) : size_t(_type); }
//! Returns the capacity of the string.
inline size_t capacity() const noexcept { return isLargeOrExternal() ? _large.capacity : size_t(kSSOCapacity); }
//! Returns the data of the string.
inline char* data() noexcept { return isLargeOrExternal() ? _large.data : _small.data; }
//! \overload
inline const char* data() const noexcept { return isLargeOrExternal() ? _large.data : _small.data; }
inline char* start() noexcept { return data(); }
inline const char* start() const noexcept { return data(); }
inline char* end() noexcept { return data() + size(); }
inline const char* end() const noexcept { return data() + size(); }
//! \}
//! \name String Operations
//! \{
//! Swaps the content of this string with `other`.
inline void swap(String& other) noexcept {
std::swap(_raw, other._raw);
}
//! Clears the content of the string.
ASMJIT_API Error clear() noexcept;
ASMJIT_API char* prepare(ModifyOp op, size_t size) noexcept;
ASMJIT_API Error _opString(ModifyOp op, const char* str, size_t size = SIZE_MAX) noexcept;
ASMJIT_API Error _opChar(ModifyOp op, char c) noexcept;
ASMJIT_API Error _opChars(ModifyOp op, char c, size_t n) noexcept;
ASMJIT_API Error _opNumber(ModifyOp op, uint64_t i, uint32_t base = 0, size_t width = 0, StringFormatFlags flags = StringFormatFlags::kNone) noexcept;
ASMJIT_API Error _opHex(ModifyOp op, const void* data, size_t size, char separator = '\0') noexcept;
ASMJIT_API Error _opFormat(ModifyOp op, const char* fmt, ...) noexcept;
ASMJIT_API Error _opVFormat(ModifyOp op, const char* fmt, va_list ap) noexcept;
//! Replaces the current of the string with `data` of the given `size`.
//!
//! Null terminated strings can set `size` to `SIZE_MAX`.
ASMJIT_API Error assign(const char* data, size_t size = SIZE_MAX) noexcept;
//! Replaces the current of the string with `other` string.
inline Error assign(const String& other) noexcept {
return assign(other.data(), other.size());
}
//! Replaces the current of the string by a single `c` character.
inline Error assign(char c) noexcept {
return _opChar(ModifyOp::kAssign, c);
}
//! Replaces the current of the string by a `c` character, repeated `n` times.
inline Error assignChars(char c, size_t n) noexcept {
return _opChars(ModifyOp::kAssign, c, n);
}
//! Replaces the current of the string by a formatted integer `i` (signed).
inline Error assignInt(int64_t i, uint32_t base = 0, size_t width = 0, StringFormatFlags flags = StringFormatFlags::kNone) noexcept {
return _opNumber(ModifyOp::kAssign, uint64_t(i), base, width, flags | StringFormatFlags::kSigned);
}
//! Replaces the current of the string by a formatted integer `i` (unsigned).
inline Error assignUInt(uint64_t i, uint32_t base = 0, size_t width = 0, StringFormatFlags flags = StringFormatFlags::kNone) noexcept {
return _opNumber(ModifyOp::kAssign, i, base, width, flags);
}
//! Replaces the current of the string by the given `data` converted to a HEX string.
inline Error assignHex(const void* data, size_t size, char separator = '\0') noexcept {
return _opHex(ModifyOp::kAssign, data, size, separator);
}
//! Replaces the current of the string by a formatted string `fmt`.
template<typename... Args>
inline Error assignFormat(const char* fmt, Args&&... args) noexcept {
return _opFormat(ModifyOp::kAssign, fmt, std::forward<Args>(args)...);
}
//! Replaces the current of the string by a formatted string `fmt` (va_list version).
inline Error assignVFormat(const char* fmt, va_list ap) noexcept {
return _opVFormat(ModifyOp::kAssign, fmt, ap);
}
//! Appends `str` having the given size `size` to the string.
//!
//! Null terminated strings can set `size` to `SIZE_MAX`.
inline Error append(const char* str, size_t size = SIZE_MAX) noexcept {
return _opString(ModifyOp::kAppend, str, size);
}
//! Appends `other` string to this string.
inline Error append(const String& other) noexcept {
return append(other.data(), other.size());
}
//! Appends a single `c` character.
inline Error append(char c) noexcept {
return _opChar(ModifyOp::kAppend, c);
}
//! Appends `c` character repeated `n` times.
inline Error appendChars(char c, size_t n) noexcept {
return _opChars(ModifyOp::kAppend, c, n);
}
//! Appends a formatted integer `i` (signed).
inline Error appendInt(int64_t i, uint32_t base = 0, size_t width = 0, StringFormatFlags flags = StringFormatFlags::kNone) noexcept {
return _opNumber(ModifyOp::kAppend, uint64_t(i), base, width, flags | StringFormatFlags::kSigned);
}
//! Appends a formatted integer `i` (unsigned).
inline Error appendUInt(uint64_t i, uint32_t base = 0, size_t width = 0, StringFormatFlags flags = StringFormatFlags::kNone) noexcept {
return _opNumber(ModifyOp::kAppend, i, base, width, flags);
}
//! Appends the given `data` converted to a HEX string.
inline Error appendHex(const void* data, size_t size, char separator = '\0') noexcept {
return _opHex(ModifyOp::kAppend, data, size, separator);
}
//! Appends a formatted string `fmt` with `args`.
template<typename... Args>
inline Error appendFormat(const char* fmt, Args&&... args) noexcept {
return _opFormat(ModifyOp::kAppend, fmt, std::forward<Args>(args)...);
}
//! Appends a formatted string `fmt` (va_list version).
inline Error appendVFormat(const char* fmt, va_list ap) noexcept {
return _opVFormat(ModifyOp::kAppend, fmt, ap);
}
ASMJIT_API Error padEnd(size_t n, char c = ' ') noexcept;
//! Truncate the string length into `newSize`.
ASMJIT_API Error truncate(size_t newSize) noexcept;
ASMJIT_API bool eq(const char* other, size_t size = SIZE_MAX) const noexcept;
inline bool eq(const String& other) const noexcept { return eq(other.data(), other.size()); }
//! \}
//! \name Internal Functions
//! \{
//! Resets string to embedded and makes it empty (zero length, zero first char)
//!
//! \note This is always called internally after an external buffer was released as it zeroes all bytes
//! used by String's embedded storage.
inline void _resetInternal() noexcept {
for (size_t i = 0; i < ASMJIT_ARRAY_SIZE(_raw.uptr); i++)
_raw.uptr[i] = 0;
}
inline void _setSize(size_t newSize) noexcept {
if (isLargeOrExternal())
_large.size = newSize;
else
_small.type = uint8_t(newSize);
}
//! \}
};
//! Temporary string builder, has statically allocated `N` bytes.
template<size_t N>
class StringTmp : public String {
public:
ASMJIT_NONCOPYABLE(StringTmp)
//! Embedded data.
char _embeddedData[Support::alignUp(N + 1, sizeof(size_t))];
//! \name Construction & Destruction
//! \{
inline StringTmp() noexcept {
_resetToTemporary();
}
inline void _resetToTemporary() noexcept {
_large.type = kTypeExternal;
_large.capacity = ASMJIT_ARRAY_SIZE(_embeddedData) - 1;
_large.data = _embeddedData;
_embeddedData[0] = '\0';
}
//! \}
};
//! \}
ASMJIT_END_NAMESPACE
#endif // ASMJIT_CORE_STRING_H_INCLUDED
libraries/asmjit/asmjit/core/support.cpp 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
#include "../core/support.h"
ASMJIT_BEGIN_NAMESPACE
// Support - Tests
// ===============
#if defined(ASMJIT_TEST)
template<typename T>
static void testArrays(const T* a, const T* b, size_t size) noexcept {
for (size_t i = 0; i < size; i++)
EXPECT(a[i] == b[i], "Mismatch at %u", unsigned(i));
}
static void testAlignment() noexcept {
INFO("Support::isAligned()");
EXPECT(Support::isAligned<size_t>(0xFFFF, 4) == false);
EXPECT(Support::isAligned<size_t>(0xFFF4, 4) == true);
EXPECT(Support::isAligned<size_t>(0xFFF8, 8) == true);
EXPECT(Support::isAligned<size_t>(0xFFF0, 16) == true);
INFO("Support::alignUp()");
EXPECT(Support::alignUp<size_t>(0xFFFF, 4) == 0x10000);
EXPECT(Support::alignUp<size_t>(0xFFF4, 4) == 0x0FFF4);
EXPECT(Support::alignUp<size_t>(0xFFF8, 8) == 0x0FFF8);
EXPECT(Support::alignUp<size_t>(0xFFF0, 16) == 0x0FFF0);
EXPECT(Support::alignUp<size_t>(0xFFF0, 32) == 0x10000);
INFO("Support::alignUpDiff()");
EXPECT(Support::alignUpDiff<size_t>(0xFFFF, 4) == 1);
EXPECT(Support::alignUpDiff<size_t>(0xFFF4, 4) == 0);
EXPECT(Support::alignUpDiff<size_t>(0xFFF8, 8) == 0);
EXPECT(Support::alignUpDiff<size_t>(0xFFF0, 16) == 0);
EXPECT(Support::alignUpDiff<size_t>(0xFFF0, 32) == 16);
INFO("Support::alignUpPowerOf2()");
EXPECT(Support::alignUpPowerOf2<size_t>(0x0000) == 0x00000);
EXPECT(Support::alignUpPowerOf2<size_t>(0xFFFF) == 0x10000);
EXPECT(Support::alignUpPowerOf2<size_t>(0xF123) == 0x10000);
EXPECT(Support::alignUpPowerOf2<size_t>(0x0F00) == 0x01000);
EXPECT(Support::alignUpPowerOf2<size_t>(0x0100) == 0x00100);
EXPECT(Support::alignUpPowerOf2<size_t>(0x1001) == 0x02000);
}
static void testBitUtils() noexcept {
uint32_t i;
INFO("Support::shl() / shr()");
EXPECT(Support::shl(int32_t(0x00001111), 16) == int32_t(0x11110000u));
EXPECT(Support::shl(uint32_t(0x00001111), 16) == uint32_t(0x11110000u));
EXPECT(Support::shr(int32_t(0x11110000u), 16) == int32_t(0x00001111u));
EXPECT(Support::shr(uint32_t(0x11110000u), 16) == uint32_t(0x00001111u));
EXPECT(Support::sar(int32_t(0xFFFF0000u), 16) == int32_t(0xFFFFFFFFu));
EXPECT(Support::sar(uint32_t(0xFFFF0000u), 16) == uint32_t(0xFFFFFFFFu));
INFO("Support::blsi()");
for (i = 0; i < 32; i++) EXPECT(Support::blsi(uint32_t(1) << i) == uint32_t(1) << i);
for (i = 0; i < 31; i++) EXPECT(Support::blsi(uint32_t(3) << i) == uint32_t(1) << i);
for (i = 0; i < 64; i++) EXPECT(Support::blsi(uint64_t(1) << i) == uint64_t(1) << i);
for (i = 0; i < 63; i++) EXPECT(Support::blsi(uint64_t(3) << i) == uint64_t(1) << i);
INFO("Support::ctz()");
for (i = 0; i < 32; i++) EXPECT(Support::Internal::clzFallback(uint32_t(1) << i) == 31 - i);
for (i = 0; i < 64; i++) EXPECT(Support::Internal::clzFallback(uint64_t(1) << i) == 63 - i);
for (i = 0; i < 32; i++) EXPECT(Support::Internal::ctzFallback(uint32_t(1) << i) == i);
for (i = 0; i < 64; i++) EXPECT(Support::Internal::ctzFallback(uint64_t(1) << i) == i);
for (i = 0; i < 32; i++) EXPECT(Support::clz(uint32_t(1) << i) == 31 - i);
for (i = 0; i < 64; i++) EXPECT(Support::clz(uint64_t(1) << i) == 63 - i);
for (i = 0; i < 32; i++) EXPECT(Support::ctz(uint32_t(1) << i) == i);
for (i = 0; i < 64; i++) EXPECT(Support::ctz(uint64_t(1) << i) == i);
INFO("Support::bitMask()");
EXPECT(Support::bitMask(0, 1, 7) == 0x83u);
for (i = 0; i < 32; i++)
EXPECT(Support::bitMask(i) == (1u << i));
INFO("Support::bitTest()");
for (i = 0; i < 32; i++) {
EXPECT(Support::bitTest((1 << i), i) == true, "Support::bitTest(%X, %u) should return true", (1 << i), i);
}
INFO("Support::lsbMask<uint32_t>()");
for (i = 0; i < 32; i++) {
uint32_t expectedBits = 0;
for (uint32_t b = 0; b < i; b++)
expectedBits |= uint32_t(1) << b;
EXPECT(Support::lsbMask<uint32_t>(i) == expectedBits);
}
INFO("Support::lsbMask<uint64_t>()");
for (i = 0; i < 64; i++) {
uint64_t expectedBits = 0;
for (uint32_t b = 0; b < i; b++)
expectedBits |= uint64_t(1) << b;
EXPECT(Support::lsbMask<uint64_t>(i) == expectedBits);
}
INFO("Support::popcnt()");
for (i = 0; i < 32; i++) EXPECT(Support::popcnt((uint32_t(1) << i)) == 1);
for (i = 0; i < 64; i++) EXPECT(Support::popcnt((uint64_t(1) << i)) == 1);
EXPECT(Support::popcnt(0x000000F0) == 4);
EXPECT(Support::popcnt(0x10101010) == 4);
EXPECT(Support::popcnt(0xFF000000) == 8);
EXPECT(Support::popcnt(0xFFFFFFF7) == 31);
EXPECT(Support::popcnt(0x7FFFFFFF) == 31);
INFO("Support::isPowerOf2()");
for (i = 0; i < 64; i++) {
EXPECT(Support::isPowerOf2(uint64_t(1) << i) == true);
EXPECT(Support::isPowerOf2((uint64_t(1) << i) ^ 0x001101) == false);
}
}
static void testIntUtils() noexcept {
INFO("Support::byteswap()");
EXPECT(Support::byteswap16(int32_t(0x0102)) == int32_t(0x0201));
EXPECT(Support::byteswap32(int32_t(0x01020304)) == int32_t(0x04030201));
EXPECT(Support::byteswap32(uint32_t(0x01020304)) == uint32_t(0x04030201));
EXPECT(Support::byteswap64(uint64_t(0x0102030405060708)) == uint64_t(0x0807060504030201));
INFO("Support::bytepack()");
union BytePackData {
uint8_t bytes[4];
uint32_t u32;
} bpdata;
bpdata.u32 = Support::bytepack32_4x8(0x00, 0x11, 0x22, 0x33);
EXPECT(bpdata.bytes[0] == 0x00);
EXPECT(bpdata.bytes[1] == 0x11);
EXPECT(bpdata.bytes[2] == 0x22);
EXPECT(bpdata.bytes[3] == 0x33);
INFO("Support::isBetween()");
EXPECT(Support::isBetween<int>(10 , 10, 20) == true);
EXPECT(Support::isBetween<int>(11 , 10, 20) == true);
EXPECT(Support::isBetween<int>(20 , 10, 20) == true);
EXPECT(Support::isBetween<int>(9 , 10, 20) == false);
EXPECT(Support::isBetween<int>(21 , 10, 20) == false);
EXPECT(Support::isBetween<int>(101, 10, 20) == false);
INFO("Support::isInt8()");
EXPECT(Support::isInt8(-128) == true);
EXPECT(Support::isInt8( 127) == true);
EXPECT(Support::isInt8(-129) == false);
EXPECT(Support::isInt8( 128) == false);
INFO("Support::isInt16()");
EXPECT(Support::isInt16(-32768) == true);
EXPECT(Support::isInt16( 32767) == true);
EXPECT(Support::isInt16(-32769) == false);
EXPECT(Support::isInt16( 32768) == false);
INFO("Support::isInt32()");
EXPECT(Support::isInt32( 2147483647 ) == true);
EXPECT(Support::isInt32(-2147483647 - 1) == true);
EXPECT(Support::isInt32(uint64_t(2147483648u)) == false);
EXPECT(Support::isInt32(uint64_t(0xFFFFFFFFu)) == false);
EXPECT(Support::isInt32(uint64_t(0xFFFFFFFFu) + 1) == false);
INFO("Support::isUInt8()");
EXPECT(Support::isUInt8(0) == true);
EXPECT(Support::isUInt8(255) == true);
EXPECT(Support::isUInt8(256) == false);
EXPECT(Support::isUInt8(-1) == false);
INFO("Support::isUInt12()");
EXPECT(Support::isUInt12(0) == true);
EXPECT(Support::isUInt12(4095) == true);
EXPECT(Support::isUInt12(4096) == false);
EXPECT(Support::isUInt12(-1) == false);
INFO("Support::isUInt16()");
EXPECT(Support::isUInt16(0) == true);
EXPECT(Support::isUInt16(65535) == true);
EXPECT(Support::isUInt16(65536) == false);
EXPECT(Support::isUInt16(-1) == false);
INFO("Support::isUInt32()");
EXPECT(Support::isUInt32(uint64_t(0xFFFFFFFF)) == true);
EXPECT(Support::isUInt32(uint64_t(0xFFFFFFFF) + 1) == false);
EXPECT(Support::isUInt32(-1) == false);
}
static void testReadWrite() noexcept {
INFO("Support::readX() / writeX()");
uint8_t arr[32] = { 0 };
Support::writeU16uBE(arr + 1, 0x0102u);
Support::writeU16uBE(arr + 3, 0x0304u);
EXPECT(Support::readU32uBE(arr + 1) == 0x01020304u);
EXPECT(Support::readU32uLE(arr + 1) == 0x04030201u);
EXPECT(Support::readU32uBE(arr + 2) == 0x02030400u);
EXPECT(Support::readU32uLE(arr + 2) == 0x00040302u);
Support::writeU32uLE(arr + 5, 0x05060708u);
EXPECT(Support::readU64uBE(arr + 1) == 0x0102030408070605u);
EXPECT(Support::readU64uLE(arr + 1) == 0x0506070804030201u);
Support::writeU64uLE(arr + 7, 0x1122334455667788u);
EXPECT(Support::readU32uBE(arr + 8) == 0x77665544u);
}
static void testBitVector() noexcept {
INFO("Support::bitVectorOp");
{
uint32_t vec[3] = { 0 };
Support::bitVectorFill(vec, 1, 64);
EXPECT(vec[0] == 0xFFFFFFFEu);
EXPECT(vec[1] == 0xFFFFFFFFu);
EXPECT(vec[2] == 0x00000001u);
Support::bitVectorClear(vec, 1, 1);
EXPECT(vec[0] == 0xFFFFFFFCu);
EXPECT(vec[1] == 0xFFFFFFFFu);
EXPECT(vec[2] == 0x00000001u);
Support::bitVectorFill(vec, 0, 32);
EXPECT(vec[0] == 0xFFFFFFFFu);
EXPECT(vec[1] == 0xFFFFFFFFu);
EXPECT(vec[2] == 0x00000001u);
Support::bitVectorClear(vec, 0, 32);
EXPECT(vec[0] == 0x00000000u);
EXPECT(vec[1] == 0xFFFFFFFFu);
EXPECT(vec[2] == 0x00000001u);
Support::bitVectorFill(vec, 1, 30);
EXPECT(vec[0] == 0x7FFFFFFEu);
EXPECT(vec[1] == 0xFFFFFFFFu);
EXPECT(vec[2] == 0x00000001u);
Support::bitVectorClear(vec, 1, 95);
EXPECT(vec[0] == 0x00000000u);
EXPECT(vec[1] == 0x00000000u);
EXPECT(vec[2] == 0x00000000u);
Support::bitVectorFill(vec, 32, 64);
EXPECT(vec[0] == 0x00000000u);
EXPECT(vec[1] == 0xFFFFFFFFu);
EXPECT(vec[2] == 0xFFFFFFFFu);
Support::bitVectorSetBit(vec, 1, true);
EXPECT(vec[0] == 0x00000002u);
EXPECT(vec[1] == 0xFFFFFFFFu);
EXPECT(vec[2] == 0xFFFFFFFFu);
Support::bitVectorSetBit(vec, 95, false);
EXPECT(vec[0] == 0x00000002u);
EXPECT(vec[1] == 0xFFFFFFFFu);
EXPECT(vec[2] == 0x7FFFFFFFu);
Support::bitVectorClear(vec, 33, 32);
EXPECT(vec[0] == 0x00000002u);
EXPECT(vec[1] == 0x00000001u);
EXPECT(vec[2] == 0x7FFFFFFEu);
}
INFO("Support::bitVectorIndexOf");
{
uint32_t vec1[1] = { 0x80000000 };
EXPECT(Support::bitVectorIndexOf(vec1, 0, true) == 31);
EXPECT(Support::bitVectorIndexOf(vec1, 1, true) == 31);
EXPECT(Support::bitVectorIndexOf(vec1, 31, true) == 31);
uint32_t vec2[2] = { 0x00000000, 0x80000000 };
EXPECT(Support::bitVectorIndexOf(vec2, 0, true) == 63);
EXPECT(Support::bitVectorIndexOf(vec2, 1, true) == 63);
EXPECT(Support::bitVectorIndexOf(vec2, 31, true) == 63);
EXPECT(Support::bitVectorIndexOf(vec2, 32, true) == 63);
EXPECT(Support::bitVectorIndexOf(vec2, 33, true) == 63);
EXPECT(Support::bitVectorIndexOf(vec2, 63, true) == 63);
uint32_t vec3[3] = { 0x00000001, 0x00000000, 0x80000000 };
EXPECT(Support::bitVectorIndexOf(vec3, 0, true) == 0);
EXPECT(Support::bitVectorIndexOf(vec3, 1, true) == 95);
EXPECT(Support::bitVectorIndexOf(vec3, 2, true) == 95);
EXPECT(Support::bitVectorIndexOf(vec3, 31, true) == 95);
EXPECT(Support::bitVectorIndexOf(vec3, 32, true) == 95);
EXPECT(Support::bitVectorIndexOf(vec3, 63, true) == 95);
EXPECT(Support::bitVectorIndexOf(vec3, 64, true) == 95);
EXPECT(Support::bitVectorIndexOf(vec3, 95, true) == 95);
uint32_t vec4[3] = { ~vec3[0], ~vec3[1], ~vec3[2] };
EXPECT(Support::bitVectorIndexOf(vec4, 0, false) == 0);
EXPECT(Support::bitVectorIndexOf(vec4, 1, false) == 95);
EXPECT(Support::bitVectorIndexOf(vec4, 2, false) == 95);
EXPECT(Support::bitVectorIndexOf(vec4, 31, false) == 95);
EXPECT(Support::bitVectorIndexOf(vec4, 32, false) == 95);
EXPECT(Support::bitVectorIndexOf(vec4, 63, false) == 95);
EXPECT(Support::bitVectorIndexOf(vec4, 64, false) == 95);
EXPECT(Support::bitVectorIndexOf(vec4, 95, false) == 95);
}
INFO("Support::BitWordIterator<uint32_t>");
{
Support::BitWordIterator<uint32_t> it(0x80000F01u);
EXPECT(it.hasNext());
EXPECT(it.next() == 0);
EXPECT(it.hasNext());
EXPECT(it.next() == 8);
EXPECT(it.hasNext());
EXPECT(it.next() == 9);
EXPECT(it.hasNext());
EXPECT(it.next() == 10);
EXPECT(it.hasNext());
EXPECT(it.next() == 11);
EXPECT(it.hasNext());
EXPECT(it.next() == 31);
EXPECT(!it.hasNext());
// No bits set.
it.init(0x00000000u);
ASMJIT_ASSERT(!it.hasNext());
// Only first bit set.
it.init(0x00000001u);
EXPECT(it.hasNext());
EXPECT(it.next() == 0);
ASMJIT_ASSERT(!it.hasNext());
// Only last bit set (special case).
it.init(0x80000000u);
ASMJIT_ASSERT(it.hasNext());
ASMJIT_ASSERT(it.next() == 31);
ASMJIT_ASSERT(!it.hasNext());
}
INFO("Support::BitWordIterator<uint64_t>");
{
Support::BitWordIterator<uint64_t> it(uint64_t(1) << 63);
ASMJIT_ASSERT(it.hasNext());
ASMJIT_ASSERT(it.next() == 63);
ASMJIT_ASSERT(!it.hasNext());
}
INFO("Support::BitVectorIterator<uint32_t>");
{
// Border cases.
static const uint32_t bitsNone[] = { 0xFFFFFFFFu };
Support::BitVectorIterator<uint32_t> it(bitsNone, 0);
EXPECT(!it.hasNext());
it.init(bitsNone, 0, 1);
EXPECT(!it.hasNext());
it.init(bitsNone, 0, 128);
EXPECT(!it.hasNext());
static const uint32_t bits1[] = { 0x80000008u, 0x80000001u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00003000u };
it.init(bits1, ASMJIT_ARRAY_SIZE(bits1));
EXPECT(it.hasNext());
EXPECT(it.next() == 3);
EXPECT(it.hasNext());
EXPECT(it.next() == 31);
EXPECT(it.hasNext());
EXPECT(it.next() == 32);
EXPECT(it.hasNext());
EXPECT(it.next() == 63);
EXPECT(it.hasNext());
EXPECT(it.next() == 127);
EXPECT(it.hasNext());
EXPECT(it.next() == 204);
EXPECT(it.hasNext());
EXPECT(it.next() == 205);
EXPECT(!it.hasNext());
it.init(bits1, ASMJIT_ARRAY_SIZE(bits1), 4);
EXPECT(it.hasNext());
EXPECT(it.next() == 31);
it.init(bits1, ASMJIT_ARRAY_SIZE(bits1), 64);
EXPECT(it.hasNext());
EXPECT(it.next() == 127);
it.init(bits1, ASMJIT_ARRAY_SIZE(bits1), 127);
EXPECT(it.hasNext());
EXPECT(it.next() == 127);
static const uint32_t bits2[] = { 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u };
it.init(bits2, ASMJIT_ARRAY_SIZE(bits2));
EXPECT(it.hasNext());
EXPECT(it.next() == 31);
EXPECT(it.hasNext());
EXPECT(it.next() == 63);
EXPECT(it.hasNext());
EXPECT(it.next() == 127);
EXPECT(!it.hasNext());
static const uint32_t bits3[] = { 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u };
it.init(bits3, ASMJIT_ARRAY_SIZE(bits3));
EXPECT(!it.hasNext());
static const uint32_t bits4[] = { 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u };
it.init(bits4, ASMJIT_ARRAY_SIZE(bits4));
EXPECT(it.hasNext());
EXPECT(it.next() == 127);
EXPECT(!it.hasNext());
}
INFO("Support::BitVectorIterator<uint64_t>");
{
static const uint64_t bits1[] = { 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u };
Support::BitVectorIterator<uint64_t> it(bits1, ASMJIT_ARRAY_SIZE(bits1));
EXPECT(it.hasNext());
EXPECT(it.next() == 31);
EXPECT(it.hasNext());
EXPECT(it.next() == 95);
EXPECT(it.hasNext());
EXPECT(it.next() == 223);
EXPECT(!it.hasNext());
static const uint64_t bits2[] = { 0x8000000000000000u, 0, 0, 0 };
it.init(bits2, ASMJIT_ARRAY_SIZE(bits2));
EXPECT(it.hasNext());
EXPECT(it.next() == 63);
EXPECT(!it.hasNext());
}
}
static void testSorting() noexcept {
INFO("Support::qSort() - Testing qsort and isort of predefined arrays");
{
constexpr size_t kArraySize = 11;
int ref_[kArraySize] = { -4, -2, -1, 0, 1, 9, 12, 13, 14, 19, 22 };
int arr1[kArraySize] = { 0, 1, -1, 19, 22, 14, -4, 9, 12, 13, -2 };
int arr2[kArraySize];
memcpy(arr2, arr1, kArraySize * sizeof(int));
Support::iSort(arr1, kArraySize);
Support::qSort(arr2, kArraySize);
testArrays(arr1, ref_, kArraySize);
testArrays(arr2, ref_, kArraySize);
}
INFO("Support::qSort() - Testing qsort and isort of artificial arrays");
{
constexpr size_t kArraySize = 200;
int arr1[kArraySize];
int arr2[kArraySize];
int ref_[kArraySize];
for (size_t size = 2; size < kArraySize; size++) {
for (size_t i = 0; i < size; i++) {
arr1[i] = int(size - 1 - i);
arr2[i] = int(size - 1 - i);
ref_[i] = int(i);
}
Support::iSort(arr1, size);
Support::qSort(arr2, size);
testArrays(arr1, ref_, size);
testArrays(arr2, ref_, size);
}
}
INFO("Support::qSort() - Testing qsort and isort with an unstable compare function");
{
constexpr size_t kArraySize = 5;
float arr1[kArraySize] = { 1.0f, 0.0f, 3.0f, -1.0f, std::numeric_limits<float>::quiet_NaN() };
float arr2[kArraySize] = { };
memcpy(arr2, arr1, kArraySize * sizeof(float));
// We don't test as it's undefined where the NaN would be.
Support::iSort(arr1, kArraySize);
Support::qSort(arr2, kArraySize);
}
}
UNIT(support) {
testAlignment();
testBitUtils();
testIntUtils();
testReadWrite();
testBitVector();
testSorting();
}
#endif
ASMJIT_END_NAMESPACE
libraries/asmjit/asmjit/core/support.h 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#ifndef ASMJIT_CORE_SUPPORT_H_INCLUDED
#define ASMJIT_CORE_SUPPORT_H_INCLUDED
#include "../core/globals.h"
#if defined(_MSC_VER)
#include <intrin.h>
#endif
ASMJIT_BEGIN_NAMESPACE
//! \addtogroup asmjit_utilities
//! \{
//! Contains support classes and functions that may be used by AsmJit source and header files. Anything defined
//! here is considered internal and should not be used outside of AsmJit and related projects like AsmTK.
namespace Support {
// Support - Basic Traits
// ======================
#if ASMJIT_ARCH_X86
typedef uint8_t FastUInt8;
#else
typedef uint32_t FastUInt8;
#endif
//! \cond INTERNAL
namespace Internal {
template<typename T, size_t Alignment>
struct AliasedUInt {};
template<> struct AliasedUInt<uint16_t, 2> { typedef uint16_t ASMJIT_MAY_ALIAS T; };
template<> struct AliasedUInt<uint32_t, 4> { typedef uint32_t ASMJIT_MAY_ALIAS T; };
template<> struct AliasedUInt<uint64_t, 8> { typedef uint64_t ASMJIT_MAY_ALIAS T; };
template<> struct AliasedUInt<uint16_t, 1> { typedef uint16_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); };
template<> struct AliasedUInt<uint32_t, 1> { typedef uint32_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); };
template<> struct AliasedUInt<uint32_t, 2> { typedef uint32_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 2); };
template<> struct AliasedUInt<uint64_t, 1> { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 1); };
template<> struct AliasedUInt<uint64_t, 2> { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 2); };
template<> struct AliasedUInt<uint64_t, 4> { typedef uint64_t ASMJIT_MAY_ALIAS ASMJIT_ALIGN_TYPE(T, 4); };
// StdInt - Make an int-type by size (signed or unsigned) that is the
// same as types defined by <stdint.h>.
// Int32Or64 - Make an int-type that has at least 32 bits: [u]int[32|64]_t.
template<size_t Size, unsigned Unsigned>
struct StdInt {}; // Fail if not specialized.
template<> struct StdInt<1, 0> { typedef int8_t Type; };
template<> struct StdInt<1, 1> { typedef uint8_t Type; };
template<> struct StdInt<2, 0> { typedef int16_t Type; };
template<> struct StdInt<2, 1> { typedef uint16_t Type; };
template<> struct StdInt<4, 0> { typedef int32_t Type; };
template<> struct StdInt<4, 1> { typedef uint32_t Type; };
template<> struct StdInt<8, 0> { typedef int64_t Type; };
template<> struct StdInt<8, 1> { typedef uint64_t Type; };
template<typename T, int Unsigned = std::is_unsigned<T>::value>
struct Int32Or64 : public StdInt<sizeof(T) <= 4 ? size_t(4) : sizeof(T), Unsigned> {};
}
//! \endcond
template<typename T>
static constexpr bool isUnsigned() noexcept { return std::is_unsigned<T>::value; }
//! Casts an integer `x` to either `int32_t` or `int64_t` depending on `T`.
template<typename T>
static constexpr typename Internal::Int32Or64<T, 0>::Type asInt(const T& x) noexcept {
return (typename Internal::Int32Or64<T, 0>::Type)x;
}
//! Casts an integer `x` to either `uint32_t` or `uint64_t` depending on `T`.
template<typename T>
static constexpr typename Internal::Int32Or64<T, 1>::Type asUInt(const T& x) noexcept {
return (typename Internal::Int32Or64<T, 1>::Type)x;
}
//! Casts an integer `x` to either `int32_t`, uint32_t`, `int64_t`, or `uint64_t` depending on `T`.
template<typename T>
static constexpr typename Internal::Int32Or64<T>::Type asNormalized(const T& x) noexcept {
return (typename Internal::Int32Or64<T>::Type)x;
}
//! Casts an integer `x` to the same type as defined by `<stdint.h>`.
template<typename T>
static constexpr typename Internal::StdInt<sizeof(T), isUnsigned<T>()>::Type asStdInt(const T& x) noexcept {
return (typename Internal::StdInt<sizeof(T), isUnsigned<T>()>::Type)x;
}
//! A helper class that can be used to iterate over enum values.
template<typename T, T from = (T)0, T to = T::kMaxValue>
struct EnumValues {
typedef typename std::underlying_type<T>::type ValueType;
struct Iterator {
ValueType value;
inline T operator*() const { return (T)value; }
inline void operator++() { ++value; }
inline bool operator==(const Iterator& other) const noexcept { return value == other.value; }
inline bool operator!=(const Iterator& other) const noexcept { return value != other.value; }
};
inline Iterator begin() const noexcept { return Iterator{ValueType(from)}; }
inline Iterator end() const noexcept { return Iterator{ValueType(to) + 1}; }
};
// Support - BitCast
// =================
//! \cond
namespace Internal {
template<typename DstT, typename SrcT>
union BitCastUnion {
inline BitCastUnion(SrcT src) noexcept : src(src) {}
SrcT src;
DstT dst;
};
}
//! \endcond
//! Bit-casts from `Src` type to `Dst` type.
//!
//! Useful to bit-cast between integers and floating points.
template<typename Dst, typename Src>
static inline Dst bitCast(const Src& x) noexcept { return Internal::BitCastUnion<Dst, Src>(x).dst; }
// Support - BitOps
// ================
//! Storage used to store a pack of bits (should by compatible with a machine word).
typedef Internal::StdInt<sizeof(uintptr_t), 1>::Type BitWord;
template<typename T>
static constexpr uint32_t bitSizeOf() noexcept { return uint32_t(sizeof(T) * 8u); }
//! Number of bits stored in a single `BitWord`.
static constexpr uint32_t kBitWordSizeInBits = bitSizeOf<BitWord>();
//! Returns `0 - x` in a safe way (no undefined behavior), works for unsigned numbers as well.
template<typename T>
static constexpr T neg(const T& x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return T(U(0) - U(x));
}
template<typename T>
static constexpr T allOnes() noexcept { return neg<T>(T(1)); }
//! Returns `x << y` (shift left logical) by explicitly casting `x` to an unsigned type and back.
template<typename X, typename Y>
static constexpr X shl(const X& x, const Y& y) noexcept {
typedef typename std::make_unsigned<X>::type U;
return X(U(x) << y);
}
//! Returns `x >> y` (shift right logical) by explicitly casting `x` to an unsigned type and back.
template<typename X, typename Y>
static constexpr X shr(const X& x, const Y& y) noexcept {
typedef typename std::make_unsigned<X>::type U;
return X(U(x) >> y);
}
//! Returns `x >> y` (shift right arithmetic) by explicitly casting `x` to a signed type and back.
template<typename X, typename Y>
static constexpr X sar(const X& x, const Y& y) noexcept {
typedef typename std::make_signed<X>::type S;
return X(S(x) >> y);
}
template<typename X, typename Y>
static constexpr X ror(const X& x, const Y& y) noexcept {
typedef typename std::make_unsigned<X>::type U;
return X((U(x) >> y) | (U(x) << (bitSizeOf<U>() - y)));
}
//! Returns `x | (x >> y)` - helper used by some bit manipulation helpers.
template<typename X, typename Y>
static constexpr X or_shr(const X& x, const Y& y) noexcept { return X(x | shr(x, y)); }
//! Returns `x & -x` - extracts lowest set isolated bit (like BLSI instruction).
template<typename T>
static constexpr T blsi(T x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return T(U(x) & neg(U(x)));
}
//! Tests whether the given value `x` has `n`th bit set.
template<typename T, typename IndexT>
static constexpr bool bitTest(T x, IndexT n) noexcept {
typedef typename std::make_unsigned<T>::type U;
return (U(x) & (U(1) << asStdInt(n))) != 0;
}
// Tests whether the given `value` is a consecutive mask of bits that starts at
// the least significant bit.
template<typename T>
static inline constexpr bool isLsbMask(const T& value) {
typedef typename std::make_unsigned<T>::type U;
return value && ((U(value) + 1u) & U(value)) == 0;
}
// Tests whether the given value contains at least one bit or whether it's a
// bit-mask of consecutive bits.
//
// This function is similar to \ref isLsbMask(), but the mask doesn't have to
// start at a least significant bit.
template<typename T>
static inline constexpr bool isConsecutiveMask(const T& value) {
typedef typename std::make_unsigned<T>::type U;
return value && isLsbMask((U(value) - 1u) | U(value));
}
//! Generates a trailing bit-mask that has `n` least significant (trailing) bits set.
template<typename T, typename CountT>
static constexpr T lsbMask(const CountT& n) noexcept {
typedef typename std::make_unsigned<T>::type U;
return (sizeof(U) < sizeof(uintptr_t))
// Prevent undefined behavior by using a larger type than T.
? T(U((uintptr_t(1) << n) - uintptr_t(1)))
// Prevent undefined behavior by checking `n` before shift.
: n ? T(shr(allOnes<T>(), bitSizeOf<T>() - size_t(n))) : T(0);
}
//! Generats a leading bit-mask that has `n` most significant (leading) bits set.
template<typename T, typename CountT>
static constexpr T msbMask(const CountT& n) noexcept {
typedef typename std::make_unsigned<T>::type U;
return (sizeof(U) < sizeof(uintptr_t))
// Prevent undefined behavior by using a larger type than T.
? T(allOnes<uintptr_t>() >> (bitSizeOf<uintptr_t>() - n))
// Prevent undefined behavior by performing `n & (nBits - 1)` so it's always within the range.
: T(sar(U(n != 0) << (bitSizeOf<U>() - 1), n ? uint32_t(n - 1) : uint32_t(0)));
}
//! Returns a bit-mask that has `x` bit set.
template<typename Index>
static constexpr uint32_t bitMask(const Index& x) noexcept { return (1u << asUInt(x)); }
//! Returns a bit-mask that has `x` bit set (multiple arguments).
template<typename Index, typename... Args>
static constexpr uint32_t bitMask(const Index& x, Args... args) noexcept { return bitMask(x) | bitMask(args...); }
//! Converts a boolean value `b` to zero or full mask (all bits set).
template<typename DstT, typename SrcT>
static constexpr DstT bitMaskFromBool(SrcT b) noexcept {
typedef typename std::make_unsigned<DstT>::type U;
return DstT(U(0) - U(b));
}
//! Tests whether `a & b` is non-zero.
template<typename A, typename B>
static inline constexpr bool test(A a, B b) noexcept { return (asUInt(a) & asUInt(b)) != 0; }
//! \cond
namespace Internal {
// Fills all trailing bits right from the first most significant bit set.
static constexpr uint8_t fillTrailingBitsImpl(uint8_t x) noexcept { return or_shr(or_shr(or_shr(x, 1), 2), 4); }
// Fills all trailing bits right from the first most significant bit set.
static constexpr uint16_t fillTrailingBitsImpl(uint16_t x) noexcept { return or_shr(or_shr(or_shr(or_shr(x, 1), 2), 4), 8); }
// Fills all trailing bits right from the first most significant bit set.
static constexpr uint32_t fillTrailingBitsImpl(uint32_t x) noexcept { return or_shr(or_shr(or_shr(or_shr(or_shr(x, 1), 2), 4), 8), 16); }
// Fills all trailing bits right from the first most significant bit set.
static constexpr uint64_t fillTrailingBitsImpl(uint64_t x) noexcept { return or_shr(or_shr(or_shr(or_shr(or_shr(or_shr(x, 1), 2), 4), 8), 16), 32); }
}
//! \endcond
// Fills all trailing bits right from the first most significant bit set.
template<typename T>
static constexpr T fillTrailingBits(const T& x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return T(Internal::fillTrailingBitsImpl(U(x)));
}
// Support - Count Leading/Trailing Zeros
// ======================================
//! \cond
namespace Internal {
namespace {
template<typename T>
struct BitScanData { T x; uint32_t n; };
template<typename T, uint32_t N>
struct BitScanCalc {
static constexpr BitScanData<T> advanceLeft(const BitScanData<T>& data, uint32_t n) noexcept {
return BitScanData<T> { data.x << n, data.n + n };
}
static constexpr BitScanData<T> advanceRight(const BitScanData<T>& data, uint32_t n) noexcept {
return BitScanData<T> { data.x >> n, data.n + n };
}
static constexpr BitScanData<T> clz(const BitScanData<T>& data) noexcept {
return BitScanCalc<T, N / 2>::clz(advanceLeft(data, data.x & (allOnes<T>() << (bitSizeOf<T>() - N)) ? uint32_t(0) : N));
}
static constexpr BitScanData<T> ctz(const BitScanData<T>& data) noexcept {
return BitScanCalc<T, N / 2>::ctz(advanceRight(data, data.x & (allOnes<T>() >> (bitSizeOf<T>() - N)) ? uint32_t(0) : N));
}
};
template<typename T>
struct BitScanCalc<T, 0> {
static constexpr BitScanData<T> clz(const BitScanData<T>& ctx) noexcept {
return BitScanData<T> { 0, ctx.n - uint32_t(ctx.x >> (bitSizeOf<T>() - 1)) };
}
static constexpr BitScanData<T> ctz(const BitScanData<T>& ctx) noexcept {
return BitScanData<T> { 0, ctx.n - uint32_t(ctx.x & 0x1) };
}
};
template<typename T>
constexpr uint32_t clzFallback(const T& x) noexcept {
return BitScanCalc<T, bitSizeOf<T>() / 2u>::clz(BitScanData<T>{x, 1}).n;
}
template<typename T>
constexpr uint32_t ctzFallback(const T& x) noexcept {
return BitScanCalc<T, bitSizeOf<T>() / 2u>::ctz(BitScanData<T>{x, 1}).n;
}
template<typename T> inline uint32_t clzImpl(const T& x) noexcept { return clzFallback(asUInt(x)); }
template<typename T> inline uint32_t ctzImpl(const T& x) noexcept { return ctzFallback(asUInt(x)); }
#if !defined(ASMJIT_NO_INTRINSICS)
# if defined(__GNUC__)
template<> inline uint32_t clzImpl(const uint32_t& x) noexcept { return uint32_t(__builtin_clz(x)); }
template<> inline uint32_t clzImpl(const uint64_t& x) noexcept { return uint32_t(__builtin_clzll(x)); }
template<> inline uint32_t ctzImpl(const uint32_t& x) noexcept { return uint32_t(__builtin_ctz(x)); }
template<> inline uint32_t ctzImpl(const uint64_t& x) noexcept { return uint32_t(__builtin_ctzll(x)); }
# elif defined(_MSC_VER)
template<> inline uint32_t clzImpl(const uint32_t& x) noexcept { unsigned long i; _BitScanReverse(&i, x); return uint32_t(i ^ 31); }
template<> inline uint32_t ctzImpl(const uint32_t& x) noexcept { unsigned long i; _BitScanForward(&i, x); return uint32_t(i); }
# if ASMJIT_ARCH_X86 == 64 || ASMJIT_ARCH_ARM == 64
template<> inline uint32_t clzImpl(const uint64_t& x) noexcept { unsigned long i; _BitScanReverse64(&i, x); return uint32_t(i ^ 63); }
template<> inline uint32_t ctzImpl(const uint64_t& x) noexcept { unsigned long i; _BitScanForward64(&i, x); return uint32_t(i); }
# endif
# endif
#endif
} // {anonymous}
} // {Internal}
//! \endcond
//! Count leading zeros in `x` (returns a position of a first bit set in `x`).
//!
//! \note The input MUST NOT be zero, otherwise the result is undefined.
template<typename T>
static inline uint32_t clz(T x) noexcept { return Internal::clzImpl(asUInt(x)); }
//! Count trailing zeros in `x` (returns a position of a first bit set in `x`).
//!
//! \note The input MUST NOT be zero, otherwise the result is undefined.
template<typename T>
static inline uint32_t ctz(T x) noexcept { return Internal::ctzImpl(asUInt(x)); }
template<uint64_t kInput>
struct ConstCTZ {
static constexpr uint32_t value =
(kInput & (uint64_t(1) << 0)) ? 0 :
(kInput & (uint64_t(1) << 1)) ? 1 :
(kInput & (uint64_t(1) << 2)) ? 2 :
(kInput & (uint64_t(1) << 3)) ? 3 :
(kInput & (uint64_t(1) << 4)) ? 4 :
(kInput & (uint64_t(1) << 5)) ? 5 :
(kInput & (uint64_t(1) << 6)) ? 6 :
(kInput & (uint64_t(1) << 7)) ? 7 :
(kInput & (uint64_t(1) << 8)) ? 8 :
(kInput & (uint64_t(1) << 9)) ? 9 :
(kInput & (uint64_t(1) << 10)) ? 10 :
(kInput & (uint64_t(1) << 11)) ? 11 :
(kInput & (uint64_t(1) << 12)) ? 12 :
(kInput & (uint64_t(1) << 13)) ? 13 :
(kInput & (uint64_t(1) << 14)) ? 14 :
(kInput & (uint64_t(1) << 15)) ? 15 :
(kInput & (uint64_t(1) << 16)) ? 16 :
(kInput & (uint64_t(1) << 17)) ? 17 :
(kInput & (uint64_t(1) << 18)) ? 18 :
(kInput & (uint64_t(1) << 19)) ? 19 :
(kInput & (uint64_t(1) << 20)) ? 20 :
(kInput & (uint64_t(1) << 21)) ? 21 :
(kInput & (uint64_t(1) << 22)) ? 22 :
(kInput & (uint64_t(1) << 23)) ? 23 :
(kInput & (uint64_t(1) << 24)) ? 24 :
(kInput & (uint64_t(1) << 25)) ? 25 :
(kInput & (uint64_t(1) << 26)) ? 26 :
(kInput & (uint64_t(1) << 27)) ? 27 :
(kInput & (uint64_t(1) << 28)) ? 28 :
(kInput & (uint64_t(1) << 29)) ? 29 :
(kInput & (uint64_t(1) << 30)) ? 30 :
(kInput & (uint64_t(1) << 31)) ? 31 :
(kInput & (uint64_t(1) << 32)) ? 32 :
(kInput & (uint64_t(1) << 33)) ? 33 :
(kInput & (uint64_t(1) << 34)) ? 34 :
(kInput & (uint64_t(1) << 35)) ? 35 :
(kInput & (uint64_t(1) << 36)) ? 36 :
(kInput & (uint64_t(1) << 37)) ? 37 :
(kInput & (uint64_t(1) << 38)) ? 38 :
(kInput & (uint64_t(1) << 39)) ? 39 :
(kInput & (uint64_t(1) << 40)) ? 40 :
(kInput & (uint64_t(1) << 41)) ? 41 :
(kInput & (uint64_t(1) << 42)) ? 42 :
(kInput & (uint64_t(1) << 43)) ? 43 :
(kInput & (uint64_t(1) << 44)) ? 44 :
(kInput & (uint64_t(1) << 45)) ? 45 :
(kInput & (uint64_t(1) << 46)) ? 46 :
(kInput & (uint64_t(1) << 47)) ? 47 :
(kInput & (uint64_t(1) << 48)) ? 48 :
(kInput & (uint64_t(1) << 49)) ? 49 :
(kInput & (uint64_t(1) << 50)) ? 50 :
(kInput & (uint64_t(1) << 51)) ? 51 :
(kInput & (uint64_t(1) << 52)) ? 52 :
(kInput & (uint64_t(1) << 53)) ? 53 :
(kInput & (uint64_t(1) << 54)) ? 54 :
(kInput & (uint64_t(1) << 55)) ? 55 :
(kInput & (uint64_t(1) << 56)) ? 56 :
(kInput & (uint64_t(1) << 57)) ? 57 :
(kInput & (uint64_t(1) << 58)) ? 58 :
(kInput & (uint64_t(1) << 59)) ? 59 :
(kInput & (uint64_t(1) << 60)) ? 60 :
(kInput & (uint64_t(1) << 61)) ? 61 :
(kInput & (uint64_t(1) << 62)) ? 62 :
(kInput & (uint64_t(1) << 63)) ? 63 : 64;
};
// Support - PopCnt
// ================
// Based on the following resource:
// http://graphics.stanford.edu/~seander/bithacks.html
//
// Alternatively, for a very small number of bits in `x`:
// uint32_t n = 0;
// while (x) {
// x &= x - 1;
// n++;
// }
// return n;
//! \cond
namespace Internal {
static inline uint32_t constPopcntImpl(uint32_t x) noexcept {
x = x - ((x >> 1) & 0x55555555u);
x = (x & 0x33333333u) + ((x >> 2) & 0x33333333u);
return (((x + (x >> 4)) & 0x0F0F0F0Fu) * 0x01010101u) >> 24;
}
static inline uint32_t constPopcntImpl(uint64_t x) noexcept {
if (ASMJIT_ARCH_BITS >= 64) {
x = x - ((x >> 1) & 0x5555555555555555u);
x = (x & 0x3333333333333333u) + ((x >> 2) & 0x3333333333333333u);
return uint32_t((((x + (x >> 4)) & 0x0F0F0F0F0F0F0F0Fu) * 0x0101010101010101u) >> 56);
}
else {
return constPopcntImpl(uint32_t(x >> 32)) +
constPopcntImpl(uint32_t(x & 0xFFFFFFFFu));
}
}
static inline uint32_t popcntImpl(uint32_t x) noexcept {
#if defined(__GNUC__)
return uint32_t(__builtin_popcount(x));
#else
return constPopcntImpl(asUInt(x));
#endif
}
static inline uint32_t popcntImpl(uint64_t x) noexcept {
#if defined(__GNUC__)
return uint32_t(__builtin_popcountll(x));
#else
return constPopcntImpl(asUInt(x));
#endif
}
}
//! \endcond
//! Calculates count of bits in `x`.
template<typename T>
static inline uint32_t popcnt(T x) noexcept { return Internal::popcntImpl(asUInt(x)); }
//! Calculates count of bits in `x` (useful in constant expressions).
template<typename T>
static inline uint32_t constPopcnt(T x) noexcept { return Internal::constPopcntImpl(asUInt(x)); }
// Support - Min/Max
// =================
// NOTE: These are constexpr `min()` and `max()` implementations that are not
// exactly the same as `std::min()` and `std::max()`. The return value is not
// a reference to `a` or `b` but it's a new value instead.
template<typename T>
static constexpr T min(const T& a, const T& b) noexcept { return b < a ? b : a; }
template<typename T, typename... Args>
static constexpr T min(const T& a, const T& b, Args&&... args) noexcept { return min(min(a, b), std::forward<Args>(args)...); }
template<typename T>
static constexpr T max(const T& a, const T& b) noexcept { return a < b ? b : a; }
template<typename T, typename... Args>
static constexpr T max(const T& a, const T& b, Args&&... args) noexcept { return max(max(a, b), std::forward<Args>(args)...); }
// Support - Immediate Helpers
// ===========================
namespace Internal {
template<typename T, bool IsFloat>
struct ImmConv {
static inline int64_t fromT(const T& x) noexcept { return int64_t(x); }
static inline T toT(int64_t x) noexcept { return T(uint64_t(x) & Support::allOnes<typename std::make_unsigned<T>::type>()); }
};
template<typename T>
struct ImmConv<T, true> {
static inline int64_t fromT(const T& x) noexcept { return int64_t(bitCast<int64_t>(double(x))); }
static inline T toT(int64_t x) noexcept { return T(bitCast<double>(x)); }
};
}
template<typename T>
static inline int64_t immediateFromT(const T& x) noexcept { return Internal::ImmConv<T, std::is_floating_point<T>::value>::fromT(x); }
template<typename T>
static inline T immediateToT(int64_t x) noexcept { return Internal::ImmConv<T, std::is_floating_point<T>::value>::toT(x); }
// Support - Overflow Arithmetic
// =============================
//! \cond
namespace Internal {
template<typename T>
inline T addOverflowFallback(T x, T y, FastUInt8* of) noexcept {
typedef typename std::make_unsigned<T>::type U;
U result = U(x) + U(y);
*of = FastUInt8(*of | FastUInt8(isUnsigned<T>() ? result < U(x) : T((U(x) ^ ~U(y)) & (U(x) ^ result)) < 0));
return T(result);
}
template<typename T>
inline T subOverflowFallback(T x, T y, FastUInt8* of) noexcept {
typedef typename std::make_unsigned<T>::type U;
U result = U(x) - U(y);
*of = FastUInt8(*of | FastUInt8(isUnsigned<T>() ? result > U(x) : T((U(x) ^ U(y)) & (U(x) ^ result)) < 0));
return T(result);
}
template<typename T>
inline T mulOverflowFallback(T x, T y, FastUInt8* of) noexcept {
typedef typename Internal::StdInt<sizeof(T) * 2, isUnsigned<T>()>::Type I;
typedef typename std::make_unsigned<I>::type U;
U mask = allOnes<U>();
if (std::is_signed<T>::value) {
U prod = U(I(x)) * U(I(y));
*of = FastUInt8(*of | FastUInt8(I(prod) < I(std::numeric_limits<T>::lowest()) || I(prod) > I(std::numeric_limits<T>::max())));
return T(I(prod & mask));
}
else {
U prod = U(x) * U(y);
*of = FastUInt8(*of | FastUInt8((prod & ~mask) != 0));
return T(prod & mask);
}
}
template<>
inline int64_t mulOverflowFallback(int64_t x, int64_t y, FastUInt8* of) noexcept {
int64_t result = int64_t(uint64_t(x) * uint64_t(y));
*of = FastUInt8(*of | FastUInt8(x && (result / x != y)));
return result;
}
template<>
inline uint64_t mulOverflowFallback(uint64_t x, uint64_t y, FastUInt8* of) noexcept {
uint64_t result = x * y;
*of = FastUInt8(*of | FastUInt8(y != 0 && allOnes<uint64_t>() / y < x));
return result;
}
// These can be specialized.
template<typename T> inline T addOverflowImpl(const T& x, const T& y, FastUInt8* of) noexcept { return addOverflowFallback(x, y, of); }
template<typename T> inline T subOverflowImpl(const T& x, const T& y, FastUInt8* of) noexcept { return subOverflowFallback(x, y, of); }
template<typename T> inline T mulOverflowImpl(const T& x, const T& y, FastUInt8* of) noexcept { return mulOverflowFallback(x, y, of); }
#if defined(__GNUC__) && !defined(ASMJIT_NO_INTRINSICS)
#if defined(__clang__) || __GNUC__ >= 5
#define ASMJIT_ARITH_OVERFLOW_SPECIALIZE(FUNC, T, RESULT_T, BUILTIN) \
template<> \
inline T FUNC(const T& x, const T& y, FastUInt8* of) noexcept { \
RESULT_T result; \
*of = FastUInt8(*of | (BUILTIN((RESULT_T)x, (RESULT_T)y, &result))); \
return T(result); \
}
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, int32_t , int , __builtin_sadd_overflow )
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, uint32_t, unsigned int , __builtin_uadd_overflow )
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, int64_t , long long , __builtin_saddll_overflow)
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, uint64_t, unsigned long long, __builtin_uaddll_overflow)
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, int32_t , int , __builtin_ssub_overflow )
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, uint32_t, unsigned int , __builtin_usub_overflow )
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, int64_t , long long , __builtin_ssubll_overflow)
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, uint64_t, unsigned long long, __builtin_usubll_overflow)
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(mulOverflowImpl, int32_t , int , __builtin_smul_overflow )
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(mulOverflowImpl, uint32_t, unsigned int , __builtin_umul_overflow )
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(mulOverflowImpl, int64_t , long long , __builtin_smulll_overflow)
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(mulOverflowImpl, uint64_t, unsigned long long, __builtin_umulll_overflow)
#undef ASMJIT_ARITH_OVERFLOW_SPECIALIZE
#endif
#endif
// There is a bug in MSVC that makes these specializations unusable, maybe in the future...
#if defined(_MSC_VER) && 0
#define ASMJIT_ARITH_OVERFLOW_SPECIALIZE(FUNC, T, ALT_T, BUILTIN) \
template<> \
inline T FUNC(T x, T y, FastUInt8* of) noexcept { \
ALT_T result; \
*of = FastUInt8(*of | BUILTIN(0, (ALT_T)x, (ALT_T)y, &result)); \
return T(result); \
}
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, uint32_t, unsigned int , _addcarry_u32 )
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, uint32_t, unsigned int , _subborrow_u32)
#if ARCH_BITS >= 64
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(addOverflowImpl, uint64_t, unsigned __int64 , _addcarry_u64 )
ASMJIT_ARITH_OVERFLOW_SPECIALIZE(subOverflowImpl, uint64_t, unsigned __int64 , _subborrow_u64)
#endif
#undef ASMJIT_ARITH_OVERFLOW_SPECIALIZE
#endif
} // {Internal}
//! \endcond
template<typename T>
static inline T addOverflow(const T& x, const T& y, FastUInt8* of) noexcept { return T(Internal::addOverflowImpl(asStdInt(x), asStdInt(y), of)); }
template<typename T>
static inline T subOverflow(const T& x, const T& y, FastUInt8* of) noexcept { return T(Internal::subOverflowImpl(asStdInt(x), asStdInt(y), of)); }
template<typename T>
static inline T mulOverflow(const T& x, const T& y, FastUInt8* of) noexcept { return T(Internal::mulOverflowImpl(asStdInt(x), asStdInt(y), of)); }
// Support - Alignment
// ===================
template<typename X, typename Y>
static constexpr bool isAligned(X base, Y alignment) noexcept {
typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
return ((U)base % (U)alignment) == 0;
}
//! Tests whether the `x` is a power of two (only one bit is set).
template<typename T>
static constexpr bool isPowerOf2(T x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return x && !(U(x) & (U(x) - U(1)));
}
template<typename X, typename Y>
static constexpr X alignUp(X x, Y alignment) noexcept {
typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
return (X)( ((U)x + ((U)(alignment) - 1u)) & ~((U)(alignment) - 1u) );
}
template<typename T>
static constexpr T alignUpPowerOf2(T x) noexcept {
typedef typename Internal::StdInt<sizeof(T), 1>::Type U;
return (T)(fillTrailingBits(U(x) - 1u) + 1u);
}
//! Returns either zero or a positive difference between `base` and `base` when
//! aligned to `alignment`.
template<typename X, typename Y>
static constexpr typename Internal::StdInt<sizeof(X), 1>::Type alignUpDiff(X base, Y alignment) noexcept {
typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
return alignUp(U(base), alignment) - U(base);
}
template<typename X, typename Y>
static constexpr X alignDown(X x, Y alignment) noexcept {
typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
return (X)( (U)x & ~((U)(alignment) - 1u) );
}
// Support - NumGranularized
// =========================
//! Calculates the number of elements that would be required if `base` is
//! granularized by `granularity`. This function can be used to calculate
//! the number of BitWords to represent N bits, for example.
template<typename X, typename Y>
static constexpr X numGranularized(X base, Y granularity) noexcept {
typedef typename Internal::StdInt<sizeof(X), 1>::Type U;
return X((U(base) + U(granularity) - 1) / U(granularity));
}
// Support - IsBetween
// ===================
//! Checks whether `x` is greater than or equal to `a` and lesser than or equal to `b`.
template<typename T>
static constexpr bool isBetween(const T& x, const T& a, const T& b) noexcept {
return x >= a && x <= b;
}
// Support - IsInt & IsUInt
// ========================
//! Checks whether the given integer `x` can be casted to a 4-bit signed integer.
template<typename T>
static constexpr bool isInt4(T x) noexcept {
typedef typename std::make_signed<T>::type S;
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? isBetween<S>(S(x), -8, 7) : U(x) <= U(7u);
}
//! Checks whether the given integer `x` can be casted to a 7-bit signed integer.
template<typename T>
static constexpr bool isInt7(T x) noexcept {
typedef typename std::make_signed<T>::type S;
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? isBetween<S>(S(x), -64, 63) : U(x) <= U(63u);
}
//! Checks whether the given integer `x` can be casted to an 8-bit signed integer.
template<typename T>
static constexpr bool isInt8(T x) noexcept {
typedef typename std::make_signed<T>::type S;
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? sizeof(T) <= 1 || isBetween<S>(S(x), -128, 127) : U(x) <= U(127u);
}
//! Checks whether the given integer `x` can be casted to a 9-bit signed integer.
template<typename T>
static constexpr bool isInt9(T x) noexcept {
typedef typename std::make_signed<T>::type S;
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? sizeof(T) <= 1 || isBetween<S>(S(x), -256, 255)
: sizeof(T) <= 1 || U(x) <= U(255u);
}
//! Checks whether the given integer `x` can be casted to a 10-bit signed integer.
template<typename T>
static constexpr bool isInt10(T x) noexcept {
typedef typename std::make_signed<T>::type S;
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? sizeof(T) <= 1 || isBetween<S>(S(x), -512, 511)
: sizeof(T) <= 1 || U(x) <= U(511u);
}
//! Checks whether the given integer `x` can be casted to a 16-bit signed integer.
template<typename T>
static constexpr bool isInt16(T x) noexcept {
typedef typename std::make_signed<T>::type S;
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? sizeof(T) <= 2 || isBetween<S>(S(x), -32768, 32767)
: sizeof(T) <= 1 || U(x) <= U(32767u);
}
//! Checks whether the given integer `x` can be casted to a 32-bit signed integer.
template<typename T>
static constexpr bool isInt32(T x) noexcept {
typedef typename std::make_signed<T>::type S;
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? sizeof(T) <= 4 || isBetween<S>(S(x), -2147483647 - 1, 2147483647)
: sizeof(T) <= 2 || U(x) <= U(2147483647u);
}
//! Checks whether the given integer `x` can be casted to a 4-bit unsigned integer.
template<typename T>
static constexpr bool isUInt4(T x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? x >= T(0) && x <= T(15)
: U(x) <= U(15u);
}
//! Checks whether the given integer `x` can be casted to an 8-bit unsigned integer.
template<typename T>
static constexpr bool isUInt8(T x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? (sizeof(T) <= 1 || T(x) <= T(255)) && x >= T(0)
: (sizeof(T) <= 1 || U(x) <= U(255u));
}
//! Checks whether the given integer `x` can be casted to a 12-bit unsigned integer (ARM specific).
template<typename T>
static constexpr bool isUInt12(T x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? (sizeof(T) <= 1 || T(x) <= T(4095)) && x >= T(0)
: (sizeof(T) <= 1 || U(x) <= U(4095u));
}
//! Checks whether the given integer `x` can be casted to a 16-bit unsigned integer.
template<typename T>
static constexpr bool isUInt16(T x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? (sizeof(T) <= 2 || T(x) <= T(65535)) && x >= T(0)
: (sizeof(T) <= 2 || U(x) <= U(65535u));
}
//! Checks whether the given integer `x` can be casted to a 32-bit unsigned integer.
template<typename T>
static constexpr bool isUInt32(T x) noexcept {
typedef typename std::make_unsigned<T>::type U;
return std::is_signed<T>::value ? (sizeof(T) <= 4 || T(x) <= T(4294967295u)) && x >= T(0)
: (sizeof(T) <= 4 || U(x) <= U(4294967295u));
}
//! Checks whether the given integer `x` can be casted to a 32-bit unsigned integer.
template<typename T>
static constexpr bool isIntOrUInt32(T x) noexcept {
return sizeof(T) <= 4 ? true : (uint32_t(uint64_t(x) >> 32) + 1u) <= 1u;
}
static bool inline isEncodableOffset32(int32_t offset, uint32_t nBits) noexcept {
uint32_t nRev = 32 - nBits;
return Support::sar(Support::shl(offset, nRev), nRev) == offset;
}
static bool inline isEncodableOffset64(int64_t offset, uint32_t nBits) noexcept {
uint32_t nRev = 64 - nBits;
return Support::sar(Support::shl(offset, nRev), nRev) == offset;
}
// Support - ByteSwap
// ==================
static inline uint16_t byteswap16(uint16_t x) noexcept {
return uint16_t(((x >> 8) & 0xFFu) | ((x & 0xFFu) << 8));
}
static inline uint32_t byteswap32(uint32_t x) noexcept {
return (x << 24) | (x >> 24) | ((x << 8) & 0x00FF0000u) | ((x >> 8) & 0x0000FF00);
}
static inline uint64_t byteswap64(uint64_t x) noexcept {
#if (defined(__GNUC__) || defined(__clang__)) && !defined(ASMJIT_NO_INTRINSICS)
return uint64_t(__builtin_bswap64(uint64_t(x)));
#elif defined(_MSC_VER) && !defined(ASMJIT_NO_INTRINSICS)
return uint64_t(_byteswap_uint64(uint64_t(x)));
#else
return (uint64_t(byteswap32(uint32_t(uint64_t(x) >> 32 ))) ) |
(uint64_t(byteswap32(uint32_t(uint64_t(x) & 0xFFFFFFFFu))) << 32) ;
#endif
}
// Support - BytePack & Unpack
// ===========================
//! Pack four 8-bit integer into a 32-bit integer as it is an array of `{b0,b1,b2,b3}`.
static constexpr uint32_t bytepack32_4x8(uint32_t a, uint32_t b, uint32_t c, uint32_t d) noexcept {
return ASMJIT_ARCH_LE ? (a | (b << 8) | (c << 16) | (d << 24))
: (d | (c << 8) | (b << 16) | (a << 24));
}
template<typename T>
static constexpr uint32_t unpackU32At0(T x) noexcept { return ASMJIT_ARCH_LE ? uint32_t(uint64_t(x) & 0xFFFFFFFFu) : uint32_t(uint64_t(x) >> 32); }
template<typename T>
static constexpr uint32_t unpackU32At1(T x) noexcept { return ASMJIT_ARCH_BE ? uint32_t(uint64_t(x) & 0xFFFFFFFFu) : uint32_t(uint64_t(x) >> 32); }
// Support - Position of byte (in bit-shift)
// =========================================
static inline uint32_t byteShiftOfDWordStruct(uint32_t index) noexcept {
return ASMJIT_ARCH_LE ? index * 8 : (uint32_t(sizeof(uint32_t)) - 1u - index) * 8;
}
// Support - String Utilities
// ==========================
template<typename T>
static constexpr T asciiToLower(T c) noexcept { return T(c ^ T(T(c >= T('A') && c <= T('Z')) << 5)); }
template<typename T>
static constexpr T asciiToUpper(T c) noexcept { return T(c ^ T(T(c >= T('a') && c <= T('z')) << 5)); }
static ASMJIT_FORCE_INLINE size_t strLen(const char* s, size_t maxSize) noexcept {
size_t i = 0;
while (i < maxSize && s[i] != '\0')
i++;
return i;
}
static constexpr uint32_t hashRound(uint32_t hash, uint32_t c) noexcept { return hash * 65599 + c; }
// Gets a hash of the given string `data` of size `size`. Size must be valid
// as this function doesn't check for a null terminator and allows it in the
// middle of the string.
static inline uint32_t hashString(const char* data, size_t size) noexcept {
uint32_t hashCode = 0;
for (uint32_t i = 0; i < size; i++)
hashCode = hashRound(hashCode, uint8_t(data[i]));
return hashCode;
}
static ASMJIT_FORCE_INLINE const char* findPackedString(const char* p, uint32_t id) noexcept {
uint32_t i = 0;
while (i < id) {
while (p[0])
p++;
p++;
i++;
}
return p;
}
//! Compares two instruction names.
//!
//! `a` is a null terminated instruction name from arch-specific `nameData[]`
//! table. `b` is a possibly non-null terminated instruction name passed to
//! `InstAPI::stringToInstId()`.
static ASMJIT_FORCE_INLINE int cmpInstName(const char* a, const char* b, size_t size) noexcept {
for (size_t i = 0; i < size; i++) {
int c = int(uint8_t(a[i])) - int(uint8_t(b[i]));
if (c != 0) return c;
}
return int(uint8_t(a[size]));
}
// Support - Memory Read Access - 8 Bits
// =====================================
static inline uint8_t readU8(const void* p) noexcept { return static_cast<const uint8_t*>(p)[0]; }
static inline int8_t readI8(const void* p) noexcept { return static_cast<const int8_t*>(p)[0]; }
// Support - Memory Read Access - 16 Bits
// ======================================
template<ByteOrder BO, size_t Alignment>
static inline uint16_t readU16x(const void* p) noexcept {
typedef typename Internal::AliasedUInt<uint16_t, Alignment>::T U16AlignedToN;
uint16_t x = static_cast<const U16AlignedToN*>(p)[0];
return BO == ByteOrder::kNative ? x : byteswap16(x);
}
template<size_t Alignment = 1>
static inline uint16_t readU16u(const void* p) noexcept { return readU16x<ByteOrder::kNative, Alignment>(p); }
template<size_t Alignment = 1>
static inline uint16_t readU16uLE(const void* p) noexcept { return readU16x<ByteOrder::kLE, Alignment>(p); }
template<size_t Alignment = 1>
static inline uint16_t readU16uBE(const void* p) noexcept { return readU16x<ByteOrder::kBE, Alignment>(p); }
static inline uint16_t readU16a(const void* p) noexcept { return readU16x<ByteOrder::kNative, 2>(p); }
static inline uint16_t readU16aLE(const void* p) noexcept { return readU16x<ByteOrder::kLE, 2>(p); }
static inline uint16_t readU16aBE(const void* p) noexcept { return readU16x<ByteOrder::kBE, 2>(p); }
template<ByteOrder BO, size_t Alignment>
static inline int16_t readI16x(const void* p) noexcept { return int16_t(readU16x<BO, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int16_t readI16u(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kNative, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int16_t readI16uLE(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kLE, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int16_t readI16uBE(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kBE, Alignment>(p)); }
static inline int16_t readI16a(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kNative, 2>(p)); }
static inline int16_t readI16aLE(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kLE, 2>(p)); }
static inline int16_t readI16aBE(const void* p) noexcept { return int16_t(readU16x<ByteOrder::kBE, 2>(p)); }
// Support - Memory Read Access - 24 Bits
// ======================================
template<ByteOrder BO = ByteOrder::kNative>
static inline uint32_t readU24u(const void* p) noexcept {
uint32_t b0 = readU8(static_cast<const uint8_t*>(p) + (BO == ByteOrder::kLE ? 2 : 0));
uint32_t b1 = readU8(static_cast<const uint8_t*>(p) + (BO == ByteOrder::kLE ? 1 : 1));
uint32_t b2 = readU8(static_cast<const uint8_t*>(p) + (BO == ByteOrder::kLE ? 0 : 2));
return (b0 << 16) | (b1 << 8) | b2;
}
static inline uint32_t readU24uLE(const void* p) noexcept { return readU24u<ByteOrder::kLE>(p); }
static inline uint32_t readU24uBE(const void* p) noexcept { return readU24u<ByteOrder::kBE>(p); }
// Support - Memory Read Access - 32 Bits
// ======================================
template<ByteOrder BO, size_t Alignment>
static inline uint32_t readU32x(const void* p) noexcept {
typedef typename Internal::AliasedUInt<uint32_t, Alignment>::T U32AlignedToN;
uint32_t x = static_cast<const U32AlignedToN*>(p)[0];
return BO == ByteOrder::kNative ? x : byteswap32(x);
}
template<size_t Alignment = 1>
static inline uint32_t readU32u(const void* p) noexcept { return readU32x<ByteOrder::kNative, Alignment>(p); }
template<size_t Alignment = 1>
static inline uint32_t readU32uLE(const void* p) noexcept { return readU32x<ByteOrder::kLE, Alignment>(p); }
template<size_t Alignment = 1>
static inline uint32_t readU32uBE(const void* p) noexcept { return readU32x<ByteOrder::kBE, Alignment>(p); }
static inline uint32_t readU32a(const void* p) noexcept { return readU32x<ByteOrder::kNative, 4>(p); }
static inline uint32_t readU32aLE(const void* p) noexcept { return readU32x<ByteOrder::kLE, 4>(p); }
static inline uint32_t readU32aBE(const void* p) noexcept { return readU32x<ByteOrder::kBE, 4>(p); }
template<ByteOrder BO, size_t Alignment>
static inline uint32_t readI32x(const void* p) noexcept { return int32_t(readU32x<BO, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int32_t readI32u(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kNative, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int32_t readI32uLE(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kLE, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int32_t readI32uBE(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kBE, Alignment>(p)); }
static inline int32_t readI32a(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kNative, 4>(p)); }
static inline int32_t readI32aLE(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kLE, 4>(p)); }
static inline int32_t readI32aBE(const void* p) noexcept { return int32_t(readU32x<ByteOrder::kBE, 4>(p)); }
// Support - Memory Read Access - 64 Bits
// ======================================
template<ByteOrder BO, size_t Alignment>
static inline uint64_t readU64x(const void* p) noexcept {
typedef typename Internal::AliasedUInt<uint64_t, Alignment>::T U64AlignedToN;
uint64_t x = static_cast<const U64AlignedToN*>(p)[0];
return BO == ByteOrder::kNative ? x : byteswap64(x);
}
template<size_t Alignment = 1>
static inline uint64_t readU64u(const void* p) noexcept { return readU64x<ByteOrder::kNative, Alignment>(p); }
template<size_t Alignment = 1>
static inline uint64_t readU64uLE(const void* p) noexcept { return readU64x<ByteOrder::kLE, Alignment>(p); }
template<size_t Alignment = 1>
static inline uint64_t readU64uBE(const void* p) noexcept { return readU64x<ByteOrder::kBE, Alignment>(p); }
static inline uint64_t readU64a(const void* p) noexcept { return readU64x<ByteOrder::kNative, 8>(p); }
static inline uint64_t readU64aLE(const void* p) noexcept { return readU64x<ByteOrder::kLE, 8>(p); }
static inline uint64_t readU64aBE(const void* p) noexcept { return readU64x<ByteOrder::kBE, 8>(p); }
template<ByteOrder BO, size_t Alignment>
static inline int64_t readI64x(const void* p) noexcept { return int64_t(readU64x<BO, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int64_t readI64u(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kNative, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int64_t readI64uLE(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kLE, Alignment>(p)); }
template<size_t Alignment = 1>
static inline int64_t readI64uBE(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kBE, Alignment>(p)); }
static inline int64_t readI64a(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kNative, 8>(p)); }
static inline int64_t readI64aLE(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kLE, 8>(p)); }
static inline int64_t readI64aBE(const void* p) noexcept { return int64_t(readU64x<ByteOrder::kBE, 8>(p)); }
// Support - Memory Write Access - 8 Bits
// ======================================
static inline void writeU8(void* p, uint8_t x) noexcept { static_cast<uint8_t*>(p)[0] = x; }
static inline void writeI8(void* p, int8_t x) noexcept { static_cast<int8_t*>(p)[0] = x; }
// Support - Memory Write Access - 16 Bits
// =======================================
template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
static inline void writeU16x(void* p, uint16_t x) noexcept {
typedef typename Internal::AliasedUInt<uint16_t, Alignment>::T U16AlignedToN;
static_cast<U16AlignedToN*>(p)[0] = BO == ByteOrder::kNative ? x : byteswap16(x);
}
template<size_t Alignment = 1>
static inline void writeU16uLE(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kLE, Alignment>(p, x); }
template<size_t Alignment = 1>
static inline void writeU16uBE(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kBE, Alignment>(p, x); }
static inline void writeU16a(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kNative, 2>(p, x); }
static inline void writeU16aLE(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kLE, 2>(p, x); }
static inline void writeU16aBE(void* p, uint16_t x) noexcept { writeU16x<ByteOrder::kBE, 2>(p, x); }
template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
static inline void writeI16x(void* p, int16_t x) noexcept { writeU16x<BO, Alignment>(p, uint16_t(x)); }
template<size_t Alignment = 1>
static inline void writeI16uLE(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kLE, Alignment>(p, uint16_t(x)); }
template<size_t Alignment = 1>
static inline void writeI16uBE(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kBE, Alignment>(p, uint16_t(x)); }
static inline void writeI16a(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kNative, 2>(p, uint16_t(x)); }
static inline void writeI16aLE(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kLE, 2>(p, uint16_t(x)); }
static inline void writeI16aBE(void* p, int16_t x) noexcept { writeU16x<ByteOrder::kBE, 2>(p, uint16_t(x)); }
// Support - Memory Write Access - 24 Bits
// =======================================
template<ByteOrder BO = ByteOrder::kNative>
static inline void writeU24u(void* p, uint32_t v) noexcept {
static_cast<uint8_t*>(p)[0] = uint8_t((v >> (BO == ByteOrder::kLE ? 0 : 16)) & 0xFFu);
static_cast<uint8_t*>(p)[1] = uint8_t((v >> (BO == ByteOrder::kLE ? 8 : 8)) & 0xFFu);
static_cast<uint8_t*>(p)[2] = uint8_t((v >> (BO == ByteOrder::kLE ? 16 : 0)) & 0xFFu);
}
static inline void writeU24uLE(void* p, uint32_t v) noexcept { writeU24u<ByteOrder::kLE>(p, v); }
static inline void writeU24uBE(void* p, uint32_t v) noexcept { writeU24u<ByteOrder::kBE>(p, v); }
// Support - Memory Write Access - 32 Bits
// =======================================
template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
static inline void writeU32x(void* p, uint32_t x) noexcept {
typedef typename Internal::AliasedUInt<uint32_t, Alignment>::T U32AlignedToN;
static_cast<U32AlignedToN*>(p)[0] = (BO == ByteOrder::kNative) ? x : Support::byteswap32(x);
}
template<size_t Alignment = 1>
static inline void writeU32u(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kNative, Alignment>(p, x); }
template<size_t Alignment = 1>
static inline void writeU32uLE(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kLE, Alignment>(p, x); }
template<size_t Alignment = 1>
static inline void writeU32uBE(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kBE, Alignment>(p, x); }
static inline void writeU32a(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kNative, 4>(p, x); }
static inline void writeU32aLE(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kLE, 4>(p, x); }
static inline void writeU32aBE(void* p, uint32_t x) noexcept { writeU32x<ByteOrder::kBE, 4>(p, x); }
template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
static inline void writeI32x(void* p, int32_t x) noexcept { writeU32x<BO, Alignment>(p, uint32_t(x)); }
template<size_t Alignment = 1>
static inline void writeI32u(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kNative, Alignment>(p, uint32_t(x)); }
template<size_t Alignment = 1>
static inline void writeI32uLE(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kLE, Alignment>(p, uint32_t(x)); }
template<size_t Alignment = 1>
static inline void writeI32uBE(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kBE, Alignment>(p, uint32_t(x)); }
static inline void writeI32a(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kNative, 4>(p, uint32_t(x)); }
static inline void writeI32aLE(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kLE, 4>(p, uint32_t(x)); }
static inline void writeI32aBE(void* p, int32_t x) noexcept { writeU32x<ByteOrder::kBE, 4>(p, uint32_t(x)); }
// Support - Memory Write Access - 64 Bits
// =======================================
template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
static inline void writeU64x(void* p, uint64_t x) noexcept {
typedef typename Internal::AliasedUInt<uint64_t, Alignment>::T U64AlignedToN;
static_cast<U64AlignedToN*>(p)[0] = BO == ByteOrder::kNative ? x : byteswap64(x);
}
template<size_t Alignment = 1>
static inline void writeU64u(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kNative, Alignment>(p, x); }
template<size_t Alignment = 1>
static inline void writeU64uLE(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kLE, Alignment>(p, x); }
template<size_t Alignment = 1>
static inline void writeU64uBE(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kBE, Alignment>(p, x); }
static inline void writeU64a(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kNative, 8>(p, x); }
static inline void writeU64aLE(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kLE, 8>(p, x); }
static inline void writeU64aBE(void* p, uint64_t x) noexcept { writeU64x<ByteOrder::kBE, 8>(p, x); }
template<ByteOrder BO = ByteOrder::kNative, size_t Alignment = 1>
static inline void writeI64x(void* p, int64_t x) noexcept { writeU64x<BO, Alignment>(p, uint64_t(x)); }
template<size_t Alignment = 1>
static inline void writeI64u(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kNative, Alignment>(p, uint64_t(x)); }
template<size_t Alignment = 1>
static inline void writeI64uLE(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kLE, Alignment>(p, uint64_t(x)); }
template<size_t Alignment = 1>
static inline void writeI64uBE(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kBE, Alignment>(p, uint64_t(x)); }
static inline void writeI64a(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kNative, 8>(p, uint64_t(x)); }
static inline void writeI64aLE(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kLE, 8>(p, uint64_t(x)); }
static inline void writeI64aBE(void* p, int64_t x) noexcept { writeU64x<ByteOrder::kBE, 8>(p, uint64_t(x)); }
// Support - Operators
// ===================
//! \cond INTERNAL
struct Set { template<typename T> static inline T op(T x, T y) noexcept { DebugUtils::unused(x); return y; } };
struct SetNot { template<typename T> static inline T op(T x, T y) noexcept { DebugUtils::unused(x); return ~y; } };
struct And { template<typename T> static inline T op(T x, T y) noexcept { return x & y; } };
struct AndNot { template<typename T> static inline T op(T x, T y) noexcept { return x & ~y; } };
struct NotAnd { template<typename T> static inline T op(T x, T y) noexcept { return ~x & y; } };
struct Or { template<typename T> static inline T op(T x, T y) noexcept { return x | y; } };
struct Xor { template<typename T> static inline T op(T x, T y) noexcept { return x ^ y; } };
struct Add { template<typename T> static inline T op(T x, T y) noexcept { return x + y; } };
struct Sub { template<typename T> static inline T op(T x, T y) noexcept { return x - y; } };
struct Min { template<typename T> static inline T op(T x, T y) noexcept { return min<T>(x, y); } };
struct Max { template<typename T> static inline T op(T x, T y) noexcept { return max<T>(x, y); } };
//! \endcond
// Support - BitWordIterator
// =========================
//! Iterates over each bit in a number which is set to 1.
//!
//! Example of use:
//!
//! ```
//! uint32_t bitsToIterate = 0x110F;
//! Support::BitWordIterator<uint32_t> it(bitsToIterate);
//!
//! while (it.hasNext()) {
//! uint32_t bitIndex = it.next();
//! std::printf("Bit at %u is set\n", unsigned(bitIndex));
//! }
//! ```
template<typename T>
class BitWordIterator {
public:
ASMJIT_FORCE_INLINE explicit BitWordIterator(T bitWord) noexcept
: _bitWord(bitWord) {}
ASMJIT_FORCE_INLINE void init(T bitWord) noexcept { _bitWord = bitWord; }
ASMJIT_FORCE_INLINE bool hasNext() const noexcept { return _bitWord != 0; }
ASMJIT_FORCE_INLINE uint32_t next() noexcept {
ASMJIT_ASSERT(_bitWord != 0);
uint32_t index = ctz(_bitWord);
_bitWord ^= T(1u) << index;
return index;
}
T _bitWord;
};
// Support - BitWordFlipIterator
// =============================
template<typename T>
class BitWordFlipIterator {
public:
ASMJIT_FORCE_INLINE explicit BitWordFlipIterator(T bitWord) noexcept
: _bitWord(bitWord) {}
ASMJIT_FORCE_INLINE void init(T bitWord) noexcept { _bitWord = bitWord; }
ASMJIT_FORCE_INLINE bool hasNext() const noexcept { return _bitWord != 0; }
ASMJIT_FORCE_INLINE uint32_t nextAndFlip() noexcept {
ASMJIT_ASSERT(_bitWord != 0);
uint32_t index = ctz(_bitWord);
_bitWord ^= T(1u) << index;
return index;
}
T _bitWord;
T _xorMask;
};
// Support - BitVectorOps
// ======================
//! \cond
namespace Internal {
template<typename T, class OperatorT, class FullWordOpT>
static inline void bitVectorOp(T* buf, size_t index, size_t count) noexcept {
if (count == 0)
return;
const size_t kTSizeInBits = bitSizeOf<T>();
size_t vecIndex = index / kTSizeInBits; // T[]
size_t bitIndex = index % kTSizeInBits; // T[][]
buf += vecIndex;
// The first BitWord requires special handling to preserve bits outside the fill region.
const T kFillMask = allOnes<T>();
size_t firstNBits = min<size_t>(kTSizeInBits - bitIndex, count);
buf[0] = OperatorT::op(buf[0], (kFillMask >> (kTSizeInBits - firstNBits)) << bitIndex);
buf++;
count -= firstNBits;
// All bits between the first and last affected BitWords can be just filled.
while (count >= kTSizeInBits) {
buf[0] = FullWordOpT::op(buf[0], kFillMask);
buf++;
count -= kTSizeInBits;
}
// The last BitWord requires special handling as well
if (count)
buf[0] = OperatorT::op(buf[0], kFillMask >> (kTSizeInBits - count));
}
}
//! \endcond
//! Sets bit in a bit-vector `buf` at `index`.
template<typename T>
static inline bool bitVectorGetBit(T* buf, size_t index) noexcept {
const size_t kTSizeInBits = bitSizeOf<T>();
size_t vecIndex = index / kTSizeInBits;
size_t bitIndex = index % kTSizeInBits;
return bool((buf[vecIndex] >> bitIndex) & 0x1u);
}
//! Sets bit in a bit-vector `buf` at `index` to `value`.
template<typename T>
static inline void bitVectorSetBit(T* buf, size_t index, bool value) noexcept {
const size_t kTSizeInBits = bitSizeOf<T>();
size_t vecIndex = index / kTSizeInBits;
size_t bitIndex = index % kTSizeInBits;
T bitMask = T(1u) << bitIndex;
if (value)
buf[vecIndex] |= bitMask;
else
buf[vecIndex] &= ~bitMask;
}
//! Sets bit in a bit-vector `buf` at `index` to `value`.
template<typename T>
static inline void bitVectorFlipBit(T* buf, size_t index) noexcept {
const size_t kTSizeInBits = bitSizeOf<T>();
size_t vecIndex = index / kTSizeInBits;
size_t bitIndex = index % kTSizeInBits;
T bitMask = T(1u) << bitIndex;
buf[vecIndex] ^= bitMask;
}
//! Fills `count` bits in bit-vector `buf` starting at bit-index `index`.
template<typename T>
static inline void bitVectorFill(T* buf, size_t index, size_t count) noexcept { Internal::bitVectorOp<T, Or, Set>(buf, index, count); }
//! Clears `count` bits in bit-vector `buf` starting at bit-index `index`.
template<typename T>
static inline void bitVectorClear(T* buf, size_t index, size_t count) noexcept { Internal::bitVectorOp<T, AndNot, SetNot>(buf, index, count); }
template<typename T>
static inline size_t bitVectorIndexOf(T* buf, size_t start, bool value) noexcept {
const size_t kTSizeInBits = bitSizeOf<T>();
size_t vecIndex = start / kTSizeInBits; // T[]
size_t bitIndex = start % kTSizeInBits; // T[][]
T* p = buf + vecIndex;
// We always look for zeros, if value is `true` we have to flip all bits before the search.
const T kFillMask = allOnes<T>();
const T kFlipMask = value ? T(0) : kFillMask;
// The first BitWord requires special handling as there are some bits we want to ignore.
T bits = (*p ^ kFlipMask) & (kFillMask << bitIndex);
for (;;) {
if (bits)
return (size_t)(p - buf) * kTSizeInBits + ctz(bits);
bits = *++p ^ kFlipMask;
}
}
// Support - BitVectorIterator
// ===========================
template<typename T>
class BitVectorIterator {
public:
const T* _ptr;
size_t _idx;
size_t _end;
T _current;
ASMJIT_FORCE_INLINE BitVectorIterator(const BitVectorIterator& other) noexcept = default;
ASMJIT_FORCE_INLINE BitVectorIterator(const T* data, size_t numBitWords, size_t start = 0) noexcept {
init(data, numBitWords, start);
}
ASMJIT_FORCE_INLINE void init(const T* data, size_t numBitWords, size_t start = 0) noexcept {
const T* ptr = data + (start / bitSizeOf<T>());
size_t idx = alignDown(start, bitSizeOf<T>());
size_t end = numBitWords * bitSizeOf<T>();
T bitWord = T(0);
if (idx < end) {
bitWord = *ptr++ & (allOnes<T>() << (start % bitSizeOf<T>()));
while (!bitWord && (idx += bitSizeOf<T>()) < end)
bitWord = *ptr++;
}
_ptr = ptr;
_idx = idx;
_end = end;
_current = bitWord;
}
ASMJIT_FORCE_INLINE bool hasNext() const noexcept {
return _current != T(0);
}
ASMJIT_FORCE_INLINE size_t next() noexcept {
T bitWord = _current;
ASMJIT_ASSERT(bitWord != T(0));
uint32_t bit = ctz(bitWord);
bitWord ^= T(1u) << bit;
size_t n = _idx + bit;
while (!bitWord && (_idx += bitSizeOf<T>()) < _end)
bitWord = *_ptr++;
_current = bitWord;
return n;
}
ASMJIT_FORCE_INLINE size_t peekNext() const noexcept {
ASMJIT_ASSERT(_current != T(0));
return _idx + ctz(_current);
}
};
// Support - BitVectorOpIterator
// =============================
template<typename T, class OperatorT>
class BitVectorOpIterator {
public:
enum : uint32_t {
kTSizeInBits = bitSizeOf<T>()
};
const T* _aPtr;
const T* _bPtr;
size_t _idx;
size_t _end;
T _current;
ASMJIT_FORCE_INLINE BitVectorOpIterator(const T* aData, const T* bData, size_t numBitWords, size_t start = 0) noexcept {
init(aData, bData, numBitWords, start);
}
ASMJIT_FORCE_INLINE void init(const T* aData, const T* bData, size_t numBitWords, size_t start = 0) noexcept {
const T* aPtr = aData + (start / bitSizeOf<T>());
const T* bPtr = bData + (start / bitSizeOf<T>());
size_t idx = alignDown(start, bitSizeOf<T>());
size_t end = numBitWords * bitSizeOf<T>();
T bitWord = T(0);
if (idx < end) {
bitWord = OperatorT::op(*aPtr++, *bPtr++) & (allOnes<T>() << (start % bitSizeOf<T>()));
while (!bitWord && (idx += kTSizeInBits) < end)
bitWord = OperatorT::op(*aPtr++, *bPtr++);
}
_aPtr = aPtr;
_bPtr = bPtr;
_idx = idx;
_end = end;
_current = bitWord;
}
ASMJIT_FORCE_INLINE bool hasNext() noexcept {
return _current != T(0);
}
ASMJIT_FORCE_INLINE size_t next() noexcept {
T bitWord = _current;
ASMJIT_ASSERT(bitWord != T(0));
uint32_t bit = ctz(bitWord);
bitWord ^= T(1u) << bit;
size_t n = _idx + bit;
while (!bitWord && (_idx += kTSizeInBits) < _end)
bitWord = OperatorT::op(*_aPtr++, *_bPtr++);
_current = bitWord;
return n;
}
};
// Support - Sorting
// =================
//! Sort order.
enum class SortOrder : uint32_t {
//!< Ascending order.
kAscending = 0,
//!< Descending order.
kDescending = 1
};
//! A helper class that provides comparison of any user-defined type that
//! implements `<` and `>` operators (primitive types are supported as well).
template<SortOrder kOrder = SortOrder::kAscending>
struct Compare {
template<typename A, typename B>
inline int operator()(const A& a, const B& b) const noexcept {
return kOrder == SortOrder::kAscending ? int(a > b) - int(a < b) : int(a < b) - int(a > b);
}
};
//! Insertion sort.
template<typename T, typename CompareT = Compare<SortOrder::kAscending>>
static inline void iSort(T* base, size_t size, const CompareT& cmp = CompareT()) noexcept {
for (T* pm = base + 1; pm < base + size; pm++)
for (T* pl = pm; pl > base && cmp(pl[-1], pl[0]) > 0; pl--)
std::swap(pl[-1], pl[0]);
}
//! \cond
namespace Internal {
//! Quick-sort implementation.
template<typename T, class CompareT>
struct QSortImpl {
enum : size_t {
kStackSize = 64 * 2,
kISortThreshold = 7
};
// Based on "PDCLib - Public Domain C Library" and rewritten to C++.
static void sort(T* base, size_t size, const CompareT& cmp) noexcept {
T* end = base + size;
T* stack[kStackSize];
T** stackptr = stack;
for (;;) {
if ((size_t)(end - base) > kISortThreshold) {
// We work from second to last - first will be pivot element.
T* pi = base + 1;
T* pj = end - 1;
std::swap(base[(size_t)(end - base) / 2], base[0]);
if (cmp(*pi , *pj ) > 0) std::swap(*pi , *pj );
if (cmp(*base, *pj ) > 0) std::swap(*base, *pj );
if (cmp(*pi , *base) > 0) std::swap(*pi , *base);
// Now we have the median for pivot element, entering main loop.
for (;;) {
while (pi < pj && cmp(*++pi, *base) < 0) continue; // Move `i` right until `*i >= pivot`.
while (pj > base && cmp(*--pj, *base) > 0) continue; // Move `j` left until `*j <= pivot`.
if (pi > pj) break;
std::swap(*pi, *pj);
}
// Move pivot into correct place.
std::swap(*base, *pj);
// Larger subfile base / end to stack, sort smaller.
if (pj - base > end - pi) {
// Left is larger.
*stackptr++ = base;
*stackptr++ = pj;
base = pi;
}
else {
// Right is larger.
*stackptr++ = pi;
*stackptr++ = end;
end = pj;
}
ASMJIT_ASSERT(stackptr <= stack + kStackSize);
}
else {
// UB sanitizer doesn't like applying offset to a nullptr base.
if (base != end)
iSort(base, (size_t)(end - base), cmp);
if (stackptr == stack)
break;
end = *--stackptr;
base = *--stackptr;
}
}
}
};
}
//! \endcond
//! Quick sort implementation.
//!
//! The main reason to provide a custom qsort implementation is that we needed something that will
//! never throw `bad_alloc` exception. This implementation doesn't use dynamic memory allocation.
template<typename T, class CompareT = Compare<SortOrder::kAscending>>
static inline void qSort(T* base, size_t size, const CompareT& cmp = CompareT()) noexcept {
Internal::QSortImpl<T, CompareT>::sort(base, size, cmp);
}
// Support - Array
// ===============
//! Array type, similar to std::array<T, N>, with the possibility to use enums in operator[].
//!
//! \note The array has C semantics - the elements in the array are not initialized.
template<typename T, size_t N>
struct Array {
//! \name Members
//! \{
//! The underlying array data, use \ref data() to access it.
T _data[N];
//! \}
//! \cond
// std compatibility.
typedef T value_type;
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef value_type& reference;
typedef const value_type& const_reference;
typedef value_type* pointer;
typedef const value_type* const_pointer;
typedef pointer iterator;
typedef const_pointer const_iterator;
//! \endcond
//! \name Overloaded Operators
//! \{
template<typename Index>
inline T& operator[](const Index& index) noexcept {
typedef typename Internal::StdInt<sizeof(Index), 1>::Type U;
ASMJIT_ASSERT(U(index) < N);
return _data[U(index)];
}
template<typename Index>
inline const T& operator[](const Index& index) const noexcept {
typedef typename Internal::StdInt<sizeof(Index), 1>::Type U;
ASMJIT_ASSERT(U(index) < N);
return _data[U(index)];
}
inline bool operator==(const Array& other) const noexcept {
for (size_t i = 0; i < N; i++)
if (_data[i] != other._data[i])
return false;
return true;
}
inline bool operator!=(const Array& other) const noexcept {
return !operator==(other);
}
//! \}
//! \name Accessors
//! \{
inline bool empty() const noexcept { return false; }
inline size_t size() const noexcept { return N; }
inline T* data() noexcept { return _data; }
inline const T* data() const noexcept { return _data; }
inline T& front() noexcept { return _data[0]; }
inline const T& front() const noexcept { return _data[0]; }
inline T& back() noexcept { return _data[N - 1]; }
inline const T& back() const noexcept { return _data[N - 1]; }
inline T* begin() noexcept { return _data; }
inline T* end() noexcept { return _data + N; }
inline const T* begin() const noexcept { return _data; }
inline const T* end() const noexcept { return _data + N; }
inline const T* cbegin() const noexcept { return _data; }
inline const T* cend() const noexcept { return _data + N; }
//! \}
//! \name Utilities
//! \{
inline void swap(Array& other) noexcept {
for (size_t i = 0; i < N; i++)
std::swap(_data[i], other._data[i]);
}
inline void fill(const T& value) noexcept {
for (size_t i = 0; i < N; i++)
_data[i] = value;
}
inline void copyFrom(const Array& other) noexcept {
for (size_t i = 0; i < N; i++)
_data[i] = other._data[i];
}
template<typename Operator>
inline void combine(const Array& other) noexcept {
for (size_t i = 0; i < N; i++)
_data[i] = Operator::op(_data[i], other._data[i]);
}
template<typename Operator>
inline T aggregate(T initialValue = T()) const noexcept {
T value = initialValue;
for (size_t i = 0; i < N; i++)
value = Operator::op(value, _data[i]);
return value;
}
template<typename Fn>
inline void forEach(Fn&& fn) noexcept {
for (size_t i = 0; i < N; i++)
fn(_data[i]);
}
//! \}
};
// Support::Temporary
// ==================
//! Used to pass a temporary buffer to:
//!
//! - Containers that use user-passed buffer as an initial storage (still can grow).
//! - Zone allocator that would use the temporary buffer as a first block.
struct Temporary {
//! \name Members
//! \{
void* _data;
size_t _size;
//! \}
//! \name Construction & Destruction
//! \{
inline constexpr Temporary(const Temporary& other) noexcept = default;
inline constexpr Temporary(void* data, size_t size) noexcept
: _data(data),
_size(size) {}
//! \}
//! \name Overloaded Operators
//! \{
inline Temporary& operator=(const Temporary& other) noexcept = default;
//! \}
//! \name Accessors
//! \{
//! Returns the data storage.
template<typename T = void>
inline constexpr T* data() const noexcept { return static_cast<T*>(_data); }
//! Returns the data storage size in bytes.
inline constexpr size_t size() const noexcept { return _size; }
//! \}
};
} // {Support}
//! \}
ASMJIT_END_NAMESPACE
#endif // ASMJIT_CORE_SUPPORT_H_INCLUDED
libraries/asmjit/asmjit/core/target.cpp 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
#include "../core/target.h"
ASMJIT_BEGIN_NAMESPACE
Target::Target() noexcept : _environment() {}
Target::~Target() noexcept {}
ASMJIT_END_NAMESPACE
libraries/asmjit/asmjit/core/target.h 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#ifndef ASMJIT_CORE_TARGET_H_INCLUDED
#define ASMJIT_CORE_TARGET_H_INCLUDED
#include "../core/archtraits.h"
#include "../core/func.h"
ASMJIT_BEGIN_NAMESPACE
//! \addtogroup asmjit_core
//! \{
//! Target is an abstract class that describes a machine code target.
class ASMJIT_VIRTAPI Target {
public:
ASMJIT_BASE_CLASS(Target)
ASMJIT_NONCOPYABLE(Target)
//! Target environment information.
Environment _environment;
//! \name Construction & Destruction
//! \{
//! Creates a `Target` instance.
ASMJIT_API Target() noexcept;
//! Destroys the `Target` instance.
ASMJIT_API virtual ~Target() noexcept;
//! \}
//! \name Accessors
//! \{
//! Returns target's environment.
inline const Environment& environment() const noexcept { return _environment; }
//! Returns the target architecture.
inline Arch arch() const noexcept { return _environment.arch(); }
//! Returns the target sub-architecture.
inline SubArch subArch() const noexcept { return _environment.subArch(); }
//! \}
};
//! \}
ASMJIT_END_NAMESPACE
#endif // ASMJIT_CORE_TARGET_H_INCLUDED
libraries/asmjit/asmjit/core/type.cpp 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
#include "../core/misc_p.h"
#include "../core/type.h"
ASMJIT_BEGIN_NAMESPACE
namespace TypeUtils {
template<uint32_t Index>
struct ScalarOfTypeId {
enum : uint32_t {
kTypeId = uint32_t(
isScalar(TypeId(Index)) ? TypeId(Index) :
isMask8 (TypeId(Index)) ? TypeId::kUInt8 :
isMask16(TypeId(Index)) ? TypeId::kUInt16 :
isMask32(TypeId(Index)) ? TypeId::kUInt32 :
isMask64(TypeId(Index)) ? TypeId::kUInt64 :
isMmx32 (TypeId(Index)) ? TypeId::kUInt32 :
isMmx64 (TypeId(Index)) ? TypeId::kUInt64 :
isVec32 (TypeId(Index)) ? TypeId((Index - uint32_t(TypeId::_kVec32Start ) + uint32_t(TypeId::kInt8)) & 0xFF) :
isVec64 (TypeId(Index)) ? TypeId((Index - uint32_t(TypeId::_kVec64Start ) + uint32_t(TypeId::kInt8)) & 0xFF) :
isVec128(TypeId(Index)) ? TypeId((Index - uint32_t(TypeId::_kVec128Start) + uint32_t(TypeId::kInt8)) & 0xFF) :
isVec256(TypeId(Index)) ? TypeId((Index - uint32_t(TypeId::_kVec256Start) + uint32_t(TypeId::kInt8)) & 0xFF) :
isVec512(TypeId(Index)) ? TypeId((Index - uint32_t(TypeId::_kVec512Start) + uint32_t(TypeId::kInt8)) & 0xFF) : TypeId::kVoid)
};
};
template<uint32_t Index>
struct SizeOfTypeId {
enum : uint32_t {
kTypeSize =
isInt8 (TypeId(Index)) ? 1 :
isUInt8 (TypeId(Index)) ? 1 :
isInt16 (TypeId(Index)) ? 2 :
isUInt16 (TypeId(Index)) ? 2 :
isInt32 (TypeId(Index)) ? 4 :
isUInt32 (TypeId(Index)) ? 4 :
isInt64 (TypeId(Index)) ? 8 :
isUInt64 (TypeId(Index)) ? 8 :
isFloat32(TypeId(Index)) ? 4 :
isFloat64(TypeId(Index)) ? 8 :
isFloat80(TypeId(Index)) ? 10 :
isMask8 (TypeId(Index)) ? 1 :
isMask16 (TypeId(Index)) ? 2 :
isMask32 (TypeId(Index)) ? 4 :
isMask64 (TypeId(Index)) ? 8 :
isMmx32 (TypeId(Index)) ? 4 :
isMmx64 (TypeId(Index)) ? 8 :
isVec32 (TypeId(Index)) ? 4 :
isVec64 (TypeId(Index)) ? 8 :
isVec128 (TypeId(Index)) ? 16 :
isVec256 (TypeId(Index)) ? 32 :
isVec512 (TypeId(Index)) ? 64 : 0
};
};
const TypeData _typeData = {
#define VALUE(x) TypeId(ScalarOfTypeId<x>::kTypeId)
{ ASMJIT_LOOKUP_TABLE_256(VALUE, 0) },
#undef VALUE
#define VALUE(x) SizeOfTypeId<x>::kTypeSize
{ ASMJIT_LOOKUP_TABLE_256(VALUE, 0) }
#undef VALUE
};
} // {TypeUtils}
ASMJIT_END_NAMESPACE
libraries/asmjit/asmjit/core/type.h 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#ifndef ASMJIT_CORE_TYPE_H_INCLUDED
#define ASMJIT_CORE_TYPE_H_INCLUDED
#include "../core/globals.h"
#include "../core/support.h"
ASMJIT_BEGIN_NAMESPACE
//! \addtogroup asmjit_core
//! \{
//! Type identifier provides a minimalist type system used across AsmJit library.
//!
//! This is an additional information that can be used to describe a value-type of physical or virtual register. It's
//! used mostly by BaseCompiler to describe register representation (the group of data stored in the register and the
//! width used) and it's also used by APIs that allow to describe and work with function signatures.
enum class TypeId : uint8_t {
//! Void type.
kVoid = 0,
_kBaseStart = 32,
_kBaseEnd = 44,
_kIntStart = 32,
_kIntEnd = 41,
//! Abstract signed integer type that has a native size.
kIntPtr = 32,
//! Abstract unsigned integer type that has a native size.
kUIntPtr = 33,
//! 8-bit signed integer type.
kInt8 = 34,
//! 8-bit unsigned integer type.
kUInt8 = 35,
//! 16-bit signed integer type.
kInt16 = 36,
//! 16-bit unsigned integer type.
kUInt16 = 37,
//! 32-bit signed integer type.
kInt32 = 38,
//! 32-bit unsigned integer type.
kUInt32 = 39,
//! 64-bit signed integer type.
kInt64 = 40,
//! 64-bit unsigned integer type.
kUInt64 = 41,
_kFloatStart = 42,
_kFloatEnd = 44,
//! 32-bit floating point type.
kFloat32 = 42,
//! 64-bit floating point type.
kFloat64 = 43,
//! 80-bit floating point type.
kFloat80 = 44,
_kMaskStart = 45,
_kMaskEnd = 48,
//! 8-bit opmask register (K).
kMask8 = 45,
//! 16-bit opmask register (K).
kMask16 = 46,
//! 32-bit opmask register (K).
kMask32 = 47,
//! 64-bit opmask register (K).
kMask64 = 48,
_kMmxStart = 49,
_kMmxEnd = 50,
//! 64-bit MMX register only used for 32 bits.
kMmx32 = 49,
//! 64-bit MMX register.
kMmx64 = 50,
_kVec32Start = 51,
_kVec32End = 60,
kInt8x4 = 51,
kUInt8x4 = 52,
kInt16x2 = 53,
kUInt16x2 = 54,
kInt32x1 = 55,
kUInt32x1 = 56,
kFloat32x1 = 59,
_kVec64Start = 61,
_kVec64End = 70,
kInt8x8 = 61,
kUInt8x8 = 62,
kInt16x4 = 63,
kUInt16x4 = 64,
kInt32x2 = 65,
kUInt32x2 = 66,
kInt64x1 = 67,
kUInt64x1 = 68,
kFloat32x2 = 69,
kFloat64x1 = 70,
_kVec128Start = 71,
_kVec128End = 80,
kInt8x16 = 71,
kUInt8x16 = 72,
kInt16x8 = 73,
kUInt16x8 = 74,
kInt32x4 = 75,
kUInt32x4 = 76,
kInt64x2 = 77,
kUInt64x2 = 78,
kFloat32x4 = 79,
kFloat64x2 = 80,
_kVec256Start = 81,
_kVec256End = 90,
kInt8x32 = 81,
kUInt8x32 = 82,
kInt16x16 = 83,
kUInt16x16 = 84,
kInt32x8 = 85,
kUInt32x8 = 86,
kInt64x4 = 87,
kUInt64x4 = 88,
kFloat32x8 = 89,
kFloat64x4 = 90,
_kVec512Start = 91,
_kVec512End = 100,
kInt8x64 = 91,
kUInt8x64 = 92,
kInt16x32 = 93,
kUInt16x32 = 94,
kInt32x16 = 95,
kUInt32x16 = 96,
kInt64x8 = 97,
kUInt64x8 = 98,
kFloat32x16 = 99,
kFloat64x8 = 100,
kLastAssigned = kFloat64x8,
kMaxValue = 255
};
ASMJIT_DEFINE_ENUM_COMPARE(TypeId)
//! Type identifier utilities.
namespace TypeUtils {
struct TypeData {
TypeId scalarOf[uint32_t(TypeId::kMaxValue) + 1];
uint8_t sizeOf[uint32_t(TypeId::kMaxValue) + 1];
};
ASMJIT_VARAPI const TypeData _typeData;
//! Returns the scalar type of `typeId`.
static inline TypeId scalarOf(TypeId typeId) noexcept { return _typeData.scalarOf[uint32_t(typeId)]; }
//! Returns the size [in bytes] of `typeId`.
static inline uint32_t sizeOf(TypeId typeId) noexcept { return _typeData.sizeOf[uint32_t(typeId)]; }
//! Tests whether a given type `typeId` is between `a` and `b`.
static inline constexpr bool isBetween(TypeId typeId, TypeId a, TypeId b) noexcept {
return Support::isBetween(uint32_t(typeId), uint32_t(a), uint32_t(b));
}
//! Tests whether a given type `typeId` is \ref TypeId::kVoid.
static inline constexpr bool isVoid(TypeId typeId) noexcept { return typeId == TypeId::kVoid; }
//! Tests whether a given type `typeId` is a valid non-void type.
static inline constexpr bool isValid(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kIntStart, TypeId::_kVec512End); }
//! Tests whether a given type `typeId` is scalar (has no vector part).
static inline constexpr bool isScalar(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kBaseStart, TypeId::_kBaseEnd); }
//! Tests whether a given type `typeId` is abstract, which means that its size depends on register size.
static inline constexpr bool isAbstract(TypeId typeId) noexcept { return isBetween(typeId, TypeId::kIntPtr, TypeId::kUIntPtr); }
//! Tests whether a given type is a scalar integer (signed or unsigned) of any size.
static inline constexpr bool isInt(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kIntStart, TypeId::_kIntEnd); }
//! Tests whether a given type is a scalar 8-bit integer (signed).
static inline constexpr bool isInt8(TypeId typeId) noexcept { return typeId == TypeId::kInt8; }
//! Tests whether a given type is a scalar 8-bit integer (unsigned).
static inline constexpr bool isUInt8(TypeId typeId) noexcept { return typeId == TypeId::kUInt8; }
//! Tests whether a given type is a scalar 16-bit integer (signed).
static inline constexpr bool isInt16(TypeId typeId) noexcept { return typeId == TypeId::kInt16; }
//! Tests whether a given type is a scalar 16-bit integer (unsigned).
static inline constexpr bool isUInt16(TypeId typeId) noexcept { return typeId == TypeId::kUInt16; }
//! Tests whether a given type is a scalar 32-bit integer (signed).
static inline constexpr bool isInt32(TypeId typeId) noexcept { return typeId == TypeId::kInt32; }
//! Tests whether a given type is a scalar 32-bit integer (unsigned).
static inline constexpr bool isUInt32(TypeId typeId) noexcept { return typeId == TypeId::kUInt32; }
//! Tests whether a given type is a scalar 64-bit integer (signed).
static inline constexpr bool isInt64(TypeId typeId) noexcept { return typeId == TypeId::kInt64; }
//! Tests whether a given type is a scalar 64-bit integer (unsigned).
static inline constexpr bool isUInt64(TypeId typeId) noexcept { return typeId == TypeId::kUInt64; }
static inline constexpr bool isGp8(TypeId typeId) noexcept { return isBetween(typeId, TypeId::kInt8, TypeId::kUInt8); }
static inline constexpr bool isGp16(TypeId typeId) noexcept { return isBetween(typeId, TypeId::kInt16, TypeId::kUInt16); }
static inline constexpr bool isGp32(TypeId typeId) noexcept { return isBetween(typeId, TypeId::kInt32, TypeId::kUInt32); }
static inline constexpr bool isGp64(TypeId typeId) noexcept { return isBetween(typeId, TypeId::kInt64, TypeId::kUInt64); }
//! Tests whether a given type is a scalar floating point of any size.
static inline constexpr bool isFloat(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kFloatStart, TypeId::_kFloatEnd); }
//! Tests whether a given type is a scalar 32-bit float.
static inline constexpr bool isFloat32(TypeId typeId) noexcept { return typeId == TypeId::kFloat32; }
//! Tests whether a given type is a scalar 64-bit float.
static inline constexpr bool isFloat64(TypeId typeId) noexcept { return typeId == TypeId::kFloat64; }
//! Tests whether a given type is a scalar 80-bit float.
static inline constexpr bool isFloat80(TypeId typeId) noexcept { return typeId == TypeId::kFloat80; }
static inline constexpr bool isMask(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kMaskStart, TypeId::_kMaskEnd); }
static inline constexpr bool isMask8(TypeId typeId) noexcept { return typeId == TypeId::kMask8; }
static inline constexpr bool isMask16(TypeId typeId) noexcept { return typeId == TypeId::kMask16; }
static inline constexpr bool isMask32(TypeId typeId) noexcept { return typeId == TypeId::kMask32; }
static inline constexpr bool isMask64(TypeId typeId) noexcept { return typeId == TypeId::kMask64; }
static inline constexpr bool isMmx(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kMmxStart, TypeId::_kMmxEnd); }
static inline constexpr bool isMmx32(TypeId typeId) noexcept { return typeId == TypeId::kMmx32; }
static inline constexpr bool isMmx64(TypeId typeId) noexcept { return typeId == TypeId::kMmx64; }
static inline constexpr bool isVec(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kVec32Start, TypeId::_kVec512End); }
static inline constexpr bool isVec32(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kVec32Start, TypeId::_kVec32End); }
static inline constexpr bool isVec64(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kVec64Start, TypeId::_kVec64End); }
static inline constexpr bool isVec128(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kVec128Start, TypeId::_kVec128End); }
static inline constexpr bool isVec256(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kVec256Start, TypeId::_kVec256End); }
static inline constexpr bool isVec512(TypeId typeId) noexcept { return isBetween(typeId, TypeId::_kVec512Start, TypeId::_kVec512End); }
//! \cond
enum TypeCategory : uint32_t {
kTypeCategoryUnknown = 0,
kTypeCategoryEnum = 1,
kTypeCategoryIntegral = 2,
kTypeCategoryFloatingPoint = 3,
kTypeCategoryFunction = 4
};
template<typename T, TypeCategory kCategory>
struct TypeIdOfT_ByCategory {}; // Fails if not specialized.
template<typename T>
struct TypeIdOfT_ByCategory<T, kTypeCategoryIntegral> {
enum : uint32_t {
kTypeId = uint32_t(
(sizeof(T) == 1 && std::is_signed<T>::value) ? TypeId::kInt8 :
(sizeof(T) == 1 && !std::is_signed<T>::value) ? TypeId::kUInt8 :
(sizeof(T) == 2 && std::is_signed<T>::value) ? TypeId::kInt16 :
(sizeof(T) == 2 && !std::is_signed<T>::value) ? TypeId::kUInt16 :
(sizeof(T) == 4 && std::is_signed<T>::value) ? TypeId::kInt32 :
(sizeof(T) == 4 && !std::is_signed<T>::value) ? TypeId::kUInt32 :
(sizeof(T) == 8 && std::is_signed<T>::value) ? TypeId::kInt64 :
(sizeof(T) == 8 && !std::is_signed<T>::value) ? TypeId::kUInt64 : TypeId::kVoid)
};
};
template<typename T>
struct TypeIdOfT_ByCategory<T, kTypeCategoryFloatingPoint> {
enum : uint32_t {
kTypeId = uint32_t(
(sizeof(T) == 4 ) ? TypeId::kFloat32 :
(sizeof(T) == 8 ) ? TypeId::kFloat64 :
(sizeof(T) >= 10) ? TypeId::kFloat80 : TypeId::kVoid)
};
};
template<typename T>
struct TypeIdOfT_ByCategory<T, kTypeCategoryEnum>
: public TypeIdOfT_ByCategory<typename std::underlying_type<T>::type, kTypeCategoryIntegral> {};
template<typename T>
struct TypeIdOfT_ByCategory<T, kTypeCategoryFunction> {
enum : uint32_t {
kTypeId = uint32_t(TypeId::kUIntPtr)
};
};
//! \endcond
//! TypeIdOfT<> template allows to get a TypeId from a C++ type `T`.
#ifdef _DOXYGEN
template<typename T>
struct TypeIdOfT {
//! TypeId of C++ type `T`.
static constexpr TypeId kTypeId = _TypeIdDeducedAtCompileTime_;
};
#else
template<typename T>
struct TypeIdOfT
: public TypeIdOfT_ByCategory<T,
std::is_enum<T>::value ? kTypeCategoryEnum :
std::is_integral<T>::value ? kTypeCategoryIntegral :
std::is_floating_point<T>::value ? kTypeCategoryFloatingPoint :
std::is_function<T>::value ? kTypeCategoryFunction : kTypeCategoryUnknown> {};
#endif
//! \cond
template<typename T>
struct TypeIdOfT<T*> {
enum : uint32_t {
kTypeId = uint32_t(TypeId::kUIntPtr)
};
};
template<typename T>
struct TypeIdOfT<T&> {
enum : uint32_t {
kTypeId = uint32_t(TypeId::kUIntPtr)
};
};
//! \endcond
//! Returns a corresponding \ref TypeId of `T` type.
template<typename T>
static inline constexpr TypeId typeIdOfT() noexcept { return TypeId(TypeIdOfT<T>::kTypeId); }
//! Returns offset needed to convert a `kIntPtr` and `kUIntPtr` TypeId into a type that matches `registerSize`
//! (general-purpose register size). If you find such TypeId it's then only about adding the offset to it.
//!
//! For example:
//!
//! ```
//! uint32_t registerSize = /* 4 or 8 */;
//! uint32_t deabstractDelta = TypeUtils::deabstractDeltaOfSize(registerSize);
//!
//! TypeId typeId = 'some type-id';
//!
//! // Normalize some typeId into a non-abstract typeId.
//! if (TypeUtils::isAbstract(typeId)) typeId += deabstractDelta;
//!
//! // The same, but by using TypeUtils::deabstract() function.
//! typeId = TypeUtils::deabstract(typeId, deabstractDelta);
//! ```
static inline constexpr uint32_t deabstractDeltaOfSize(uint32_t registerSize) noexcept {
return registerSize >= 8 ? uint32_t(TypeId::kInt64) - uint32_t(TypeId::kIntPtr)
: uint32_t(TypeId::kInt32) - uint32_t(TypeId::kIntPtr);
}
//! Deabstracts a given `typeId` into a native type by using `deabstractDelta`, which was previously
//! calculated by calling \ref deabstractDeltaOfSize() with a target native register size.
static inline constexpr TypeId deabstract(TypeId typeId, uint32_t deabstractDelta) noexcept {
return isAbstract(typeId) ? TypeId(uint32_t(typeId) + deabstractDelta) : typeId;
}
static inline constexpr TypeId scalarToVector(TypeId scalarTypeId, TypeId vecStartId) noexcept {
return TypeId(uint32_t(vecStartId) + uint32_t(scalarTypeId) - uint32_t(TypeId::kInt8));
}
} // {TypeUtils}
//! Provides type identifiers that can be used in templates instead of native types.
namespace Type {
//! bool as C++ type-name.
struct Bool {};
//! int8_t as C++ type-name.
struct Int8 {};
//! uint8_t as C++ type-name.
struct UInt8 {};
//! int16_t as C++ type-name.
struct Int16 {};
//! uint16_t as C++ type-name.
struct UInt16 {};
//! int32_t as C++ type-name.
struct Int32 {};
//! uint32_t as C++ type-name.
struct UInt32 {};
//! int64_t as C++ type-name.
struct Int64 {};
//! uint64_t as C++ type-name.
struct UInt64 {};
//! intptr_t as C++ type-name.
struct IntPtr {};
//! uintptr_t as C++ type-name.
struct UIntPtr {};
//! float as C++ type-name.
struct Float32 {};
//! double as C++ type-name.
struct Float64 {};
} // {Type}
//! \cond
#define ASMJIT_DEFINE_TYPE_ID(T, TYPE_ID) \
namespace TypeUtils { \
template<> \
struct TypeIdOfT<T> { \
enum : uint32_t { \
kTypeId = uint32_t(TYPE_ID) \
}; \
}; \
}
ASMJIT_DEFINE_TYPE_ID(void , TypeId::kVoid);
ASMJIT_DEFINE_TYPE_ID(Type::Bool , TypeId::kUInt8);
ASMJIT_DEFINE_TYPE_ID(Type::Int8 , TypeId::kInt8);
ASMJIT_DEFINE_TYPE_ID(Type::UInt8 , TypeId::kUInt8);
ASMJIT_DEFINE_TYPE_ID(Type::Int16 , TypeId::kInt16);
ASMJIT_DEFINE_TYPE_ID(Type::UInt16 , TypeId::kUInt16);
ASMJIT_DEFINE_TYPE_ID(Type::Int32 , TypeId::kInt32);
ASMJIT_DEFINE_TYPE_ID(Type::UInt32 , TypeId::kUInt32);
ASMJIT_DEFINE_TYPE_ID(Type::Int64 , TypeId::kInt64);
ASMJIT_DEFINE_TYPE_ID(Type::UInt64 , TypeId::kUInt64);
ASMJIT_DEFINE_TYPE_ID(Type::IntPtr , TypeId::kIntPtr);
ASMJIT_DEFINE_TYPE_ID(Type::UIntPtr, TypeId::kUIntPtr);
ASMJIT_DEFINE_TYPE_ID(Type::Float32, TypeId::kFloat32);
ASMJIT_DEFINE_TYPE_ID(Type::Float64, TypeId::kFloat64);
//! \endcond
//! \}
ASMJIT_END_NAMESPACE
#endif // ASMJIT_CORE_TYPE_H_INCLUDED
libraries/asmjit/asmjit/core/virtmem.cpp 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
#ifndef ASMJIT_NO_JIT
#include "../core/osutils.h"
#include "../core/string.h"
#include "../core/support.h"
#include "../core/virtmem.h"
#if !defined(_WIN32)
#include <errno.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
// Linux has a `memfd_create` syscall that we would like to use, if available.
#if defined(__linux__)
#include <sys/syscall.h>
#endif
// Apple recently introduced MAP_JIT flag, which we want to use.
#if defined(__APPLE__)
#include <pthread.h>
#include <TargetConditionals.h>
#if TARGET_OS_OSX
#include <sys/utsname.h>
#include <libkern/OSCacheControl.h> // sys_icache_invalidate().
#endif
// Older SDK doesn't define `MAP_JIT`.
#ifndef MAP_JIT
#define MAP_JIT 0x800
#endif
#endif
// BSD/MAC: `MAP_ANONYMOUS` is not defined, `MAP_ANON` is.
#if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON
#endif
#endif
#include <atomic>
#if defined(__APPLE__) || defined(__BIONIC__)
#define ASMJIT_VM_SHM_DETECT 0
#else
#define ASMJIT_VM_SHM_DETECT 1
#endif
// Android NDK doesn't provide `shm_open()` and `shm_unlink()`.
#if defined(__BIONIC__)
#define ASMJIT_VM_SHM_AVAILABLE 0
#else
#define ASMJIT_VM_SHM_AVAILABLE 1
#endif
#if defined(__APPLE__) && ASMJIT_ARCH_ARM >= 64
#define ASMJIT_HAS_PTHREAD_JIT_WRITE_PROTECT_NP
#endif
ASMJIT_BEGIN_SUB_NAMESPACE(VirtMem)
// Virtual Memory Utilities
// ========================
static const MemoryFlags dualMappingFilter[2] = {
MemoryFlags::kAccessWrite | MemoryFlags::kMMapMaxAccessWrite,
MemoryFlags::kAccessExecute | MemoryFlags::kMMapMaxAccessExecute
};
// Virtual Memory [Windows]
// ========================
#if defined(_WIN32)
struct ScopedHandle {
inline ScopedHandle() noexcept
: value(nullptr) {}
inline ~ScopedHandle() noexcept {
if (value != nullptr)
::CloseHandle(value);
}
HANDLE value;
};
static void getVMInfo(Info& vmInfo) noexcept {
SYSTEM_INFO systemInfo;
::GetSystemInfo(&systemInfo);
vmInfo.pageSize = Support::alignUpPowerOf2<uint32_t>(systemInfo.dwPageSize);
vmInfo.pageGranularity = systemInfo.dwAllocationGranularity;
}
// Returns windows-specific protectFlags from \ref MemoryFlags.
static DWORD protectFlagsFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
DWORD protectFlags;
// READ|WRITE|EXECUTE.
if (Support::test(memoryFlags, MemoryFlags::kAccessExecute))
protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
else if (Support::test(memoryFlags, MemoryFlags::kAccessRW))
protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_READWRITE : PAGE_READONLY;
else
protectFlags = PAGE_NOACCESS;
// Any other flags to consider?
return protectFlags;
}
static DWORD desiredAccessFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
DWORD access = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? FILE_MAP_WRITE : FILE_MAP_READ;
if (Support::test(memoryFlags, MemoryFlags::kAccessExecute))
access |= FILE_MAP_EXECUTE;
return access;
}
static HardenedRuntimeFlags getHardenedRuntimeFlags() noexcept {
return HardenedRuntimeFlags::kNone;
}
Error alloc(void** p, size_t size, MemoryFlags memoryFlags) noexcept {
*p = nullptr;
if (size == 0)
return DebugUtils::errored(kErrorInvalidArgument);
DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags);
void* result = ::VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, protectFlags);
if (!result)
return DebugUtils::errored(kErrorOutOfMemory);
*p = result;
return kErrorOk;
}
Error release(void* p, size_t size) noexcept {
DebugUtils::unused(size);
if (ASMJIT_UNLIKELY(!::VirtualFree(p, 0, MEM_RELEASE)))
return DebugUtils::errored(kErrorInvalidArgument);
return kErrorOk;
}
Error protect(void* p, size_t size, MemoryFlags memoryFlags) noexcept {
DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags);
DWORD oldFlags;
if (::VirtualProtect(p, size, protectFlags, &oldFlags))
return kErrorOk;
return DebugUtils::errored(kErrorInvalidArgument);
}
Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept {
dm->rx = nullptr;
dm->rw = nullptr;
if (size == 0)
return DebugUtils::errored(kErrorInvalidArgument);
ScopedHandle handle;
handle.value = ::CreateFileMappingW(
INVALID_HANDLE_VALUE,
nullptr,
PAGE_EXECUTE_READWRITE,
(DWORD)(uint64_t(size) >> 32),
(DWORD)(size & 0xFFFFFFFFu),
nullptr);
if (ASMJIT_UNLIKELY(!handle.value))
return DebugUtils::errored(kErrorOutOfMemory);
void* ptr[2];
for (uint32_t i = 0; i < 2; i++) {
MemoryFlags accessFlags = memoryFlags & ~dualMappingFilter[i];
DWORD desiredAccess = desiredAccessFromMemoryFlags(accessFlags);
ptr[i] = ::MapViewOfFile(handle.value, desiredAccess, 0, 0, size);
if (ptr[i] == nullptr) {
if (i == 0)
::UnmapViewOfFile(ptr[0]);
return DebugUtils::errored(kErrorOutOfMemory);
}
}
dm->rx = ptr[0];
dm->rw = ptr[1];
return kErrorOk;
}
Error releaseDualMapping(DualMapping* dm, size_t size) noexcept {
DebugUtils::unused(size);
bool failed = false;
if (!::UnmapViewOfFile(dm->rx))
failed = true;
if (dm->rx != dm->rw && !UnmapViewOfFile(dm->rw))
failed = true;
if (failed)
return DebugUtils::errored(kErrorInvalidArgument);
dm->rx = nullptr;
dm->rw = nullptr;
return kErrorOk;
}
#endif
// Virtual Memory [Posix]
// ======================
#if !defined(_WIN32)
static void getVMInfo(Info& vmInfo) noexcept {
uint32_t pageSize = uint32_t(::getpagesize());
vmInfo.pageSize = pageSize;
vmInfo.pageGranularity = Support::max<uint32_t>(pageSize, 65536);
}
#if !defined(SHM_ANON)
static const char* getTmpDir() noexcept {
const char* tmpDir = getenv("TMPDIR");
return tmpDir ? tmpDir : "/tmp";
}
#endif
// Translates libc errors specific to VirtualMemory mapping to `asmjit::Error`.
static Error asmjitErrorFromErrno(int e) noexcept {
switch (e) {
case EACCES:
case EAGAIN:
case ENODEV:
case EPERM:
return kErrorInvalidState;
case EFBIG:
case ENOMEM:
case EOVERFLOW:
return kErrorOutOfMemory;
case EMFILE:
case ENFILE:
return kErrorTooManyHandles;
default:
return kErrorInvalidArgument;
}
}
// Some operating systems don't allow /dev/shm to be executable. On Linux this happens when /dev/shm is mounted with
// 'noexec', which is enforced by systemd. Other operating systems like MacOS also restrict executable permissions
// regarding /dev/shm, so we use a runtime detection before attempting to allocate executable memory. Sometimes we
// don't need the detection as we know it would always result in `ShmStrategy::kTmpDir`.
enum class ShmStrategy : uint32_t {
kUnknown = 0,
kDevShm = 1,
kTmpDir = 2
};
class AnonymousMemory {
public:
enum FileType : uint32_t {
kFileTypeNone,
kFileTypeShm,
kFileTypeTmp
};
int _fd;
FileType _fileType;
StringTmp<128> _tmpName;
inline AnonymousMemory() noexcept
: _fd(-1),
_fileType(kFileTypeNone),
_tmpName() {}
inline ~AnonymousMemory() noexcept {
unlink();
close();
}
inline int fd() const noexcept { return _fd; }
Error open(bool preferTmpOverDevShm) noexcept {
#if defined(__linux__) && defined(__NR_memfd_create)
// Linux specific 'memfd_create' - if the syscall returns `ENOSYS` it means
// it's not available and we will never call it again (would be pointless).
// Zero initialized, if ever changed to '1' that would mean the syscall is not
// available and we must use `shm_open()` and `shm_unlink()`.
static volatile uint32_t memfd_create_not_supported;
if (!memfd_create_not_supported) {
_fd = (int)syscall(__NR_memfd_create, "vmem", 0);
if (ASMJIT_LIKELY(_fd >= 0))
return kErrorOk;
int e = errno;
if (e == ENOSYS)
memfd_create_not_supported = 1;
else
return DebugUtils::errored(asmjitErrorFromErrno(e));
}
#endif
#if defined(SHM_ANON)
// Originally FreeBSD extension, apparently works in other BSDs too.
DebugUtils::unused(preferTmpOverDevShm);
_fd = ::shm_open(SHM_ANON, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
if (ASMJIT_LIKELY(_fd >= 0))
return kErrorOk;
else
return DebugUtils::errored(asmjitErrorFromErrno(errno));
#else
// POSIX API. We have to generate somehow a unique name. This is nothing cryptographic, just using a bit from
// the stack address to always have a different base for different threads (as threads have their own stack)
// and retries for avoiding collisions. We use `shm_open()` with flags that require creation of the file so we
// never open an existing shared memory.
static std::atomic<uint32_t> internalCounter;
const char* kShmFormat = "/shm-id-%016llX";
uint32_t kRetryCount = 100;
uint64_t bits = ((uintptr_t)(void*)this) & 0x55555555u;
for (uint32_t i = 0; i < kRetryCount; i++) {
bits -= uint64_t(OSUtils::getTickCount()) * 773703683;
bits = ((bits >> 14) ^ (bits << 6)) + uint64_t(++internalCounter) * 10619863;
bool useTmp = !ASMJIT_VM_SHM_DETECT || preferTmpOverDevShm;
if (useTmp) {
_tmpName.assign(getTmpDir());
_tmpName.appendFormat(kShmFormat, (unsigned long long)bits);
_fd = ::open(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, 0);
if (ASMJIT_LIKELY(_fd >= 0)) {
_fileType = kFileTypeTmp;
return kErrorOk;
}
}
#if ASMJIT_VM_SHM_AVAILABLE
else {
_tmpName.assignFormat(kShmFormat, (unsigned long long)bits);
_fd = ::shm_open(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
if (ASMJIT_LIKELY(_fd >= 0)) {
_fileType = kFileTypeShm;
return kErrorOk;
}
}
#endif
int e = errno;
if (e != EEXIST)
return DebugUtils::errored(asmjitErrorFromErrno(e));
}
return DebugUtils::errored(kErrorFailedToOpenAnonymousMemory);
#endif
}
void unlink() noexcept {
FileType type = _fileType;
_fileType = kFileTypeNone;
#if ASMJIT_VM_SHM_AVAILABLE
if (type == kFileTypeShm) {
::shm_unlink(_tmpName.data());
return;
}
#endif
if (type == kFileTypeTmp) {
::unlink(_tmpName.data());
return;
}
}
void close() noexcept {
if (_fd >= 0) {
::close(_fd);
_fd = -1;
}
}
Error allocate(size_t size) noexcept {
// TODO: Improve this by using `posix_fallocate()` when available.
if (ftruncate(_fd, off_t(size)) != 0)
return DebugUtils::errored(asmjitErrorFromErrno(errno));
return kErrorOk;
}
};
// Returns `mmap()` protection flags from \ref MemoryFlags.
static int mmProtFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
int protection = 0;
if (Support::test(memoryFlags, MemoryFlags::kAccessRead)) protection |= PROT_READ;
if (Support::test(memoryFlags, MemoryFlags::kAccessWrite)) protection |= PROT_READ | PROT_WRITE;
if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) protection |= PROT_READ | PROT_EXEC;
return protection;
}
#if defined(__APPLE__)
// Detects whether the current process is hardened, which means that pages that have WRITE and EXECUTABLE flags cannot
// be allocated without MAP_JIT flag.
static inline bool hasHardenedRuntimeMacOS() noexcept {
#if TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64
// MacOS on AArch64 has always hardened runtime enabled.
return true;
#else
static std::atomic<uint32_t> globalHardenedFlag;
enum HardenedFlag : uint32_t {
kHardenedFlagUnknown = 0,
kHardenedFlagDisabled = 1,
kHardenedFlagEnabled = 2
};
uint32_t flag = globalHardenedFlag.load();
if (flag == kHardenedFlagUnknown) {
size_t pageSize = ::getpagesize();
void* ptr = mmap(nullptr, pageSize, PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (ptr == MAP_FAILED) {
flag = kHardenedFlagEnabled;
}
else {
flag = kHardenedFlagDisabled;
munmap(ptr, pageSize);
}
globalHardenedFlag.store(flag);
}
return flag == kHardenedFlagEnabled;
#endif
}
static inline bool hasMapJitSupportMacOS() noexcept {
#if TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64
// MacOS for 64-bit AArch architecture always uses hardened runtime. Some documentation can be found here:
// - https://developer.apple.com/documentation/apple_silicon/porting_just-in-time_compilers_to_apple_silicon
return true;
#elif TARGET_OS_OSX
// MAP_JIT flag required to run unsigned JIT code is only supported by kernel version 10.14+ (Mojave) and IOS.
static std::atomic<uint32_t> globalVersion;
int ver = globalVersion.load();
if (!ver) {
struct utsname osname {};
uname(&osname);
ver = atoi(osname.release);
globalVersion.store(ver);
}
return ver >= 18;
#else
// Assume it's available.
return true;
#endif
}
#endif // __APPLE__
// Detects whether the current process is hardened, which means that pages that have WRITE and EXECUTABLE flags
// cannot be normally allocated. On MacOS such allocation requires MAP_JIT flag.
static inline bool hasHardenedRuntime() noexcept {
#if defined(__APPLE__)
return hasHardenedRuntimeMacOS();
#else
return false;
#endif
}
// Detects whether MAP_JIT is available.
static inline bool hasMapJitSupport() noexcept {
#if defined(__APPLE__)
return hasMapJitSupportMacOS();
#else
return false;
#endif
}
// Returns either MAP_JIT or 0 based on `flags` and the host operating system.
static inline int mmMapJitFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
#if defined(__APPLE__)
// Always use MAP_JIT flag if user asked for it (could be used for testing on non-hardened processes) and detect
// whether it must be used when the process is actually hardened (in that case it doesn't make sense to rely on
// user `memoryFlags`).
bool useMapJit = Support::test(memoryFlags, MemoryFlags::kMMapEnableMapJit) || hasHardenedRuntime();
if (useMapJit)
return hasMapJitSupport() ? int(MAP_JIT) : 0;
else
return 0;
#else
DebugUtils::unused(memoryFlags);
return 0;
#endif
}
// Returns BSD-specific `PROT_MAX()` flags.
static inline int mmMaxProtFromMemoryFlags(MemoryFlags memoryFlags) noexcept {
#if defined(PROT_MAX)
static constexpr uint32_t kMaxProtShift = Support::ConstCTZ<uint32_t(MemoryFlags::kMMapMaxAccessRead)>::value;
if (Support::test(memoryFlags, MemoryFlags::kMMapMaxAccessReadWrite | MemoryFlags::kMMapMaxAccessExecute))
return PROT_MAX(mmProtFromMemoryFlags((MemoryFlags)(uint32_t(memoryFlags) >> kMaxProtShift)));
else
return 0;
#else
DebugUtils::unused(memoryFlags);
return 0;
#endif
}
#if ASMJIT_VM_SHM_DETECT
static Error detectShmStrategy(ShmStrategy* strategyOut) noexcept {
AnonymousMemory anonMem;
Info vmInfo = info();
ASMJIT_PROPAGATE(anonMem.open(false));
ASMJIT_PROPAGATE(anonMem.allocate(vmInfo.pageSize));
void* ptr = mmap(nullptr, vmInfo.pageSize, PROT_READ | PROT_EXEC, MAP_SHARED, anonMem.fd(), 0);
if (ptr == MAP_FAILED) {
int e = errno;
if (e == EINVAL) {
*strategyOut = ShmStrategy::kTmpDir;
return kErrorOk;
}
return DebugUtils::errored(asmjitErrorFromErrno(e));
}
else {
munmap(ptr, vmInfo.pageSize);
*strategyOut = ShmStrategy::kDevShm;
return kErrorOk;
}
}
#endif
static Error getShmStrategy(ShmStrategy* strategyOut) noexcept {
#if ASMJIT_VM_SHM_DETECT
// Initially don't assume anything. It has to be tested whether '/dev/shm' was mounted with 'noexec' flag or not.
static std::atomic<uint32_t> globalShmStrategy;
ShmStrategy strategy = static_cast<ShmStrategy>(globalShmStrategy.load());
if (strategy == ShmStrategy::kUnknown) {
ASMJIT_PROPAGATE(detectShmStrategy(&strategy));
globalShmStrategy.store(static_cast<uint32_t>(strategy));
}
*strategyOut = strategy;
return kErrorOk;
#else
*strategyOut = ShmStrategy::kTmpDir;
return kErrorOk;
#endif
}
static HardenedRuntimeFlags getHardenedRuntimeFlags() noexcept {
HardenedRuntimeFlags hrFlags = HardenedRuntimeFlags::kNone;
if (hasHardenedRuntime())
hrFlags |= HardenedRuntimeFlags::kEnabled;
if (hasMapJitSupport())
hrFlags |= HardenedRuntimeFlags::kMapJit;
return hrFlags;
}
Error alloc(void** p, size_t size, MemoryFlags memoryFlags) noexcept {
*p = nullptr;
if (size == 0)
return DebugUtils::errored(kErrorInvalidArgument);
int protection = mmProtFromMemoryFlags(memoryFlags) | mmMaxProtFromMemoryFlags(memoryFlags);
int mmFlags = MAP_PRIVATE | MAP_ANONYMOUS | mmMapJitFromMemoryFlags(memoryFlags);
void* ptr = mmap(nullptr, size, protection, mmFlags, -1, 0);
if (ptr == MAP_FAILED)
return DebugUtils::errored(kErrorOutOfMemory);
*p = ptr;
return kErrorOk;
}
Error release(void* p, size_t size) noexcept {
if (ASMJIT_UNLIKELY(munmap(p, size) != 0))
return DebugUtils::errored(kErrorInvalidArgument);
return kErrorOk;
}
Error protect(void* p, size_t size, MemoryFlags memoryFlags) noexcept {
int protection = mmProtFromMemoryFlags(memoryFlags);
if (mprotect(p, size, protection) == 0)
return kErrorOk;
return DebugUtils::errored(kErrorInvalidArgument);
}
Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags memoryFlags) noexcept {
dm->rx = nullptr;
dm->rw = nullptr;
if (off_t(size) <= 0)
return DebugUtils::errored(size == 0 ? kErrorInvalidArgument : kErrorTooLarge);
bool preferTmpOverDevShm = Support::test(memoryFlags, MemoryFlags::kMappingPreferTmp);
if (!preferTmpOverDevShm) {
ShmStrategy strategy;
ASMJIT_PROPAGATE(getShmStrategy(&strategy));
preferTmpOverDevShm = (strategy == ShmStrategy::kTmpDir);
}
AnonymousMemory anonMem;
ASMJIT_PROPAGATE(anonMem.open(preferTmpOverDevShm));
ASMJIT_PROPAGATE(anonMem.allocate(size));
void* ptr[2];
for (uint32_t i = 0; i < 2; i++) {
MemoryFlags accessFlags = memoryFlags & ~dualMappingFilter[i];
int protection = mmProtFromMemoryFlags(accessFlags) | mmMaxProtFromMemoryFlags(accessFlags);
ptr[i] = mmap(nullptr, size, protection, MAP_SHARED, anonMem.fd(), 0);
if (ptr[i] == MAP_FAILED) {
// Get the error now before `munmap()` has a chance to clobber it.
int e = errno;
if (i == 1)
munmap(ptr[0], size);
return DebugUtils::errored(asmjitErrorFromErrno(e));
}
}
dm->rx = ptr[0];
dm->rw = ptr[1];
return kErrorOk;
}
Error releaseDualMapping(DualMapping* dm, size_t size) noexcept {
Error err = release(dm->rx, size);
if (dm->rx != dm->rw)
err |= release(dm->rw, size);
if (err)
return DebugUtils::errored(kErrorInvalidArgument);
dm->rx = nullptr;
dm->rw = nullptr;
return kErrorOk;
}
#endif
// Virtual Memory - Flush Instruction Cache
// ========================================
void flushInstructionCache(void* p, size_t size) noexcept {
#if ASMJIT_ARCH_X86
// X86/X86_64 architecture doesn't require to do anything to flush instruction cache.
DebugUtils::unused(p, size);
#elif defined(__APPLE__)
sys_icache_invalidate(p, size);
#elif defined(_WIN32)
// Windows has a built-in support in `kernel32.dll`.
FlushInstructionCache(GetCurrentProcess(), p, size);
#elif defined(__GNUC__)
char* start = static_cast<char*>(p);
char* end = start + size;
__builtin___clear_cache(start, end);
#else
#pragma message("asmjit::VirtMem::flushInstructionCache() doesn't have implementation for the target OS and compiler")
DebugUtils::unused(p, size);
#endif
}
// Virtual Memory - Memory Info
// ============================
Info info() noexcept {
static std::atomic<uint32_t> vmInfoInitialized;
static Info vmInfo;
if (!vmInfoInitialized.load()) {
Info localMemInfo;
getVMInfo(localMemInfo);
vmInfo = localMemInfo;
vmInfoInitialized.store(1u);
}
return vmInfo;
}
// Virtual Memory - Hardened Runtime Info
// ======================================
HardenedRuntimeInfo hardenedRuntimeInfo() noexcept {
return HardenedRuntimeInfo { getHardenedRuntimeFlags() };
}
// Virtual Memory - Project JIT Memory
// ===================================
void protectJitMemory(ProtectJitAccess access) noexcept {
#if defined(ASMJIT_HAS_PTHREAD_JIT_WRITE_PROTECT_NP)
pthread_jit_write_protect_np(static_cast<uint32_t>(access));
#else
DebugUtils::unused(access);
#endif
}
ASMJIT_END_SUB_NAMESPACE
#endif
libraries/asmjit/asmjit/core/virtmem.h 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#ifndef ASMJIT_CORE_VIRTMEM_H_INCLUDED
#define ASMJIT_CORE_VIRTMEM_H_INCLUDED
#include "../core/api-config.h"
#ifndef ASMJIT_NO_JIT
#include "../core/globals.h"
ASMJIT_BEGIN_NAMESPACE
//! \addtogroup asmjit_virtual_memory
//! \{
//! Virtual memory management.
namespace VirtMem {
//! Flushes instruction cache in the given region.
//!
//! Only useful on non-x86 architectures, however, it's a good practice to call it on any platform to make your
//! code more portable.
ASMJIT_API void flushInstructionCache(void* p, size_t size) noexcept;
//! Virtual memory information.
struct Info {
//! Virtual memory page size.
uint32_t pageSize;
//! Virtual memory page granularity.
uint32_t pageGranularity;
};
//! Returns virtual memory information, see `VirtMem::Info` for more details.
ASMJIT_API Info info() noexcept;
//! Virtual memory access and mmap-specific flags.
enum class MemoryFlags : uint32_t {
//! No flags.
kNone = 0,
//! Memory is readable.
kAccessRead = 0x00000001u,
//! Memory is writable.
kAccessWrite = 0x00000002u,
//! Memory is executable.
kAccessExecute = 0x00000004u,
//! A combination of \ref MemoryFlags::kAccessRead and \ref MemoryFlags::kAccessWrite.
kAccessReadWrite = kAccessRead | kAccessWrite,
//! A combination of \ref MemoryFlags::kAccessRead, \ref MemoryFlags::kAccessWrite.
kAccessRW = kAccessRead | kAccessWrite,
//! A combination of \ref MemoryFlags::kAccessRead and \ref MemoryFlags::kAccessExecute.
kAccessRX = kAccessRead | kAccessExecute,
//! A combination of \ref MemoryFlags::kAccessRead, \ref MemoryFlags::kAccessWrite, and
//! \ref MemoryFlags::kAccessExecute.
kAccessRWX = kAccessRead | kAccessWrite | kAccessExecute,
//! Use a `MAP_JIT` flag available on Apple platforms (introduced by Mojave), which allows JIT code to be executed
//! in MAC bundles. This flag is not turned on by default, because when a process uses `fork()` the child process
//! has no access to the pages mapped with `MAP_JIT`, which could break code that doesn't expect this behavior.
//!
//! \note This flag can only be used with \ref VirtMem::alloc().
kMMapEnableMapJit = 0x00000010u,
//! Pass `PROT_MAX(PROT_READ)` to mmap() on platforms that support `PROT_MAX`.
//!
//! \note This flag can only be used with \ref VirtMem::alloc().
kMMapMaxAccessRead = 0x00000020u,
//! Pass `PROT_MAX(PROT_WRITE)` to mmap() on platforms that support `PROT_MAX`.
//!
//! \note This flag can only be used with \ref VirtMem::alloc().
kMMapMaxAccessWrite = 0x00000040u,
//! Pass `PROT_MAX(PROT_EXEC)` to mmap() on platforms that support `PROT_MAX`.
//!
//! \note This flag can only be used with \ref VirtMem::alloc().
kMMapMaxAccessExecute = 0x00000080u,
//! A combination of \ref MemoryFlags::kMMapMaxAccessRead and \ref MemoryFlags::kMMapMaxAccessWrite.
kMMapMaxAccessReadWrite = kMMapMaxAccessRead | kMMapMaxAccessWrite,
//! A combination of \ref MemoryFlags::kMMapMaxAccessRead and \ref MemoryFlags::kMMapMaxAccessWrite.
kMMapMaxAccessRW = kMMapMaxAccessRead | kMMapMaxAccessWrite,
//! A combination of \ref MemoryFlags::kMMapMaxAccessRead and \ref MemoryFlags::kMMapMaxAccessExecute.
kMMapMaxAccessRX = kMMapMaxAccessRead | kMMapMaxAccessExecute,
//! A combination of \ref MemoryFlags::kMMapMaxAccessRead, \ref MemoryFlags::kMMapMaxAccessWrite, \ref
//! MemoryFlags::kMMapMaxAccessExecute.
kMMapMaxAccessRWX = kMMapMaxAccessRead | kMMapMaxAccessWrite | kMMapMaxAccessExecute,
//! Not an access flag, only used by `allocDualMapping()` to override the default allocation strategy to always use
//! a 'tmp' directory instead of "/dev/shm" (on POSIX platforms). Please note that this flag will be ignored if the
//! operating system allows to allocate an executable memory by a different API than `open()` or `shm_open()`. For
//! example on Linux `memfd_create()` is preferred and on BSDs `shm_open(SHM_ANON, ...)` is used if SHM_ANON is
//! defined.
//!
//! \note This flag can only be used with \ref VirtMem::alloc().
kMappingPreferTmp = 0x80000000u
};
ASMJIT_DEFINE_ENUM_FLAGS(MemoryFlags)
//! Allocates virtual memory by either using `mmap()` (POSIX) or `VirtualAlloc()` (Windows).
//!
//! \note `size` should be aligned to page size, use \ref VirtMem::info() to obtain it. Invalid size will not be
//! corrected by the implementation and the allocation would not succeed in such case.
ASMJIT_API Error alloc(void** p, size_t size, MemoryFlags flags) noexcept;
//! Releases virtual memory previously allocated by \ref VirtMem::alloc().
//!
//! \note The size must be the same as used by \ref VirtMem::alloc(). If the size is not the same value the call
//! will fail on any POSIX system, but pass on Windows, because it's implemented differently.
ASMJIT_API Error release(void* p, size_t size) noexcept;
//! A cross-platform wrapper around `mprotect()` (POSIX) and `VirtualProtect()` (Windows).
ASMJIT_API Error protect(void* p, size_t size, MemoryFlags flags) noexcept;
//! Dual memory mapping used to map an anonymous memory into two memory regions where one region is read-only, but
//! executable, and the second region is read+write, but not executable. See \ref VirtMem::allocDualMapping() for
//! more details.
struct DualMapping {
//! Pointer to data with 'Read+Execute' access (this memory is not writable).
void* rx;
//! Pointer to data with 'Read+Write' access (this memory is not executable).
void* rw;
};
//! Allocates virtual memory and creates two views of it where the first view has no write access. This is an addition
//! to the API that should be used in cases in which the operating system either enforces W^X security policy or the
//! application wants to use this policy by default to improve security and prevent an accidental (or purposed)
//! self-modifying code.
//!
//! The memory returned in the `dm` are two independent mappings of the same shared memory region. You must use
//! \ref VirtMem::releaseDualMapping() to release it when it's no longer needed. Never use `VirtMem::release()` to
//! release the memory returned by `allocDualMapping()` as that would fail on Windows.
//!
//! \remarks Both pointers in `dm` would be set to `nullptr` if the function fails.
ASMJIT_API Error allocDualMapping(DualMapping* dm, size_t size, MemoryFlags flags) noexcept;
//! Releases virtual memory mapping previously allocated by \ref VirtMem::allocDualMapping().
//!
//! \remarks Both pointers in `dm` would be set to `nullptr` if the function succeeds.
ASMJIT_API Error releaseDualMapping(DualMapping* dm, size_t size) noexcept;
//! Hardened runtime flags.
enum class HardenedRuntimeFlags : uint32_t {
//! No flags.
kNone = 0,
//! Hardened runtime is enabled - it's not possible to have "Write & Execute" memory protection. The runtime
//! enforces W^X (either write or execute).
//!
//! \note If the runtime is hardened it means that an operating system specific protection is used. For example on
//! MacOS platform it's possible to allocate memory with MAP_JIT flag and then use `pthread_jit_write_protect_np()`
//! to temporarily swap access permissions for the current thread. Dual mapping is also a possibility on X86/X64
//! architecture.
kEnabled = 0x00000001u,
//! Read+Write+Execute can only be allocated with MAP_JIT flag (Apple specific).
kMapJit = 0x00000002u
};
ASMJIT_DEFINE_ENUM_FLAGS(HardenedRuntimeFlags)
//! Hardened runtime information.
struct HardenedRuntimeInfo {
//! Hardened runtime flags.
HardenedRuntimeFlags flags;
};
//! Returns runtime features provided by the OS.
ASMJIT_API HardenedRuntimeInfo hardenedRuntimeInfo() noexcept;
//! Values that can be used with `protectJitMemory()` function.
enum class ProtectJitAccess : uint32_t {
//! Protect JIT memory with Read+Write permissions.
kReadWrite = 0,
//! Protect JIT memory with Read+Execute permissions.
kReadExecute = 1
};
//! Protects access of memory mapped with MAP_JIT flag for the current thread.
//!
//! \note This feature is only available on Apple hardware (AArch64) at the moment and and uses a non-portable
//! `pthread_jit_write_protect_np()` call when available.
//!
//! This function must be called before and after a memory mapped with MAP_JIT flag is modified. Example:
//!
//! ```
//! void* codePtr = ...;
//! size_t codeSize = ...;
//!
//! VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadWrite);
//! memcpy(codePtr, source, codeSize);
//! VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadExecute);
//! VirtMem::flushInstructionCache(codePtr, codeSize);
//! ```
//!
//! See \ref ProtectJitReadWriteScope, which makes it simpler than the code above.
ASMJIT_API void protectJitMemory(ProtectJitAccess access) noexcept;
//! JIT protection scope that prepares the given memory block to be written to in the current thread.
//!
//! It calls `VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadWrite)` at construction time and
//! `VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadExecute)` combined with `flushInstructionCache()`
//! in destructor. The purpose of this class is to make writing to JIT memory easier.
class ProtectJitReadWriteScope {
public:
void* _rxPtr;
size_t _size;
//! Makes the given memory block RW protected.
ASMJIT_FORCE_INLINE ProtectJitReadWriteScope(void* rxPtr, size_t size) noexcept
: _rxPtr(rxPtr),
_size(size) {
protectJitMemory(ProtectJitAccess::kReadWrite);
}
// Not copyable.
ProtectJitReadWriteScope(const ProtectJitReadWriteScope& other) = delete;
//! Makes the memory block RX protected again and flushes instruction cache.
ASMJIT_FORCE_INLINE ~ProtectJitReadWriteScope() noexcept {
protectJitMemory(ProtectJitAccess::kReadExecute);
flushInstructionCache(_rxPtr, _size);
}
};
} // VirtMem
//! \}
ASMJIT_END_NAMESPACE
#endif
#endif // ASMJIT_CORE_VIRTMEM_H_INCLUDED
libraries/asmjit/asmjit/core/zone.cpp 0 → 100644
View file @ cc7018ea
// This file is part of AsmJit project <https://asmjit.com>
//
// See asmjit.h or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#include "../core/api-build_p.h"
#include "../core/support.h"
#include "../core/zone.h"
ASMJIT_BEGIN_NAMESPACE
// Zone - Globals
// ==============
// Zero size block used by `Zone` that doesn't have any memory allocated. Should be allocated in read-only memory
// and should never be modified.
const Zone::Block Zone::_zeroBlock = { nullptr, nullptr, 0 };
// Zone - Init & Reset
// ===================
void Zone::_init(size_t blockSize, size_t blockAlignment, const Support::Temporary* temporary) noexcept {
ASMJIT_ASSERT(blockSize >= kMinBlockSize);
ASMJIT_ASSERT(blockSize <= kMaxBlockSize);
ASMJIT_ASSERT(blockAlignment <= 64);
// Just to make the compiler happy...
constexpr size_t kBlockSizeMask = (Support::allOnes<size_t>() >> 4);
constexpr size_t kBlockAlignmentShiftMask = 0x7u;
_assignZeroBlock();
_blockSize = blockSize & kBlockSizeMask;
_isTemporary = temporary != nullptr;
_blockAlignmentShift = Support::ctz(blockAlignment) & kBlockAlignmentShiftMask;
// Setup the first [temporary] block, if necessary.
if (temporary) {
Block* block = temporary->data<Block>();
block->prev = nullptr;
block->next = nullptr;
ASMJIT_ASSERT(temporary->size() >= kBlockSize);
block->size = temporary->size() - kBlockSize;
_assignBlock(block);
}
}
void Zone::reset(ResetPolicy resetPolicy) noexcept {
Block* cur = _block;
// Can't be altered.
if (cur == &_zeroBlock)
return;
if (resetPolicy == ResetPolicy::kHard) {
Block* initial = const_cast<Zone::Block*>(&_zeroBlock);
_ptr = initial->data();
_end = initial->data();
_block = initial;
// Since cur can be in the middle of the double-linked list, we have to traverse both directions (`prev` and
// `next`) separately to visit all.
Block* next = cur->next;
do {
Block* prev = cur->prev;
// If this is the first block and this ZoneTmp is temporary then the first block is statically allocated.
// We cannot free it and it makes sense to keep it even when this is hard reset.
if (prev == nullptr && _isTemporary) {
cur->prev = nullptr;
cur->next = nullptr;
_assignBlock(cur);
break;
}
::free(cur);
cur = prev;
} while (cur);
cur = next;
while (cur) {
next = cur->next;
::free(cur);
cur = next;
}
}
else {
while (cur->prev)
cur = cur->prev;
_assignBlock(cur);
}
}
// Zone - Alloc
// ============
void* Zone::_alloc(size_t size, size_t alignment) noexcept {
Block* curBlock = _block;
Block* next = curBlock->next;
size_t rawBlockAlignment = blockAlignment();
size_t minimumAlignment = Support::max<size_t>(alignment, rawBlockAlignment);
// If the `Zone` has been cleared the current block doesn't have to be the last one. Check if there is a block
// that can be used instead of allocating a new one. If there is a `next` block it's completely unused, we don't
// have to check for remaining bytes in that case.
if (next) {
uint8_t* ptr = Support::alignUp(next->data(), minimumAlignment);
uint8_t* end = Support::alignDown(next->data() + next->size, rawBlockAlignment);
if (size <= (size_t)(end - ptr)) {
_block = next;
_ptr = ptr + size;
_end = Support::alignDown(next->data() + next->size, rawBlockAlignment);
return static_cast<void*>(ptr);
}
}
size_t blockAlignmentOverhead = alignment - Support::min<size_t>(alignment, Globals::kAllocAlignment);
size_t newSize = Support::max(blockSize(), size);
// Prevent arithmetic overflow.
if (ASMJIT_UNLIKELY(newSize > SIZE_MAX - kBlockSize - blockAlignmentOverhead))
return nullptr;
// Allocate new block - we add alignment overhead to `newSize`, which becomes the new block size, and we also add
// `kBlockOverhead` to the allocator as it includes members of `Zone::Block` structure.
newSize += blockAlignmentOverhead;
Block* newBlock = static_cast<Block*>(::malloc(newSize + kBlockSize));
if (ASMJIT_UNLIKELY(!newBlock))
return nullptr;
// Align the pointer to `minimumAlignment` and adjust the size of this block accordingly. It's the same as using
// `minimumAlignment - Support::alignUpDiff()`, just written differently.
{
newBlock->prev = nullptr;
newBlock->next = nullptr;
newBlock->size = newSize;
if (curBlock != &_zeroBlock) {
newBlock->prev = curBlock;
curBlock->next = newBlock;
// Does only happen if there is a next block, but the requested memory can't fit into it. In this case a new
// buffer is allocated and inserted between the current block and the next one.
if (next) {
newBlock->next = next;
next->prev = newBlock;
}
}
uint8_t* ptr = Support::alignUp(newBlock->data(), minimumAlignment);
uint8_t* end = Support::alignDown(newBlock->data() + newSize, rawBlockAlignment);
_ptr = ptr + size;
_end = end;
_block = newBlock;
ASMJIT_ASSERT(_ptr <= _end);
return static_cast<void*>(ptr);
}
}
void* Zone::allocZeroed(size_t size, size_t alignment) noexcept {
void* p = alloc(size, alignment);
if (ASMJIT_UNLIKELY(!p))
return p;
return memset(p, 0, size);
}
void* Zone::dup(const void* data, size_t size, bool nullTerminate) noexcept {
if (ASMJIT_UNLIKELY(!data || !size))
return nullptr;
ASMJIT_ASSERT(size != SIZE_MAX);
uint8_t* m = allocT<uint8_t>(size + nullTerminate);
if (ASMJIT_UNLIKELY(!m)) return nullptr;
memcpy(m, data, size);
if (nullTerminate) m[size] = '\0';
return static_cast<void*>(m);
}
char* Zone::sformat(const char* fmt, ...) noexcept {
if (ASMJIT_UNLIKELY(!fmt))
return nullptr;
char buf[512];
size_t size;
va_list ap;
va_start(ap, fmt);
size = unsigned(vsnprintf(buf, ASMJIT_ARRAY_SIZE(buf) - 1, fmt, ap));
va_end(ap);
buf[size++] = 0;
return static_cast<char*>(dup(buf, size));
}
// ZoneAllocator - Utilities
// =========================
#if defined(ASMJIT_BUILD_DEBUG)
static bool ZoneAllocator_hasDynamicBlock(ZoneAllocator* self, ZoneAllocator::DynamicBlock* block) noexcept {
ZoneAllocator::DynamicBlock* cur = self->_dynamicBlocks;
while (cur) {
if (cur == block)
return true;
cur = cur->next;
}
return false;
}
#endif
// ZoneAllocator - Init & Reset
// ============================
void ZoneAllocator::reset(Zone* zone) noexcept {
// Free dynamic blocks.
DynamicBlock* block = _dynamicBlocks;
while (block) {
DynamicBlock* next = block->next;
::free(block);
block = next;
}
// Zero the entire class and initialize to the given `zone`.
memset(this, 0, sizeof(*this));
_zone = zone;
}
// asmjit::ZoneAllocator - Alloc & Release
// =======================================
void* ZoneAllocator::_alloc(size_t size, size_t& allocatedSize) noexcept {
ASMJIT_ASSERT(isInitialized());
// Use the memory pool only if the requested block has a reasonable size.
uint32_t slot;
if (_getSlotIndex(size, slot, allocatedSize)) {
// Slot reuse.
uint8_t* p = reinterpret_cast<uint8_t*>(_slots[slot]);
size = allocatedSize;
if (p) {
_slots[slot] = reinterpret_cast<Slot*>(p)->next;
return p;
}
_zone->align(kBlockAlignment);
p = _zone->ptr();
size_t remain = (size_t)(_zone->end() - p);
if (ASMJIT_LIKELY(remain >= size)) {
_zone->setPtr(p + size);
return p;
}
else {
// Distribute the remaining memory to suitable slots, if possible.
if (remain >= kLoGranularity) {
do {
size_t distSize = Support::min<size_t>(remain, kLoMaxSize);
uint32_t distSlot = uint32_t((distSize - kLoGranularity) / kLoGranularity);
ASMJIT_ASSERT(distSlot < kLoCount);
reinterpret_cast<Slot*>(p)->next = _slots[distSlot];
_slots[distSlot] = reinterpret_cast<Slot*>(p);
p += distSize;
remain -= distSize;
} while (remain >= kLoGranularity);
_zone->setPtr(p);
}
p = static_cast<uint8_t*>(_zone->_alloc(size, kBlockAlignment));
if (ASMJIT_UNLIKELY(!p)) {
allocatedSize = 0;
return nullptr;
}
return p;
}
}
else {
// Allocate a dynamic block.
size_t kBlockOverhead = sizeof(DynamicBlock) + sizeof(DynamicBlock*) + kBlockAlignment;
// Handle a possible overflow.
if (ASMJIT_UNLIKELY(kBlockOverhead >= SIZE_MAX - size))
return nullptr;
void* p = ::malloc(size + kBlockOverhead);
if (ASMJIT_UNLIKELY(!p)) {
allocatedSize = 0;
return nullptr;
}
// Link as first in `_dynamicBlocks` double-linked list.
DynamicBlock* block = static_cast<DynamicBlock*>(p);
DynamicBlock* next = _dynamicBlocks;
if (next)
next->prev = block;
block->prev = nullptr;
block->next = next;
_dynamicBlocks = block;
// Align the pointer to the guaranteed alignment and store `DynamicBlock`
// at the beginning of the memory block, so `_releaseDynamic()` can find it.
p = Support::alignUp(static_cast<uint8_t*>(p) + sizeof(DynamicBlock) + sizeof(DynamicBlock*), kBlockAlignment);
reinterpret_cast<DynamicBlock**>(p)[-1] = block;
allocatedSize = size;
return p;
}
}
void* ZoneAllocator::_allocZeroed(size_t size, size_t& allocatedSize) noexcept {
ASMJIT_ASSERT(isInitialized());
void* p = _alloc(size, allocatedSize);
if (ASMJIT_UNLIKELY(!p)) return p;
return memset(p, 0, allocatedSize);
}
void ZoneAllocator::_releaseDynamic(void* p, size_t size) noexcept {
DebugUtils::unused(size);
ASMJIT_ASSERT(isInitialized());
// Pointer to `DynamicBlock` is stored at [-1].
DynamicBlock* block = reinterpret_cast<DynamicBlock**>(p)[-1];
ASMJIT_ASSERT(ZoneAllocator_hasDynamicBlock(this, block));
// Unlink and free.
DynamicBlock* prev = block->prev;
DynamicBlock* next = block->next;
if (prev)
prev->next = next;
else
_dynamicBlocks = next;
if (next)
next->prev = prev;
::free(block);
}
ASMJIT_END_NAMESPACE
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment