void AudioEngine::SetDefaultSampleRate(int sampleRate)
{
    if ((sampleRate < XAUDIO2_MIN_SAMPLE_RATE) || (sampleRate > XAUDIO2_MAX_SAMPLE_RATE))
        throw std::exception("Default sample rate is out of range");

    pImpl->defaultRate = sampleRate;
}

// File: AudioEngine.cpp

#include "pch.h"
#include "Audio.h"
#include "SoundCommon.h"

#include <unordered_map>

using namespace DirectX;
using Microsoft::WRL::ComPtr;

//#define VERBOSE_TRACE

#ifdef VERBOSE_TRACE
#pragma message("NOTE: Verbose tracing enabled")
#endif

namespace
{
    struct EngineCallback : public IXAudio2EngineCallback
    {
        EngineCallback() noexcept(false)
        {
            mCriticalError.reset(CreateEventEx(nullptr, nullptr, 0, EVENT_MODIFY_STATE | SYNCHRONIZE));
            if (!mCriticalError)
            {
                throw std::exception("CreateEvent");
            }
        }

        EngineCallback(EngineCallback&&) = default;
        EngineCallback& operator= (EngineCallback&&) = default;

        EngineCallback(EngineCallback const&) = delete;
        EngineCallback& operator= (EngineCallback const&) = delete;

        virtual ~EngineCallback() = default;

        STDMETHOD_(void, OnProcessingPassStart) () override {}
        STDMETHOD_(void, OnProcessingPassEnd)() override {}

        STDMETHOD_(void, OnCriticalError) (THIS_ HRESULT error)
        {
        #ifndef _DEBUG
            UNREFERENCED_PARAMETER(error);
        #endif
            DebugTrace("ERROR: AudioEngine encountered critical error (%08X)\n", static_cast<unsigned int>(error));
            SetEvent(mCriticalError.get());
        }

        ScopedHandle mCriticalError;
    };

    struct VoiceCallback : public IXAudio2VoiceCallback
    {
        VoiceCallback() noexcept(false)
        {
            mBufferEnd.reset(CreateEventEx(nullptr, nullptr, 0, EVENT_MODIFY_STATE | SYNCHRONIZE));
            if (!mBufferEnd)
            {
                throw std::exception("CreateEvent");
            }
        }

        VoiceCallback(VoiceCallback&&) = default;
        VoiceCallback& operator=(VoiceCallback&&) = default;

        VoiceCallback(const VoiceCallback&) = delete;
        VoiceCallback& operator=(const VoiceCallback&) = delete;

        virtual ~VoiceCallback()
        {
        }

        STDMETHOD_(void, OnVoiceProcessingPassStart) (UINT32) override {}
        STDMETHOD_(void, OnVoiceProcessingPassEnd)() override {}
        STDMETHOD_(void, OnStreamEnd)() override {}
        STDMETHOD_(void, OnBufferStart)(void*) override {}

        STDMETHOD_(void, OnBufferEnd)(void* context) override
        {
            if (context)
            {
                auto inotify = static_cast<IVoiceNotify*>(context);
                inotify->OnBufferEnd();
                SetEvent(mBufferEnd.get());
            }
        }

        STDMETHOD_(void, OnLoopEnd)(void*) override {}
        STDMETHOD_(void, OnVoiceError)(void*, HRESULT) override {}

        ScopedHandle mBufferEnd;
    };

    static const XAUDIO2FX_REVERB_I3DL2_PARAMETERS gReverbPresets[] =
    {
        XAUDIO2FX_I3DL2_PRESET_DEFAULT,             // Reverb_Off
        XAUDIO2FX_I3DL2_PRESET_DEFAULT,             // Reverb_Default
        XAUDIO2FX_I3DL2_PRESET_GENERIC,             // Reverb_Generic
        XAUDIO2FX_I3DL2_PRESET_FOREST,              // Reverb_Forest
        XAUDIO2FX_I3DL2_PRESET_PADDEDCELL,          // Reverb_PaddedCell
        XAUDIO2FX_I3DL2_PRESET_ROOM,                // Reverb_Room
        XAUDIO2FX_I3DL2_PRESET_BATHROOM,            // Reverb_Bathroom
        XAUDIO2FX_I3DL2_PRESET_LIVINGROOM,          // Reverb_LivingRoom
        XAUDIO2FX_I3DL2_PRESET_STONEROOM,           // Reverb_StoneRoom
        XAUDIO2FX_I3DL2_PRESET_AUDITORIUM,          // Reverb_Auditorium
        XAUDIO2FX_I3DL2_PRESET_CONCERTHALL,         // Reverb_ConcertHall
        XAUDIO2FX_I3DL2_PRESET_CAVE,                // Reverb_Cave
        XAUDIO2FX_I3DL2_PRESET_ARENA,               // Reverb_Arena
        XAUDIO2FX_I3DL2_PRESET_HANGAR,              // Reverb_Hangar
        XAUDIO2FX_I3DL2_PRESET_CARPETEDHALLWAY,     // Reverb_CarpetedHallway
        XAUDIO2FX_I3DL2_PRESET_HALLWAY,             // Reverb_Hallway
        XAUDIO2FX_I3DL2_PRESET_STONECORRIDOR,       // Reverb_StoneCorridor
        XAUDIO2FX_I3DL2_PRESET_ALLEY,               // Reverb_Alley
        XAUDIO2FX_I3DL2_PRESET_CITY,                // Reverb_City
        XAUDIO2FX_I3DL2_PRESET_MOUNTAINS,           // Reverb_Mountains
        XAUDIO2FX_I3DL2_PRESET_QUARRY,              // Reverb_Quarry
        XAUDIO2FX_I3DL2_PRESET_PLAIN,               // Reverb_Plain
        XAUDIO2FX_I3DL2_PRESET_PARKINGLOT,          // Reverb_ParkingLot
        XAUDIO2FX_I3DL2_PRESET_SEWERPIPE,           // Reverb_SewerPipe
        XAUDIO2FX_I3DL2_PRESET_UNDERWATER,          // Reverb_Underwater
        XAUDIO2FX_I3DL2_PRESET_SMALLROOM,           // Reverb_SmallRoom
        XAUDIO2FX_I3DL2_PRESET_MEDIUMROOM,          // Reverb_MediumRoom
        XAUDIO2FX_I3DL2_PRESET_LARGEROOM,           // Reverb_LargeRoom
        XAUDIO2FX_I3DL2_PRESET_MEDIUMHALL,          // Reverb_MediumHall
        XAUDIO2FX_I3DL2_PRESET_LARGEHALL,           // Reverb_LargeHall
        XAUDIO2FX_I3DL2_PRESET_PLATE,               // Reverb_Plate
    };

    inline unsigned int makeVoiceKey(_In_ const WAVEFORMATEX* wfx) noexcept
    {
        assert(IsValid(wfx));

        if (wfx->nChannels > 0x7F)
            return 0;

        // This hash does not use nSamplesPerSec because voice reuse can change the source sample rate.

        // nAvgBytesPerSec and nBlockAlign are derived from other values in XAudio2 supported formats.

        union KeyGen
        {
            struct
            {
                unsigned int tag : 9;
                unsigned int channels : 7;
                unsigned int bitsPerSample : 8;
            } pcm;

            struct
            {
                unsigned int tag : 9;
                unsigned int channels : 7;
                unsigned int samplesPerBlock : 16;
            } adpcm;

        #ifdef DIRECTX_ENABLE_XMA2
            struct
            {
                unsigned int tag : 9;
                unsigned int channels : 7;
                unsigned int encoderVersion : 8;
            } xma;
        #endif

            unsigned int key;
        } result;

        static_assert(sizeof(KeyGen) == sizeof(unsigned int), "KeyGen is invalid");

        result.key = 0;

        if (wfx->wFormatTag == WAVE_FORMAT_EXTENSIBLE)
        {
            // We reuse EXTENSIBLE only if it is equivalent to the standard form
            auto wfex = reinterpret_cast<const WAVEFORMATEXTENSIBLE*>(wfx);
            if (wfex->Samples.wValidBitsPerSample != 0 && wfex->Samples.wValidBitsPerSample != wfx->wBitsPerSample)
                return 0;

            if (wfex->dwChannelMask != 0 && wfex->dwChannelMask != GetDefaultChannelMask(wfx->nChannels))
                return 0;
        }

        uint32_t tag = GetFormatTag(wfx);
        switch (tag)
        {
            case WAVE_FORMAT_PCM:
                static_assert(WAVE_FORMAT_PCM < 0x1ff, "KeyGen tag is too small");
                result.pcm.tag = WAVE_FORMAT_PCM;
                result.pcm.channels = wfx->nChannels;
                result.pcm.bitsPerSample = wfx->wBitsPerSample;
                break;

            case WAVE_FORMAT_IEEE_FLOAT:
                static_assert(WAVE_FORMAT_IEEE_FLOAT < 0x1ff, "KeyGen tag is too small");

                if (wfx->wBitsPerSample != 32)
                    return 0;

                result.pcm.tag = WAVE_FORMAT_IEEE_FLOAT;
                result.pcm.channels = wfx->nChannels;
                result.pcm.bitsPerSample = 32;
                break;

            case WAVE_FORMAT_ADPCM:
                static_assert(WAVE_FORMAT_ADPCM < 0x1ff, "KeyGen tag is too small");
                result.adpcm.tag = WAVE_FORMAT_ADPCM;
                result.adpcm.channels = wfx->nChannels;

                {
                    auto wfadpcm = reinterpret_cast<const ADPCMWAVEFORMAT*>(wfx);
                    result.adpcm.samplesPerBlock = wfadpcm->wSamplesPerBlock;
                }
                break;

            #ifdef DIRECTX_ENABLE_XMA2
            case WAVE_FORMAT_XMA2:
                static_assert(WAVE_FORMAT_XMA2 < 0x1ff, "KeyGen tag is too small");
                result.xma.tag = WAVE_FORMAT_XMA2;
                result.xma.channels = wfx->nChannels;

                {
                    auto xmaFmt = reinterpret_cast<const XMA2WAVEFORMATEX*>(wfx);

                    if ((xmaFmt->LoopBegin > 0)
                        || (xmaFmt->PlayBegin > 0))
                        return 0;

                    result.xma.encoderVersion = xmaFmt->EncoderVersion;
                }
                break;
            #endif

            default:
                return 0;
        }

        return result.key;
    }
}

static_assert(_countof(gReverbPresets) == Reverb_MAX, "AUDIO_ENGINE_REVERB enum mismatch");

// AudioEngine

#define SAFE_DESTROY_VOICE(voice) if ( voice ) { voice->DestroyVoice(); voice = nullptr; }

// Internal object implementation class.
class AudioEngine::Impl
{
public:
    Impl() noexcept :
        mMasterVoice(nullptr),
        mReverbVoice(nullptr),
        masterChannelMask(0),
        masterChannels(0),
        masterRate(0),
        defaultRate(44100),
        maxVoiceOneshots(SIZE_MAX),
        maxVoiceInstances(SIZE_MAX),
        mMasterVolume(1.f),
        mX3DAudio{},
        mCriticalError(false),
        mReverbEnabled(false),
        mEngineFlags(AudioEngine_Default),
        mCategory(AudioCategory_GameEffects),
        mVoiceInstances(0)
    {
    }

    ~Impl() = default;

    Impl(Impl&&) = default;
    Impl& operator= (Impl&&) = default;

    Impl(Impl const&) = delete;
    Impl& operator= (Impl const&) = delete;

    HRESULT Initialize(AUDIO_ENGINE_FLAGS flags,
        _In_opt_ const WAVEFORMATEX* wfx,
        _In_opt_z_ const wchar_t* deviceId,
        AUDIO_STREAM_CATEGORY category);

    HRESULT Reset(_In_opt_ const WAVEFORMATEX* wfx, _In_opt_z_ const wchar_t* deviceId);

    void SetSilentMode();

    void Shutdown() noexcept;

    bool Update();

    void SetReverb(_In_opt_ const XAUDIO2FX_REVERB_PARAMETERS* native) noexcept;

    void SetMasteringLimit(int release, int loudness);

    AudioStatistics GetStatistics() const;

    void TrimVoicePool();

    void AllocateVoice(_In_ const WAVEFORMATEX* wfx,
        SOUND_EFFECT_INSTANCE_FLAGS flags, bool oneshot,
        _Outptr_result_maybenull_ IXAudio2SourceVoice** voice);
    void DestroyVoice(_In_ IXAudio2SourceVoice* voice) noexcept;

    void RegisterNotify(_In_ IVoiceNotify* notify, bool usesUpdate);
    void UnregisterNotify(_In_ IVoiceNotify* notify, bool oneshots, bool usesUpdate);

    ComPtr<IXAudio2>                    xaudio2;
    IXAudio2MasteringVoice*             mMasterVoice;
    IXAudio2SubmixVoice*                mReverbVoice;

    uint32_t                            masterChannelMask;
    uint32_t                            masterChannels;
    uint32_t                            masterRate;

    int                                 defaultRate;
    size_t                              maxVoiceOneshots;
    size_t                              maxVoiceInstances;
    float                               mMasterVolume;

    X3DAUDIO_HANDLE                     mX3DAudio;

    bool                                mCriticalError;
    bool                                mReverbEnabled;

    AUDIO_ENGINE_FLAGS                  mEngineFlags;

private:
    using notifylist_t = std::set<IVoiceNotify*>;
    using oneshotlist_t = std::list<std::pair<unsigned int, IXAudio2SourceVoice*>>;
    using voicepool_t = std::unordered_multimap<unsigned int, IXAudio2SourceVoice*>;

    AUDIO_STREAM_CATEGORY               mCategory;
    ComPtr<IUnknown>                    mReverbEffect;
    ComPtr<IUnknown>                    mVolumeLimiter;
    oneshotlist_t                       mOneShots;
    voicepool_t                         mVoicePool;
    notifylist_t                        mNotifyObjects;
    notifylist_t                        mNotifyUpdates;
    size_t                              mVoiceInstances;
    VoiceCallback                       mVoiceCallback;
    EngineCallback                      mEngineCallback;
};

_Use_decl_annotations_
HRESULT AudioEngine::Impl::Initialize(
    AUDIO_ENGINE_FLAGS flags,
    const WAVEFORMATEX* wfx,
    const wchar_t* deviceId,
    AUDIO_STREAM_CATEGORY category)
{
    mEngineFlags = flags;
    mCategory = category;

    return Reset(wfx, deviceId);
}

_Use_decl_annotations_
HRESULT AudioEngine::Impl::Reset(const WAVEFORMATEX* wfx, const wchar_t* deviceId)
{
    if (wfx)
    {
        if (wfx->wFormatTag != WAVE_FORMAT_PCM)
            return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED);

        if (!wfx->nChannels || wfx->nChannels > XAUDIO2_MAX_AUDIO_CHANNELS)
            return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED);

        if (wfx->nSamplesPerSec < XAUDIO2_MIN_SAMPLE_RATE || wfx->nSamplesPerSec > XAUDIO2_MAX_SAMPLE_RATE)
            return HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED);

        // We don't use other data members of WAVEFORMATEX here to describe the device format, so no need to fully validate
    }

    assert(!xaudio2);
    assert(mMasterVoice == nullptr);
    assert(mReverbVoice == nullptr);

    masterChannelMask = masterChannels = masterRate = 0;

    memset(&mX3DAudio, 0, X3DAUDIO_HANDLE_BYTESIZE);

    mCriticalError = false;
    mReverbEnabled = false;

    // Create XAudio2 engine
    HRESULT hr = XAudio2Create(xaudio2.ReleaseAndGetAddressOf(), 0u);
    if (FAILED(hr))
        return hr;

    if (mEngineFlags & AudioEngine_Debug)
    {
        XAUDIO2_DEBUG_CONFIGURATION debug = {};
        debug.TraceMask = XAUDIO2_LOG_ERRORS | XAUDIO2_LOG_WARNINGS;
        debug.BreakMask = XAUDIO2_LOG_ERRORS;
        xaudio2->SetDebugConfiguration(&debug, nullptr);
    #ifdef USING_XAUDIO2_9
        DebugTrace("INFO: XAudio 2.9 debugging enabled\n");
    #else // USING_XAUDIO2_8
            // To see the trace output, you need to view ETW logs for this application:
            //    Go to Control Panel, Administrative Tools, Event Viewer.
            //    View->Show Analytic and Debug Logs.
            //    Applications and Services Logs / Microsoft / Windows / XAudio2.
            //    Right click on Microsoft Windows XAudio2 debug logging, Properties, then Enable Logging, and hit OK
        DebugTrace("INFO: XAudio 2.8 debugging enabled\n");
    #endif
    }

    if (mEngineFlags & AudioEngine_DisableVoiceReuse)
    {
        DebugTrace("INFO: Voice reuse is disabled\n");
    }

    hr = xaudio2->RegisterForCallbacks(&mEngineCallback);
    if (FAILED(hr))
    {
        xaudio2.Reset();
        return hr;
    }

    // Create mastering voice for device
    hr = xaudio2->CreateMasteringVoice(&mMasterVoice,
        (wfx) ? wfx->nChannels : 0u /*XAUDIO2_DEFAULT_CHANNELS */,
        (wfx) ? wfx->nSamplesPerSec : 0u /* XAUDIO2_DEFAULT_SAMPLERATE */,
        0u, deviceId, nullptr, mCategory);
    if (FAILED(hr))
    {
        xaudio2.Reset();
        return hr;
    }

    DWORD dwChannelMask;
    hr = mMasterVoice->GetChannelMask(&dwChannelMask);
    if (FAILED(hr))
    {
        SAFE_DESTROY_VOICE(mMasterVoice)
        xaudio2.Reset();
        return hr;
    }

    XAUDIO2_VOICE_DETAILS details;
    mMasterVoice->GetVoiceDetails(&details);

    masterChannelMask = dwChannelMask;
    masterChannels = details.InputChannels;
    masterRate = details.InputSampleRate;

    DebugTrace("INFO: mastering voice has %u channels, %u sample rate, %08X channel mask\n",
        masterChannels, masterRate, masterChannelMask);

    if (mMasterVolume != 1.f)
    {
        hr = mMasterVoice->SetVolume(mMasterVolume);
        if (FAILED(hr))
        {
            SAFE_DESTROY_VOICE(mMasterVoice)
            xaudio2.Reset();
            return hr;
        }
    }

    // Setup mastering volume limiter (optional)
    if (mEngineFlags & AudioEngine_UseMasteringLimiter)
    {
        FXMASTERINGLIMITER_PARAMETERS params = {};
        params.Release = FXMASTERINGLIMITER_DEFAULT_RELEASE;
        params.Loudness = FXMASTERINGLIMITER_DEFAULT_LOUDNESS;

        hr = CreateFX(__uuidof(FXMasteringLimiter), mVolumeLimiter.ReleaseAndGetAddressOf(), &params, sizeof(params));
        if (FAILED(hr))
        {
            SAFE_DESTROY_VOICE(mMasterVoice)
            xaudio2.Reset();
            return hr;
        }

        XAUDIO2_EFFECT_DESCRIPTOR desc = {};
        desc.InitialState = TRUE;
        desc.OutputChannels = masterChannels;
        desc.pEffect = mVolumeLimiter.Get();

        XAUDIO2_EFFECT_CHAIN chain = { 1, &desc };
        hr = mMasterVoice->SetEffectChain(&chain);
        if (FAILED(hr))
        {
            SAFE_DESTROY_VOICE(mMasterVoice)
            mVolumeLimiter.Reset();
            xaudio2.Reset();
            return hr;
        }

        DebugTrace("INFO: Mastering volume limiter enabled\n");
    }

    // Setup environmental reverb for 3D audio (optional)
    if (mEngineFlags & AudioEngine_EnvironmentalReverb)
    {
        hr = XAudio2CreateReverb(mReverbEffect.ReleaseAndGetAddressOf(), 0u);
        if (FAILED(hr))
        {
            SAFE_DESTROY_VOICE(mMasterVoice)
            mVolumeLimiter.Reset();
            xaudio2.Reset();
            return hr;
        }

        XAUDIO2_EFFECT_DESCRIPTOR effects[] = { { mReverbEffect.Get(), TRUE, 1 } };
        XAUDIO2_EFFECT_CHAIN effectChain = { 1, effects };

        mReverbEnabled = true;

        hr = xaudio2->CreateSubmixVoice(&mReverbVoice, 1, masterRate,
            (mEngineFlags & AudioEngine_ReverbUseFilters) ? XAUDIO2_VOICE_USEFILTER : 0u, 0u,
            nullptr, &effectChain);
        if (FAILED(hr))
        {
            SAFE_DESTROY_VOICE(mMasterVoice)
            mReverbEffect.Reset();
            mVolumeLimiter.Reset();
            xaudio2.Reset();
            return hr;
        }

        XAUDIO2FX_REVERB_PARAMETERS native;
        ReverbConvertI3DL2ToNative(&gReverbPresets[Reverb_Default], &native);
        hr = mReverbVoice->SetEffectParameters(0, &native, sizeof(XAUDIO2FX_REVERB_PARAMETERS));
        if (FAILED(hr))
        {
            SAFE_DESTROY_VOICE(mReverbVoice)
            SAFE_DESTROY_VOICE(mMasterVoice)
            mReverbEffect.Reset();
            mVolumeLimiter.Reset();
            xaudio2.Reset();
            return hr;
        }

        DebugTrace("INFO: I3DL2 reverb effect enabled for 3D positional audio\n");
    }

    // Setup 3D audio
    constexpr float SPEEDOFSOUND = X3DAUDIO_SPEED_OF_SOUND;

    hr = X3DAudioInitialize(masterChannelMask, SPEEDOFSOUND, mX3DAudio);
    if (FAILED(hr))
    {
        SAFE_DESTROY_VOICE(mReverbVoice)
        SAFE_DESTROY_VOICE(mMasterVoice)
        mReverbEffect.Reset();
        mVolumeLimiter.Reset();
        xaudio2.Reset();
        return hr;
    }

    // Inform any notify objects we are ready to go again
    for (auto it = mNotifyObjects.begin(); it != mNotifyObjects.end(); ++it)
    {
        assert(*it != nullptr);
        (*it)->OnReset();
    }

    return S_OK;
}

void AudioEngine::Impl::SetSilentMode()
{
    for (auto it = mNotifyObjects.begin(); it != mNotifyObjects.end(); ++it)
    {
        assert(*it != nullptr);
        (*it)->OnCriticalError();
    }

    for (auto it = mOneShots.begin(); it != mOneShots.end(); ++it)
    {
        assert(it->second != nullptr);
        it->second->DestroyVoice();
    }
    mOneShots.clear();

    for (auto it = mVoicePool.begin(); it != mVoicePool.end(); ++it)
    {
        assert(it->second != nullptr);
        it->second->DestroyVoice();
    }
    mVoicePool.clear();

    mVoiceInstances = 0;

    SAFE_DESTROY_VOICE(mReverbVoice)
    SAFE_DESTROY_VOICE(mMasterVoice)

    mReverbEffect.Reset();
    mVolumeLimiter.Reset();
    xaudio2.Reset();
}

void AudioEngine::Impl::Shutdown() noexcept
{
    for (auto it = mNotifyObjects.begin(); it != mNotifyObjects.end(); ++it)
    {
        assert(*it != nullptr);
        (*it)->OnDestroyEngine();
    }

    if (xaudio2)
    {
        xaudio2->UnregisterForCallbacks(&mEngineCallback);

        xaudio2->StopEngine();

        for (auto it = mOneShots.begin(); it != mOneShots.end(); ++it)
        {
            assert(it->second != nullptr);
            it->second->DestroyVoice();
        }
        mOneShots.clear();

        for (auto it = mVoicePool.begin(); it != mVoicePool.end(); ++it)
        {
            assert(it->second != nullptr);
            it->second->DestroyVoice();
        }
        mVoicePool.clear();

        mVoiceInstances = 0;

        SAFE_DESTROY_VOICE(mReverbVoice)
        SAFE_DESTROY_VOICE(mMasterVoice)

        mReverbEffect.Reset();
        mVolumeLimiter.Reset();
        xaudio2.Reset();

        masterChannelMask = masterChannels = masterRate = 0;

        mCriticalError = false;
        mReverbEnabled = false;

        memset(&mX3DAudio, 0, X3DAUDIO_HANDLE_BYTESIZE);
    }
}

bool AudioEngine::Impl::Update()
{
    if (!xaudio2)
        return false;

    HANDLE events[2] = { mEngineCallback.mCriticalError.get(), mVoiceCallback.mBufferEnd.get() };
    switch (WaitForMultipleObjectsEx(_countof(events), events, FALSE, 0, FALSE))
    {
        default:
        case WAIT_TIMEOUT:
            break;

        case WAIT_OBJECT_0:     // OnCritialError
            mCriticalError = true;

            SetSilentMode();
            return false;

        case WAIT_OBJECT_0 + 1: // OnBufferEnd
            // Scan for completed one-shot voices
            for (auto it = mOneShots.begin(); it != mOneShots.end(); )
            {
                assert(it->second != nullptr);

                XAUDIO2_VOICE_STATE xstate;
                it->second->GetState(&xstate, XAUDIO2_VOICE_NOSAMPLESPLAYED);

                if (!xstate.BuffersQueued)
                {
                    (void)it->second->Stop(0);
                    if (it->first)
                    {
                        // Put voice back into voice pool for reuse since it has a non-zero voiceKey
                    #ifdef VERBOSE_TRACE
                        DebugTrace("INFO: One-shot voice being saved for reuse (%08X)\n", it->first);
                    #endif
                        voicepool_t::value_type v(it->first, it->second);
                        mVoicePool.emplace(v);
                    }
                    else
                    {
                        // Voice is to be destroyed rather than reused
                    #ifdef VERBOSE_TRACE
                        DebugTrace("INFO: Destroying one-shot voice\n");
                    #endif
                        it->second->DestroyVoice();
                    }
                    it = mOneShots.erase(it);
                }
                else
                    ++it;
            }
            break;

        case WAIT_FAILED:
            throw std::exception("WaitForMultipleObjects");
    }

    // Inform any notify objects of updates
    for (auto it = mNotifyUpdates.begin(); it != mNotifyUpdates.end(); ++it)
    {
        assert(*it != nullptr);
        (*it)->OnUpdate();
    }

    return true;
}

_Use_decl_annotations_
void AudioEngine::Impl::SetReverb(const XAUDIO2FX_REVERB_PARAMETERS* native) noexcept
{
    if (!mReverbVoice)
        return;

    if (native)
    {
        if (!mReverbEnabled)
        {
            mReverbEnabled = true;
            (void)mReverbVoice->EnableEffect(0);
        }

        (void)mReverbVoice->SetEffectParameters(0, native, sizeof(XAUDIO2FX_REVERB_PARAMETERS));
    }
    else if (mReverbEnabled)
    {
        mReverbEnabled = false;
        (void)mReverbVoice->DisableEffect(0);
    }
}

void AudioEngine::Impl::SetMasteringLimit(int release, int loudness)
{
    if (!mVolumeLimiter || !mMasterVoice)
        return;

    if ((release < FXMASTERINGLIMITER_MIN_RELEASE) || (release > FXMASTERINGLIMITER_MAX_RELEASE))
        throw std::out_of_range("AudioEngine::SetMasteringLimit");

    if ((loudness < FXMASTERINGLIMITER_MIN_LOUDNESS) || (loudness > FXMASTERINGLIMITER_MAX_LOUDNESS))
        throw std::out_of_range("AudioEngine::SetMasteringLimit");

    FXMASTERINGLIMITER_PARAMETERS params = {};
    params.Release = static_cast<UINT32>(release);
    params.Loudness = static_cast<UINT32>(loudness);

    HRESULT hr = mMasterVoice->SetEffectParameters(0, &params, sizeof(params));
    ThrowIfFailed(hr);
}

AudioStatistics AudioEngine::Impl::GetStatistics() const
{
    AudioStatistics stats = {};

    stats.allocatedVoices = stats.allocatedVoicesOneShot = mOneShots.size() + mVoicePool.size();
    stats.allocatedVoicesIdle = mVoicePool.size();

    for (auto it = mNotifyObjects.begin(); it != mNotifyObjects.end(); ++it)
    {
        assert(*it != nullptr);
        (*it)->GatherStatistics(stats);
    }

    assert(stats.allocatedVoices == (mOneShots.size() + mVoicePool.size() + mVoiceInstances));

    return stats;
}

void AudioEngine::Impl::TrimVoicePool()
{
    for (auto it = mNotifyObjects.begin(); it != mNotifyObjects.end(); ++it)
    {
        assert(*it != nullptr);
        (*it)->OnTrim();
    }

    for (auto it = mVoicePool.begin(); it != mVoicePool.end(); ++it)
    {
        assert(it->second != nullptr);
        it->second->DestroyVoice();
    }
    mVoicePool.clear();
}

_Use_decl_annotations_
void AudioEngine::Impl::AllocateVoice(
    const WAVEFORMATEX* wfx,
    SOUND_EFFECT_INSTANCE_FLAGS flags,
    bool oneshot,
    IXAudio2SourceVoice** voice)
{
    if (!wfx)
        throw std::exception("Wave format is required\n");

    // No need to call IsValid on wfx because CreateSourceVoice will do that

    if (!voice)
        throw std::exception("Voice pointer must be non-null");

    *voice = nullptr;

    if (!xaudio2 || mCriticalError)
        return;

#ifndef NDEBUG
    const float maxFrequencyRatio = XAudio2SemitonesToFrequencyRatio(12);
    assert(maxFrequencyRatio <= XAUDIO2_DEFAULT_FREQ_RATIO);
#endif

    unsigned int voiceKey = 0;
    if (oneshot)
    {
        if (flags & (SoundEffectInstance_Use3D | SoundEffectInstance_ReverbUseFilters | SoundEffectInstance_NoSetPitch))
        {
            DebugTrace((flags & SoundEffectInstance_NoSetPitch)
                       ? "ERROR: One-shot voices must support pitch-shifting for voice reuse\n"
                       : "ERROR: One-use voices cannot use 3D positional audio\n");
            throw std::exception("Invalid flags for one-shot voice");
        }

    #ifdef VERBOSE_TRACE
        if (wfx->wFormatTag == WAVE_FORMAT_EXTENSIBLE)
        {
            DebugTrace("INFO: Requesting one-shot: Format Tag EXTENSIBLE %u, %u channels, %u-bit, %u blkalign, %u Hz\n",
                GetFormatTag(wfx), wfx->nChannels, wfx->wBitsPerSample, wfx->nBlockAlign, wfx->nSamplesPerSec);
        }
        else
        {
            DebugTrace("INFO: Requesting one-shot: Format Tag %u, %u channels, %u-bit, %u blkalign, %u Hz\n",
                wfx->wFormatTag, wfx->nChannels, wfx->wBitsPerSample, wfx->nBlockAlign, wfx->nSamplesPerSec);
        }
    #endif

        if (!(mEngineFlags & AudioEngine_DisableVoiceReuse))
        {
            voiceKey = makeVoiceKey(wfx);
            if (voiceKey != 0)
            {
                auto it = mVoicePool.find(voiceKey);
                if (it != mVoicePool.end())
                {
                    // Found a matching (stopped) voice to reuse
                    assert(it->second != nullptr);
                    *voice = it->second;
                    mVoicePool.erase(it);

                    // Reset any volume/pitch-shifting
                    HRESULT hr = (*voice)->SetVolume(1.f);
                    ThrowIfFailed(hr);

                    hr = (*voice)->SetFrequencyRatio(1.f);
                    ThrowIfFailed(hr);

                    if (wfx->nChannels == 1 || wfx->nChannels == 2)
                    {
                        // Reset any panning
                        float matrix[16] = {};
                        ComputePan(0.f, wfx->nChannels, matrix);

                        hr = (*voice)->SetOutputMatrix(nullptr, wfx->nChannels, masterChannels, matrix);
                        ThrowIfFailed(hr);
                    }
                }
                else if ((mVoicePool.size() + mOneShots.size() + 1) >= maxVoiceOneshots)
                {
                    DebugTrace("WARNING: Too many one-shot voices in use (%zu + %zu >= %zu); one-shot not played\n",
                               mVoicePool.size(), mOneShots.size() + 1, maxVoiceOneshots);
                    return;
                }
                else
                {
                    // makeVoiceKey already constrained the supported wfx formats to those supported for reuse

                    char buff[64] = {};
                    auto wfmt = reinterpret_cast<WAVEFORMATEX*>(buff);

                    uint32_t tag = GetFormatTag(wfx);
                    switch (tag)
                    {
                        case WAVE_FORMAT_PCM:
                            CreateIntegerPCM(wfmt, defaultRate, wfx->nChannels, wfx->wBitsPerSample);
                            break;

                        case WAVE_FORMAT_IEEE_FLOAT:
                            CreateFloatPCM(wfmt, defaultRate, wfx->nChannels);
                            break;

                        case WAVE_FORMAT_ADPCM:
                        {
                            auto wfadpcm = reinterpret_cast<const ADPCMWAVEFORMAT*>(wfx);
                            CreateADPCM(wfmt, sizeof(buff), defaultRate, wfx->nChannels, wfadpcm->wSamplesPerBlock);
                        }
                        break;

                        #ifdef DIRECTX_ENABLE_XMA2
                        case WAVE_FORMAT_XMA2:
                            CreateXMA2(wfmt, sizeof(buff), defaultRate, wfx->nChannels, 65536, 2, 0);
                            break;
                        #endif
                    }

                #ifdef VERBOSE_TRACE
                    DebugTrace("INFO: Allocate reuse voice: Format Tag %u, %u channels, %u-bit, %u blkalign, %u Hz\n",
                        wfmt->wFormatTag, wfmt->nChannels, wfmt->wBitsPerSample, wfmt->nBlockAlign, wfmt->nSamplesPerSec);
                #endif

                    assert(voiceKey == makeVoiceKey(wfmt));

                    HRESULT hr = xaudio2->CreateSourceVoice(voice, wfmt, 0, XAUDIO2_DEFAULT_FREQ_RATIO, &mVoiceCallback, nullptr, nullptr);
                    if (FAILED(hr))
                    {
                        DebugTrace("ERROR: CreateSourceVoice (reuse) failed with error %08X\n", static_cast<unsigned int>(hr));
                        throw std::exception("CreateSourceVoice");
                    }
                }

                assert(*voice != nullptr);
                HRESULT hr = (*voice)->SetSourceSampleRate(wfx->nSamplesPerSec);
                if (FAILED(hr))
                {
                    DebugTrace("ERROR: SetSourceSampleRate failed with error %08X\n", static_cast<unsigned int>(hr));
                    throw std::exception("SetSourceSampleRate");
                }
            }
        }
    }

    if (!*voice)
    {
        if (oneshot)
        {
            if ((mVoicePool.size() + mOneShots.size() + 1) >= maxVoiceOneshots)
            {
                DebugTrace("WARNING: Too many one-shot voices in use (%zu + %zu >= %zu); one-shot not played; see TrimVoicePool\n",
                           mVoicePool.size(), mOneShots.size() + 1, maxVoiceOneshots);
                return;
            }
        }
        else if ((mVoiceInstances + 1) >= maxVoiceInstances)
        {
            DebugTrace("ERROR: Too many instance voices (%zu >= %zu); see TrimVoicePool\n",
                mVoiceInstances + 1, maxVoiceInstances);
            throw std::exception("Too many instance voices");
        }

        UINT32 vflags = (flags & SoundEffectInstance_NoSetPitch) ? XAUDIO2_VOICE_NOPITCH : 0u;

        HRESULT hr;
        if (flags & SoundEffectInstance_Use3D)
        {
            XAUDIO2_SEND_DESCRIPTOR sendDescriptors[2] = {};
            sendDescriptors[0].Flags = sendDescriptors[1].Flags = (flags & SoundEffectInstance_ReverbUseFilters)
                ? XAUDIO2_SEND_USEFILTER : 0u;
            sendDescriptors[0].pOutputVoice = mMasterVoice;
            sendDescriptors[1].pOutputVoice = mReverbVoice;
            const XAUDIO2_VOICE_SENDS sendList = { mReverbVoice ? 2U : 1U, sendDescriptors };

        #ifdef VERBOSE_TRACE
            DebugTrace("INFO: Allocate voice 3D: Format Tag %u, %u channels, %u-bit, %u blkalign, %u Hz\n",
                wfx->wFormatTag, wfx->nChannels, wfx->wBitsPerSample, wfx->nBlockAlign, wfx->nSamplesPerSec);
        #endif

            hr = xaudio2->CreateSourceVoice(voice, wfx, vflags, XAUDIO2_DEFAULT_FREQ_RATIO, &mVoiceCallback, &sendList, nullptr);
        }
        else
        {
        #ifdef VERBOSE_TRACE
            DebugTrace("INFO: Allocate voice: Format Tag %u, %u channels, %u-bit, %u blkalign, %u Hz\n",
                wfx->wFormatTag, wfx->nChannels, wfx->wBitsPerSample, wfx->nBlockAlign, wfx->nSamplesPerSec);
        #endif

            hr = xaudio2->CreateSourceVoice(voice, wfx, vflags, XAUDIO2_DEFAULT_FREQ_RATIO, &mVoiceCallback, nullptr, nullptr);
        }

        if (FAILED(hr))
        {
            DebugTrace("ERROR: CreateSourceVoice failed with error %08X\n", static_cast<unsigned int>(hr));
            throw std::exception("CreateSourceVoice");
        }
        else if (!oneshot)
        {
            ++mVoiceInstances;
        }
    }

    if (oneshot)
    {
        assert(*voice != nullptr);
        mOneShots.emplace_back(std::make_pair(voiceKey, *voice));
    }
}

void AudioEngine::Impl::DestroyVoice(_In_ IXAudio2SourceVoice* voice) noexcept
{
    if (!voice)
        return;

#ifndef NDEBUG
    for (auto it = mOneShots.cbegin(); it != mOneShots.cend(); ++it)
    {
        if (it->second == voice)
        {
            DebugTrace("ERROR: DestroyVoice should not be called for a one-shot voice\n");
            return;
        }
    }

    for (auto it = mVoicePool.cbegin(); it != mVoicePool.cend(); ++it)
    {
        if (it->second == voice)
        {
            DebugTrace("ERROR: DestroyVoice should not be called for a one-shot voice; see TrimVoicePool\n");
            return;
        }
    }
#endif

    assert(mVoiceInstances > 0);
    --mVoiceInstances;
    voice->DestroyVoice();
}

void AudioEngine::Impl::RegisterNotify(_In_ IVoiceNotify* notify, bool usesUpdate)
{
    assert(notify != nullptr);
    mNotifyObjects.insert(notify);

    if (usesUpdate)
    {
        mNotifyUpdates.insert(notify);
    }
}

void AudioEngine::Impl::UnregisterNotify(_In_ IVoiceNotify* notify, bool usesOneShots, bool usesUpdate)
{
    assert(notify != nullptr);
    mNotifyObjects.erase(notify);

    // Check for any pending one-shots for this notification object
    if (usesOneShots)
    {
        bool setevent = false;

        for (auto it = mOneShots.begin(); it != mOneShots.end(); ++it)
        {
            assert(it->second != nullptr);

            XAUDIO2_VOICE_STATE state;
            it->second->GetState(&state, XAUDIO2_VOICE_NOSAMPLESPLAYED);

            if (state.pCurrentBufferContext == notify)
            {
                (void)it->second->Stop(0);
                (void)it->second->FlushSourceBuffers();
                setevent = true;
            }
        }

        if (setevent)
        {
            // Trigger scan on next call to Update...
            SetEvent(mVoiceCallback.mBufferEnd.get());
        }
    }

    if (usesUpdate)
    {
        mNotifyUpdates.erase(notify);
    }
}

// AudioEngine

_Use_decl_annotations_
AudioEngine::AudioEngine(
    AUDIO_ENGINE_FLAGS flags,
    const WAVEFORMATEX* wfx,
    const wchar_t* deviceId,
    AUDIO_STREAM_CATEGORY category) noexcept(false)
    : pImpl(std::make_unique<Impl>())
{
    HRESULT hr = pImpl->Initialize(flags, wfx, deviceId, category);
    if (FAILED(hr))
    {
        if (hr == HRESULT_FROM_WIN32(ERROR_NOT_FOUND))
        {
            if (flags & AudioEngine_ThrowOnNoAudioHW)
            {
                DebugTrace("ERROR: AudioEngine found no default audio device\n");
                throw std::exception("AudioEngineNoAudioHW");
            }
            else
            {
                DebugTrace("WARNING: AudioEngine found no default audio device; running in 'silent mode'\n");
            }
        }
        else
        {
            DebugTrace("ERROR: AudioEngine failed (%08X) to initialize using device [%ls]\n",
                static_cast<unsigned int>(hr),
                (deviceId) ? deviceId : L"default");
            throw std::exception("AudioEngine");
        }
    }
}

AudioEngine::AudioEngine(AudioEngine&& moveFrom) noexcept
    : pImpl(std::move(moveFrom.pImpl))
{
}

// Move assignment.
AudioEngine& AudioEngine::operator= (AudioEngine&& moveFrom) noexcept
{
    pImpl = std::move(moveFrom.pImpl);
    return *this;
}

// Public destructor.
AudioEngine::~AudioEngine()
{
    if (pImpl)
    {
        pImpl->Shutdown();
    }
}

// Public methods.
bool AudioEngine::Update()
{
    return pImpl->Update();
}

_Use_decl_annotations_
bool AudioEngine::Reset(const WAVEFORMATEX* wfx, const wchar_t* deviceId)
{
    if (pImpl->xaudio2)
    {
        DebugTrace("WARNING: Called Reset for active audio graph; going silent in preparation for migration\n");
        pImpl->SetSilentMode();
    }

    HRESULT hr = pImpl->Reset(wfx, deviceId);
    if (FAILED(hr))
    {
        if (hr == HRESULT_FROM_WIN32(ERROR_NOT_FOUND))
        {
            if (pImpl->mEngineFlags & AudioEngine_ThrowOnNoAudioHW)
            {
                DebugTrace("ERROR: AudioEngine found no default audio device on Reset\n");
                throw std::exception("AudioEngineNoAudioHW");
            }
            else
            {
                DebugTrace("WARNING: AudioEngine found no default audio device on Reset; running in 'silent mode'\n");
                return false;
            }
        }
        else
        {
            DebugTrace("ERROR: AudioEngine failed (%08X) to Reset using device [%ls]\n",
                static_cast<unsigned int>(hr), (deviceId) ? deviceId : L"default");
            throw std::exception("AudioEngine::Reset");
        }
    }

    DebugTrace("INFO: AudioEngine Reset using device [%ls]\n", (deviceId) ? deviceId : L"default");

    return true;
}

void AudioEngine::Suspend() noexcept
{
    if (!pImpl->xaudio2)
        return;

    pImpl->xaudio2->StopEngine();
}

void AudioEngine::Resume()
{
    if (!pImpl->xaudio2)
        return;

    HRESULT hr = pImpl->xaudio2->StartEngine();
    ThrowIfFailed(hr);
}

float AudioEngine::GetMasterVolume() const noexcept
{
    return pImpl->mMasterVolume;
}

void AudioEngine::SetMasterVolume(float volume)
{
    assert(volume >= -XAUDIO2_MAX_VOLUME_LEVEL && volume <= XAUDIO2_MAX_VOLUME_LEVEL);

    pImpl->mMasterVolume = volume;

    if (pImpl->mMasterVoice)
    {
        HRESULT hr = pImpl->mMasterVoice->SetVolume(volume);
        ThrowIfFailed(hr);
    }
}

void AudioEngine::SetReverb(AUDIO_ENGINE_REVERB reverb)
{
    if (reverb >= Reverb_MAX)
        throw std::out_of_range("AudioEngine::SetReverb");

    if (reverb == Reverb_Off)
    {
        pImpl->SetReverb(nullptr);
    }
    else
    {
        XAUDIO2FX_REVERB_PARAMETERS native;
        ReverbConvertI3DL2ToNative(&gReverbPresets[reverb], &native);
        pImpl->SetReverb(&native);
    }
}

_Use_decl_annotations_
void AudioEngine::SetReverb(const XAUDIO2FX_REVERB_PARAMETERS* native)
{
    pImpl->SetReverb(native);
}

void AudioEngine::SetMasteringLimit(int release, int loudness)
{
    pImpl->SetMasteringLimit(release, loudness);
}

// Public accessors.
AudioStatistics AudioEngine::GetStatistics() const
{
    return pImpl->GetStatistics();
}

WAVEFORMATEXTENSIBLE AudioEngine::GetOutputFormat() const noexcept
{
    WAVEFORMATEXTENSIBLE wfx = {};

    if (!pImpl->xaudio2)
        return wfx;

    wfx.Format.wFormatTag = WAVE_FORMAT_EXTENSIBLE;
    wfx.Format.wBitsPerSample = wfx.Samples.wValidBitsPerSample = 16; // This is a guess
    wfx.Format.cbSize = sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX);

    wfx.Format.nChannels = static_cast<WORD>(pImpl->masterChannels);
    wfx.Format.nSamplesPerSec = pImpl->masterRate;
    wfx.dwChannelMask = pImpl->masterChannelMask;

    wfx.Format.nBlockAlign = static_cast<WORD>(wfx.Format.nChannels * wfx.Format.wBitsPerSample / 8);
    wfx.Format.nAvgBytesPerSec = wfx.Format.nSamplesPerSec * wfx.Format.nBlockAlign;

    static const GUID s_pcm = { WAVE_FORMAT_PCM, 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 } };
    memcpy(&wfx.SubFormat, &s_pcm, sizeof(GUID));

    return wfx;
}

uint32_t AudioEngine::GetChannelMask() const noexcept
{
    return pImpl->masterChannelMask;
}

unsigned int AudioEngine::GetOutputChannels() const noexcept
{
    return pImpl->masterChannels;
}

bool AudioEngine::IsAudioDevicePresent() const noexcept
{
    return pImpl->xaudio2 && !pImpl->mCriticalError;
}

bool AudioEngine::IsCriticalError() const noexcept
{
    return pImpl->mCriticalError;
}

// Voice management.

void AudioEngine::SetMaxVoicePool(size_t maxOneShots, size_t maxInstances)
{
    if (maxOneShots > 0)
        pImpl->maxVoiceOneshots = maxOneShots;

    if (maxInstances > 0)
        pImpl->maxVoiceInstances = maxInstances;
}

void AudioEngine::TrimVoicePool()
{
    pImpl->TrimVoicePool();
}

_Use_decl_annotations_
void AudioEngine::AllocateVoice(
    const WAVEFORMATEX* wfx,
    SOUND_EFFECT_INSTANCE_FLAGS flags,
    bool oneshot,
    IXAudio2SourceVoice** voice)
{
    pImpl->AllocateVoice(wfx, flags, oneshot, voice);
}

void AudioEngine::DestroyVoice(_In_ IXAudio2SourceVoice* voice) noexcept
{
    pImpl->DestroyVoice(voice);
}

void AudioEngine::RegisterNotify(_In_ IVoiceNotify* notify, bool usesUpdate)
{
    pImpl->RegisterNotify(notify, usesUpdate);
}

void AudioEngine::UnregisterNotify(_In_ IVoiceNotify* notify, bool oneshots, bool usesUpdate)
{
    pImpl->UnregisterNotify(notify, oneshots, usesUpdate);
}

IXAudio2* AudioEngine::GetInterface() const noexcept
{
    return pImpl->xaudio2.Get();
}

IXAudio2MasteringVoice* AudioEngine::GetMasterVoice() const noexcept
{
    return pImpl->mMasterVoice;
}

IXAudio2SubmixVoice* AudioEngine::GetReverbVoice() const noexcept
{
    return pImpl->mReverbVoice;
}

X3DAUDIO_HANDLE& AudioEngine::Get3DHandle() const noexcept
{
    return pImpl->mX3DAudio;
}

// Static methods.
#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_GAMES)
#include <mmdeviceapi.h>
#elif defined(_XBOX_ONE)
#include <Windows.Media.Devices.h>
#include <wrl.h>
#elif defined(USING_XAUDIO2_REDIST) || defined(_GAMING_DESKTOP)
#include <mmdeviceapi.h>
#include <functiondiscoverykeys_devpkey.h>
#elif (_WIN32_WINNT >= _WIN32_WINNT_WIN8)
#pragma comment(lib,"runtimeobject.lib")
#pragma warning(push)
#pragma warning(disable: 4471 5204)
#include <windows.devices.enumeration.h>
#pragma warning(pop)
#include <wrl.h>
#endif

std::vector<AudioEngine::RendererDetail> AudioEngine::GetRendererDetails()
{
    std::vector<RendererDetail> list;

#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_GAMES)

    ComPtr<IMMDeviceEnumerator> devEnum;
    HRESULT hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(devEnum.GetAddressOf()));
    ThrowIfFailed(hr);

    ComPtr<IMMDeviceCollection> devices;
    hr = devEnum->EnumAudioEndpoints(eRender, DEVICE_STATE_ACTIVE, &devices);
    ThrowIfFailed(hr);

    ComPtr<IMMDevice> endpoint;
    ThrowIfFailed(devices->Item(0, endpoint.GetAddressOf()));

    LPWSTR id = nullptr;
    ThrowIfFailed(endpoint->GetId(&id));

    RendererDetail device;
    device.deviceId = id;
    device.description = L"Default";

    CoTaskMemFree(id);

    list.emplace_back(device);

#elif defined(_XBOX_ONE)

    using namespace Microsoft::WRL;
    using namespace Microsoft::WRL::Wrappers;
    using namespace ABI::Windows::Foundation;
    using namespace ABI::Windows::Media::Devices;

    ComPtr<IMediaDeviceStatics> mdStatics;
    HRESULT hr = GetActivationFactory(HStringReference(RuntimeClass_Windows_Media_Devices_MediaDevice).Get(), &mdStatics);
    ThrowIfFailed(hr);

    HString id;
    hr = mdStatics->GetDefaultAudioRenderId(AudioDeviceRole_Default, id.GetAddressOf());
    ThrowIfFailed(hr);

    RendererDetail device;
    device.deviceId = id.GetRawBuffer(nullptr);
    device.description = L"Default";
    list.emplace_back(device);

#elif defined(USING_XAUDIO2_REDIST) || defined(_GAMING_DESKTOP)

    ComPtr<IMMDeviceEnumerator> devEnum;
    HRESULT hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(devEnum.GetAddressOf()));
    ThrowIfFailed(hr);

    ComPtr<IMMDeviceCollection> devices;
    hr = devEnum->EnumAudioEndpoints(eRender, DEVICE_STATE_ACTIVE, &devices);
    ThrowIfFailed(hr);

    UINT count = 0;
    ThrowIfFailed(devices->GetCount(&count));

    if (!count)
        return list;

    for (UINT j = 0; j < count; ++j)
    {
        ComPtr<IMMDevice> endpoint;
        hr = devices->Item(j, endpoint.GetAddressOf());
        ThrowIfFailed(hr);

        LPWSTR id = nullptr;
        ThrowIfFailed(endpoint->GetId(&id));

        RendererDetail device;
        device.deviceId = id;
        CoTaskMemFree(id);

        ComPtr<IPropertyStore> props;
        if (SUCCEEDED(endpoint->OpenPropertyStore(STGM_READ, props.GetAddressOf())))
        {
            PROPVARIANT var;
            PropVariantInit(&var);

            if (SUCCEEDED(props->GetValue(PKEY_Device_FriendlyName, &var)))
            {
                if (var.vt == VT_LPWSTR)
                {
                    device.description = var.pwszVal;
                }
                PropVariantClear(&var);
            }
        }

        list.emplace_back(device);
    }

#elif (_WIN32_WINNT >= _WIN32_WINNT_WIN8)

#if defined(__cplusplus_winrt)

    // Enumerating with WinRT using C++/CX (Windows Store apps)
    using Windows::Devices::Enumeration::DeviceClass;
    using Windows::Devices::Enumeration::DeviceInformation;
    using Windows::Devices::Enumeration::DeviceInformationCollection;

    auto operation = DeviceInformation::FindAllAsync(DeviceClass::AudioRender);
    while (operation->Status == Windows::Foundation::AsyncStatus::Started) { Sleep(100); }
    if (operation->Status != Windows::Foundation::AsyncStatus::Completed)
    {
        throw std::exception("FindAllAsync");
    }

    DeviceInformationCollection^ devices = operation->GetResults();

    for (unsigned i = 0; i < devices->Size; ++i)
    {
        using Windows::Devices::Enumeration::DeviceInformation;

        DeviceInformation^ d = devices->GetAt(i);

        RendererDetail device;
        device.deviceId = d->Id->Data();
        device.description = d->Name->Data();
        list.emplace_back(device);
    }
#else

    // Enumerating with WinRT using WRL (Win32 desktop app for Windows 8.x)
    using namespace Microsoft::WRL;
    using namespace Microsoft::WRL::Wrappers;
    using namespace ABI::Windows::Foundation;
    using namespace ABI::Windows::Foundation::Collections;
    using namespace ABI::Windows::Devices::Enumeration;

#if !defined(WINAPI_FAMILY) || (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP)
    RoInitializeWrapper initialize(RO_INIT_MULTITHREADED);
    ThrowIfFailed(initialize);
#endif

    ComPtr<IDeviceInformationStatics> diFactory;
    HRESULT hr = GetActivationFactory(HStringReference(RuntimeClass_Windows_Devices_Enumeration_DeviceInformation).Get(), &diFactory);
    ThrowIfFailed(hr);

    ComPtr<IAsyncOperation<DeviceInformationCollection*>> operation;
    hr = diFactory->FindAllAsyncDeviceClass(DeviceClass_AudioRender, operation.GetAddressOf());
    ThrowIfFailed(hr);

    ComPtr<IAsyncInfo> asyncinfo;
    hr = operation.As(&asyncinfo);
    ThrowIfFailed(hr);

    AsyncStatus status;
    hr = asyncinfo->get_Status(&status);
    ThrowIfFailed(hr);

    while (status == ABI::Windows::Foundation::AsyncStatus::Started)
    {
        Sleep(100);
        hr = asyncinfo->get_Status(&status);
        ThrowIfFailed(hr);
    }

    if (status != ABI::Windows::Foundation::AsyncStatus::Completed)
    {
        throw std::exception("FindAllAsyncDeviceClass");
    }

    ComPtr<IVectorView<DeviceInformation*>> devices;
    hr = operation->GetResults(devices.GetAddressOf());
    ThrowIfFailed(hr);

    unsigned int count = 0;
    hr = devices->get_Size(&count);
    ThrowIfFailed(hr);

    if (!count)
        return list;

    for (unsigned int j = 0; j < count; ++j)
    {
        ComPtr<IDeviceInformation> deviceInfo;
        hr = devices->GetAt(j, deviceInfo.GetAddressOf());
        if (SUCCEEDED(hr))
        {
            RendererDetail device;

            HString id;
            if (SUCCEEDED(deviceInfo->get_Id(id.GetAddressOf())))
            {
                device.deviceId = id.GetRawBuffer(nullptr);
            }

            HString name;
            if (SUCCEEDED(deviceInfo->get_Name(name.GetAddressOf())))
            {
                device.description = name.GetRawBuffer(nullptr);
            }

            list.emplace_back(device);
        }
    }
#endif
#else
#error DirectX Tool Kit for Audio not supported on this platform
#endif

    return list;
}


try
    {
      Real result = task.get();
      // Get rid of unused variable warning:
      std::ostream cnull(0);
      cnull << result;
    }
    catch(std::exception const &)
    {
        caught_exception = true;
    }

/*
 */
#define BOOST_TEST_MODULE naive_monte_carlo_test
#define BOOST_NAIVE_MONTE_CARLO_DEBUG_FAILURES
#include <cmath>
#include <ostream>
#include <boost/lexical_cast.hpp>
#include <boost/type_index.hpp>
#include <boost/test/included/unit_test.hpp>

#include <boost/test/floating_point_comparison.hpp>
#include <boost/math/constants/constants.hpp>
#include <boost/math/quadrature/naive_monte_carlo.hpp>

using std::abs;
using std::vector;
using std::pair;
using boost::math::constants::pi;
using boost::math::quadrature::naive_monte_carlo;

template<class Real>
void test_pi_multithreaded()
{
    std::cout << "Testing pi is calculated correctly (multithreaded) using Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const & x)->Real {
        Real r = x[0]*x[0]+x[1]*x[1];
        if (r <= 1) {
          return 4;
        }
        return 0;
    };

    std::vector<std::pair<Real, Real>> bounds{{Real(0), Real(1)}, {Real(0), Real(1)}};
    Real error_goal = 0.0002;
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, error_goal,
                                          /*singular =*/ false,/* threads = */ 2, /* seed = */ 18012);
    auto task = mc.integrate();
    Real pi_estimated = task.get();
    if (abs(pi_estimated - pi<Real>())/pi<Real>() > 0.005) {
        std::cout << "Error in estimation of pi too high, function calls: " << mc.calls() << "\n";
        std::cout << "Final error estimate : " << mc.current_error_estimate() << "\n";
        std::cout << "Error goal           : " << error_goal << "\n";
        BOOST_CHECK_CLOSE_FRACTION(pi_estimated, pi<Real>(), 0.005);
    }
}

template<class Real>
void test_pi()
{
    std::cout << "Testing pi is calculated correctly using Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const & x)->Real
    {
        Real r = x[0]*x[0]+x[1]*x[1];
        if (r <= 1)
        {
            return 4;
        }
        return 0;
    };

    std::vector<std::pair<Real, Real>> bounds{{Real(0), Real(1)}, {Real(0), Real(1)}};
    Real error_goal = (Real) 0.0002;
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, error_goal,
                                            /*singular =*/ false,/* threads = */ 1, /* seed = */ 128402);
    auto task = mc.integrate();
    Real pi_estimated = task.get();
    if (abs(pi_estimated - pi<Real>())/pi<Real>() > 0.005)
    {
        std::cout << "Error in estimation of pi too high, function calls: " << mc.calls() << "\n";
        std::cout << "Final error estimate : " << mc.current_error_estimate() << "\n";
        std::cout << "Error goal           : " << error_goal << "\n";
        BOOST_CHECK_CLOSE_FRACTION(pi_estimated, pi<Real>(), 0.005);
    }

}

template<class Real>
void test_constant()
{
    std::cout << "Testing constants are integrated correctly using Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const &)->Real
    {
      return 1;
    };

    std::vector<std::pair<Real, Real>> bounds{{Real(0), Real(1)}, { Real(0), Real(1)}};
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.0001,
                                            /* singular = */ false, /* threads = */ 1, /* seed = */ 87);

    auto task = mc.integrate();
    Real one = task.get();
    BOOST_CHECK_CLOSE_FRACTION(one, 1, 0.001);
    BOOST_CHECK_SMALL(mc.current_error_estimate(), std::numeric_limits<Real>::epsilon());
    BOOST_CHECK(mc.calls() > 1000);
}

template<class Real>
void test_exception_from_integrand()
{
    std::cout << "Testing that a reasonable action is performed by the Monte-Carlo integrator when the integrand throws an exception on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const & x)->Real
    {
        if (x[0] > 0.5 && x[0] < 0.5001)
        {
            throw std::domain_error("You have done something wrong.\n");
        }
        return (Real) 1;
    };

    std::vector<std::pair<Real, Real>> bounds{{ Real(0), Real(1)}, { Real(0), Real(1)}};
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.0001);

    auto task = mc.integrate();
    bool caught_exception = false;

    BOOST_CHECK(caught_exception);
}

template<class Real>
void test_cancel_and_restart()
{
    std::cout << "Testing that cancellation and restarting works on naive Monte-Carlo integration on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    Real exact = boost::lexical_cast<Real>("1.3932039296856768591842462603255");
    BOOST_CONSTEXPR const Real A = 1.0 / (pi<Real>() * pi<Real>() * pi<Real>());
    auto g = [&](std::vector<Real> const & x)->Real
    {
        return A / (1.0 - cos(x[0])*cos(x[1])*cos(x[2]));
    };
    vector<pair<Real, Real>> bounds{{ Real(0), pi<Real>()}, { Real(0), pi<Real>()}, { Real(0), pi<Real>()}};
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.05, true, 1, 888889);

    auto task = mc.integrate();
    mc.cancel();
    double y = task.get();
    // Super low tolerance; because it got canceled so fast:
    BOOST_CHECK_CLOSE_FRACTION(y, exact, 1.0);

    mc.update_target_error((Real) 0.01);
    task = mc.integrate();
    y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, exact, 0.1);
}

template<class Real>
void test_finite_singular_boundary()
{
    std::cout << "Testing that finite singular boundaries work on naive Monte-Carlo integration on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    using std::pow;
    using std::log;
    auto g = [](std::vector<Real> const & x)->Real
    {
        // The first term is singular at x = 0.
        // The second at x = 1:
        return pow(log(1.0/x[0]), 2) + log1p(-x[0]);
    };
    vector<pair<Real, Real>> bounds{{Real(0), Real(1)}};
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.01, true, 1, 1922);

    auto task = mc.integrate();

    double y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 1.0, 0.1);
}

template<class Real>
void test_multithreaded_variance()
{
    std::cout << "Testing that variance computed by naive Monte-Carlo integration converges to integral formula on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    Real exact_variance = (Real) 1/(Real) 12;
    auto g = [&](std::vector<Real> const & x)->Real
    {
        return x[0];
    };
    vector<pair<Real, Real>> bounds{{ Real(0), Real(1)}};
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.001, false, 2, 12341);

    auto task = mc.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 0.5, 0.01);
    BOOST_CHECK_CLOSE_FRACTION(mc.variance(), exact_variance, 0.05);
}

template<class Real>
void test_variance()
{
    std::cout << "Testing that variance computed by naive Monte-Carlo integration converges to integral formula on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    Real exact_variance = (Real) 1/(Real) 12;
    auto g = [&](std::vector<Real> const & x)->Real
    {
        return x[0];
    };
    vector<pair<Real, Real>> bounds{{ Real(0), Real(1)}};
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.001, false, 1, 12341);

    auto task = mc.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 0.5, 0.01);
    BOOST_CHECK_CLOSE_FRACTION(mc.variance(), exact_variance, 0.05);
}

template<class Real, uint64_t dimension>
void test_product()
{
    std::cout << "Testing that product functions are integrated correctly by naive Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [&](std::vector<Real> const & x)->Real
    {
        double y = 1;
        for (uint64_t i = 0; i < x.size(); ++i)
        {
            y *= 2*x[i];
        }
        return y;
    };

    vector<pair<Real, Real>> bounds(dimension);
    for (uint64_t i = 0; i < dimension; ++i)
    {
        bounds[i] = std::make_pair<Real, Real>(0, 1);
    }
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.001, false, 1, 13999);

    auto task = mc.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 1, 0.01);
    using std::pow;
    Real exact_variance = pow(4.0/3.0, dimension) - 1;
    BOOST_CHECK_CLOSE_FRACTION(mc.variance(), exact_variance, 0.1);
}

template<class Real, uint64_t dimension>
void test_alternative_rng_1()
{
    std::cout << "Testing that alternative RNGs work correctly using naive Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [&](std::vector<Real> const & x)->Real
    {
        double y = 1;
        for (uint64_t i = 0; i < x.size(); ++i)
        {
            y *= 2*x[i];
        }
        return y;
    };

    vector<pair<Real, Real>> bounds(dimension);
    for (uint64_t i = 0; i < dimension; ++i)
    {
        bounds[i] = std::make_pair<Real, Real>(0, 1);
    }
    std::cout << "Testing std::mt19937" << std::endl;

    naive_monte_carlo<Real, decltype(g), std::mt19937> mc1(g, bounds, (Real) 0.001, false, 1, 1882);

    auto task = mc1.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 1, 0.01);
    using std::pow;
    Real exact_variance = pow(4.0/3.0, dimension) - 1;
    BOOST_CHECK_CLOSE_FRACTION(mc1.variance(), exact_variance, 0.05);

    std::cout << "Testing std::knuth_b" << std::endl;
    naive_monte_carlo<Real, decltype(g), std::knuth_b> mc2(g, bounds, (Real) 0.001, false, 1, 1883);
    task = mc2.integrate();
    y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 1, 0.01);

    std::cout << "Testing std::ranlux48" << std::endl;
    naive_monte_carlo<Real, decltype(g), std::ranlux48> mc3(g, bounds, (Real) 0.001, false, 1, 1884);
    task = mc3.integrate();
    y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 1, 0.01);
}

template<class Real, uint64_t dimension>
void test_alternative_rng_2()
{
    std::cout << "Testing that alternative RNGs work correctly using naive Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [&](std::vector<Real> const & x)->Real
    {
        double y = 1;
        for (uint64_t i = 0; i < x.size(); ++i)
        {
            y *= 2*x[i];
        }
        return y;
    };

    vector<pair<Real, Real>> bounds(dimension);
    for (uint64_t i = 0; i < dimension; ++i)
    {
        bounds[i] = std::make_pair<Real, Real>(0, 1);
    }

    std::cout << "Testing std::default_random_engine" << std::endl;
    naive_monte_carlo<Real, decltype(g), std::default_random_engine> mc4(g, bounds, (Real) 0.001, false, 1, 1884);
    auto task = mc4.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 1, 0.01);

    std::cout << "Testing std::minstd_rand" << std::endl;
    naive_monte_carlo<Real, decltype(g), std::minstd_rand> mc5(g, bounds, (Real) 0.001, false, 1, 1887);
    task = mc5.integrate();
    y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 1, 0.01);

    std::cout << "Testing std::minstd_rand0" << std::endl;
    naive_monte_carlo<Real, decltype(g), std::minstd_rand0> mc6(g, bounds, (Real) 0.001, false, 1, 1889);
    task = mc6.integrate();
    y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, 1, 0.01);

}

template<class Real>
void test_upper_bound_infinite()
{
    std::cout << "Testing that infinite upper bounds are integrated correctly by naive Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const & x)->Real
    {
        return 1.0/(x[0]*x[0] + 1.0);
    };

    vector<pair<Real, Real>> bounds(1);
    for (uint64_t i = 0; i < bounds.size(); ++i)
    {
        bounds[i] = std::make_pair<Real, Real>(0, std::numeric_limits<Real>::infinity());
    }
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.001, true, 1, 8765);
    auto task = mc.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, boost::math::constants::half_pi<Real>(), 0.01);
}

template<class Real>
void test_lower_bound_infinite()
{
    std::cout << "Testing that infinite lower bounds are integrated correctly by naive Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const & x)->Real
    {
        return 1.0/(x[0]*x[0] + 1.0);
    };

    vector<pair<Real, Real>> bounds(1);
    for (uint64_t i = 0; i < bounds.size(); ++i)
    {
        bounds[i] = std::make_pair<Real, Real>(-std::numeric_limits<Real>::infinity(), 0);
    }
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.001, true, 1, 1208);

    auto task = mc.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, boost::math::constants::half_pi<Real>(), 0.01);
}

template<class Real>
void test_lower_bound_infinite2()
{
    std::cout << "Testing that infinite lower bounds (2) are integrated correctly by naive Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const & x)->Real
    {
        // If x[0] = inf, this should blow up:
        return (x[0]*x[0])/(x[0]*x[0]*x[0]*x[0] + 1.0);
    };

    vector<pair<Real, Real>> bounds(1);
    for (uint64_t i = 0; i < bounds.size(); ++i)
    {
        bounds[i] = std::make_pair<Real, Real>(-std::numeric_limits<Real>::infinity(), 0);
    }
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.001, true, 1, 1208);
    auto task = mc.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, boost::math::constants::half_pi<Real>()/boost::math::constants::root_two<Real>(), 0.01);
}

template<class Real>
void test_double_infinite()
{
    std::cout << "Testing that double infinite bounds are integrated correctly by naive Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const & x)->Real
    {
        return 1.0/(x[0]*x[0] + 1.0);
    };

    vector<pair<Real, Real>> bounds(1);
    for (uint64_t i = 0; i < bounds.size(); ++i)
    {
        bounds[i] = std::make_pair<Real, Real>(-std::numeric_limits<Real>::infinity(), std::numeric_limits<Real>::infinity());
    }
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, (Real) 0.001, true, 1, 1776);

    auto task = mc.integrate();
    Real y = task.get();
    BOOST_CHECK_CLOSE_FRACTION(y, boost::math::constants::pi<Real>(), 0.01);
}

template<class Real, uint64_t dimension>
void test_radovic()
{
    // See: Generalized Halton Sequences in 2008: A Comparative Study, function g1:
    std::cout << "Testing that the Radovic function is integrated correctly by naive Monte-Carlo on type " << boost::typeindex::type_id<Real>().pretty_name() << "\n";
    auto g = [](std::vector<Real> const & x)->Real
    {
        using std::abs;
        Real alpha = (Real)0.01;
        Real z = 1;
        for (uint64_t i = 0; i < dimension; ++i)
        {
            z *= (abs(4*x[i]-2) + alpha)/(1+alpha);
        }
        return z;
    };

    vector<pair<Real, Real>> bounds(dimension);
    for (uint64_t i = 0; i < bounds.size(); ++i)
    {
        bounds[i] = std::make_pair<Real, Real>(0, 1);
    }
    Real error_goal = (Real) 0.001;
    naive_monte_carlo<Real, decltype(g)> mc(g, bounds, error_goal, false, 1, 1982);

    auto task = mc.integrate();
    Real y = task.get();
    if (abs(y - 1) > 0.01)
    {
        std::cout << "Error in estimation of Radovic integral too high, function calls: " << mc.calls() << "\n";
        std::cout << "Final error estimate: " << mc.current_error_estimate() << std::endl;
        std::cout << "Error goal          : " << error_goal << std::endl;
        std::cout << "Variance estimate   : " << mc.variance() << std::endl;
        BOOST_CHECK_CLOSE_FRACTION(y, 1, 0.01);
    }
}

BOOST_AUTO_TEST_CASE(naive_monte_carlo_test)
{
   std::cout << "Default hardware concurrency = " << std::thread::hardware_concurrency() << std::endl;
#if !defined(TEST) || TEST == 1
    test_finite_singular_boundary<double>();
    test_finite_singular_boundary<float>();
#endif
#if !defined(TEST) || TEST == 2
    test_pi<float>();
    test_pi<double>();
#if !defined(TEST) || TEST == 3
    test_pi_multithreaded<float>();
    test_constant<float>();
#endif
#endif
#if !defined(TEST) || TEST == 4
    test_constant<double>();
    test_cancel_and_restart<float>();
#endif
#if !defined(TEST) || TEST == 5
    test_exception_from_integrand<float>();
    test_variance<float>();
#endif
#if !defined(TEST) || TEST == 6
    test_variance<double>();
    test_multithreaded_variance<double>();
#endif
#if !defined(TEST) || TEST == 7
    test_product<float, 1>();
    test_product<float, 2>();
#endif
#if !defined(TEST) || TEST == 8
    test_product<float, 3>();
    test_product<float, 4>();
    test_product<float, 5>();
#endif
#if !defined(TEST) || TEST == 9
    test_product<float, 6>();
    test_product<double, 1>();
#endif
#if !defined(TEST) || TEST == 10
    test_product<double, 2>();
#endif
#if !defined(TEST) || TEST == 11
    test_product<double, 3>();
    test_product<double, 4>();
#endif
#if !defined(TEST) || TEST == 12
    test_upper_bound_infinite<float>();
    test_upper_bound_infinite<double>();
#endif
#if !defined(TEST) || TEST == 13
    test_lower_bound_infinite<float>();
    test_lower_bound_infinite<double>();
#endif
#if !defined(TEST) || TEST == 14
    test_lower_bound_infinite2<float>();
#endif
#if !defined(TEST) || TEST == 15
    test_double_infinite<float>();
    test_double_infinite<double>();
#endif
#if !defined(TEST) || TEST == 16
    test_radovic<float, 1>();
    test_radovic<float, 2>();
#endif
#if !defined(TEST) || TEST == 17
    test_radovic<float, 3>();
    test_radovic<double, 1>();
#endif
#if !defined(TEST) || TEST == 18
    test_radovic<double, 2>();
    test_radovic<double, 3>();
#endif
#if !defined(TEST) || TEST == 19
    test_radovic<double, 4>();
    test_radovic<double, 5>();
#endif
#if !defined(TEST) || TEST == 20
    test_alternative_rng_1<float, 3>();
#endif
#if !defined(TEST) || TEST == 21
    test_alternative_rng_1<double, 3>();
#endif
#if !defined(TEST) || TEST == 22
    test_alternative_rng_2<float, 3>();
#endif
#if !defined(TEST) || TEST == 23
    test_alternative_rng_2<double, 3>();
#endif

}


extern "C" {
  extern "C" void f(int);
}

extern "C++" {
  extern "C++" int& g(int);
  float& g();
}
double& g(double);

void test(int x, double d) {
  f(x);
  float &f1 = g();
  int& i1 = g(x);
  double& d1 = g(d);
}

extern "C" int foo;
extern "C" int foo;

extern "C" const int bar;
extern "C" int const bar;

extern "C" struct bar d;
extern struct bar e;

extern "C++" {
  namespace N0 {
    struct X0 {
      int foo(int x) { return x; }
    };
  }
}

// PR5430
namespace pr5430 {
  extern "C" void func(void);
}
using namespace pr5430;
extern "C" void pr5430::func(void) { }

// PR5404

extern "C"
{
    int f2(int)
    {
        return f2((char *)0);
    }
}

// PR6991
extern "C" typedef int (*PutcFunc_t)(int);

// PR7859
extern "C" void pr7859_a(int) {} // expected-note {{previous definition}}
extern "C" void pr7859_a(int) {} // expected-error {{redefinition}}

extern "C" void pr7859_b() {} // expected-note {{previous definition}}
extern "C" void pr7859_b(int) {} // expected-error {{conflicting}}

extern "C" void pr7859_c(short) {} // expected-note {{previous definition}}
extern "C" void pr7859_c(int) {} // expected-error {{conflicting}}

extern "C" {
  struct s0 {
  private:
    s0();
    s0(const s0 &);
  };
}

//PR7754
extern "C++" template <class T> int pr7754(T param);

namespace N {
  int value;
}

extern "C++" using N::value;

// PR7076
extern "C" const char *Version_string = "2.9";

namespace PR9162 {
  extern "C" {
    typedef struct _ArtsSink ArtsSink;
    struct _ArtsSink {
      int sink;
    };
  }
  int arts_sink_get_type()
  {
    return sizeof(ArtsSink);
  }
}


FenceNV11::FenceNV11(Renderer11 *renderer) : FenceNVImpl(), mRenderer(renderer), mQuery(nullptr) {}

// Fence11.cpp: Defines the rx::FenceNV11 and rx::Sync11 classes which implement
// rx::FenceNVImpl and rx::SyncImpl.

#include "libANGLE/renderer/d3d/d3d11/Fence11.h"

#include "common/utilities.h"
#include "libANGLE/Context.h"
#include "libANGLE/renderer/d3d/d3d11/Context11.h"
#include "libANGLE/renderer/d3d/d3d11/Renderer11.h"

namespace rx
{

// Template helpers for set and test operations.

template <class FenceClass>
angle::Result FenceSetHelper(const gl::Context *context, FenceClass *fence)
{
    if (!fence->mQuery)
    {
        D3D11_QUERY_DESC queryDesc;
        queryDesc.Query     = D3D11_QUERY_EVENT;
        queryDesc.MiscFlags = 0;

        Context11 *context11 = GetImplAs<Context11>(context);
        HRESULT result = fence->mRenderer->getDevice()->CreateQuery(&queryDesc, &fence->mQuery);
        ANGLE_TRY_HR(context11, result, "Failed to create event query");
    }

    fence->mRenderer->getDeviceContext()->End(fence->mQuery);
    return angle::Result::Continue;
}

template <class FenceClass>
angle::Result FenceTestHelper(const gl::Context *context,
                              FenceClass *fence,
                              bool flushCommandBuffer,
                              GLboolean *outFinished)
{
    ASSERT(fence->mQuery);

    UINT getDataFlags = (flushCommandBuffer ? 0 : D3D11_ASYNC_GETDATA_DONOTFLUSH);

    Context11 *context11 = GetImplAs<Context11>(context);
    HRESULT result =
        fence->mRenderer->getDeviceContext()->GetData(fence->mQuery, nullptr, 0, getDataFlags);
    ANGLE_TRY_HR(context11, result, "Failed to get query data");

    ASSERT(result == S_OK || result == S_FALSE);
    *outFinished = ((result == S_OK) ? GL_TRUE : GL_FALSE);
    return angle::Result::Continue;
}

// FenceNV11

FenceNV11::~FenceNV11()
{
    SafeRelease(mQuery);
}

angle::Result FenceNV11::set(const gl::Context *context, GLenum condition)
{
    return FenceSetHelper(context, this);
}

angle::Result FenceNV11::test(const gl::Context *context, GLboolean *outFinished)
{
    return FenceTestHelper(context, this, true, outFinished);
}

angle::Result FenceNV11::finish(const gl::Context *context)
{
    GLboolean finished = GL_FALSE;

    int loopCount = 0;
    while (finished != GL_TRUE)
    {
        loopCount++;
        ANGLE_TRY(FenceTestHelper(context, this, true, &finished));

        bool checkDeviceLost = (loopCount % kPollingD3DDeviceLostCheckFrequency) == 0;
        if (checkDeviceLost && mRenderer->testDeviceLost())
        {
            ANGLE_TRY_HR(GetImplAs<Context11>(context), DXGI_ERROR_DRIVER_INTERNAL_ERROR,
                         "Device was lost while querying result of an event query.");
        }

        ScheduleYield();
    }

    return angle::Result::Continue;
}

// Sync11

// Important note on accurate timers in Windows:
// QueryPerformanceCounter has a few major issues, including being 10x as expensive to call
// as timeGetTime on laptops and "jumping" during certain hardware events.
// See the comments at the top of the Chromium source file "chromium/src/base/time/time_win.cc"
// We still opt to use QPC. In the present and moving forward, most newer systems will not suffer
// from buggy implementations.

Sync11::Sync11(Renderer11 *renderer) : SyncImpl(), mRenderer(renderer), mQuery(nullptr)
{
    LARGE_INTEGER counterFreqency = {};
    BOOL success                  = QueryPerformanceFrequency(&counterFreqency);
    ASSERT(success);

    mCounterFrequency = counterFreqency.QuadPart;
}

Sync11::~Sync11()
{
    SafeRelease(mQuery);
}

angle::Result Sync11::set(const gl::Context *context, GLenum condition, GLbitfield flags)
{
    ASSERT(condition == GL_SYNC_GPU_COMMANDS_COMPLETE && flags == 0);
    return FenceSetHelper(context, this);
}

angle::Result Sync11::clientWait(const gl::Context *context,
                                 GLbitfield flags,
                                 GLuint64 timeout,
                                 GLenum *outResult)
{
    ASSERT(outResult);

    bool flushCommandBuffer = ((flags & GL_SYNC_FLUSH_COMMANDS_BIT) != 0);

    *outResult = GL_WAIT_FAILED;

    GLboolean result = GL_FALSE;
    ANGLE_TRY(FenceTestHelper(context, this, flushCommandBuffer, &result));

    if (result == GL_TRUE)
    {
        *outResult = GL_ALREADY_SIGNALED;
        return angle::Result::Continue;
    }

    if (timeout == 0)
    {
        *outResult = GL_TIMEOUT_EXPIRED;
        return angle::Result::Continue;
    }

    LARGE_INTEGER currentCounter = {};
    BOOL success                 = QueryPerformanceCounter(&currentCounter);
    ASSERT(success);

    LONGLONG timeoutInSeconds = static_cast<LONGLONG>(timeout / 1000000000ull);
    LONGLONG endCounter       = currentCounter.QuadPart + mCounterFrequency * timeoutInSeconds;

    // Extremely unlikely, but if mCounterFrequency is large enough, endCounter can wrap
    if (endCounter < currentCounter.QuadPart)
    {
        endCounter = MAXLONGLONG;
    }

    int loopCount = 0;
    while (currentCounter.QuadPart < endCounter && !result)
    {
        loopCount++;
        ScheduleYield();
        success = QueryPerformanceCounter(&currentCounter);
        ASSERT(success);

        *outResult = GL_WAIT_FAILED;

        ANGLE_TRY(FenceTestHelper(context, this, flushCommandBuffer, &result));

        bool checkDeviceLost = (loopCount % kPollingD3DDeviceLostCheckFrequency) == 0;
        if (checkDeviceLost && mRenderer->testDeviceLost())
        {
            *outResult = GL_WAIT_FAILED;
            ANGLE_TRY_HR(GetImplAs<Context11>(context), DXGI_ERROR_DRIVER_INTERNAL_ERROR,
                         "Device was lost while querying result of an event query.");
        }
    }

    if (currentCounter.QuadPart >= endCounter)
    {
        *outResult = GL_TIMEOUT_EXPIRED;
    }
    else
    {
        *outResult = GL_CONDITION_SATISFIED;
    }

    return angle::Result::Continue;
}

angle::Result Sync11::serverWait(const gl::Context *context, GLbitfield flags, GLuint64 timeout)
{
    // Because our API is currently designed to be called from a single thread, we don't need to do
    // extra work for a server-side fence. GPU commands issued after the fence is created will
    // always be processed after the fence is signaled.
    return angle::Result::Continue;
}

angle::Result Sync11::getStatus(const gl::Context *context, GLint *outResult)
{
    GLboolean result = GL_FALSE;

    // The spec does not specify any way to report errors during the status test (e.g. device
    // lost) so we report the fence is unblocked in case of error or signaled.
    *outResult = GL_SIGNALED;
    ANGLE_TRY(FenceTestHelper(context, this, false, &result));

    *outResult = (result ? GL_SIGNALED : GL_UNSIGNALED);
    return angle::Result::Continue;
}

}  // namespace rx


static int resultIndex(int i) { return -1 - i; }

// Operator wrapper to simplify using TableGen Record defining a MLIR Op.

#include "mlir/TableGen/Operator.h"
#include "mlir/TableGen/OpTrait.h"
#include "mlir/TableGen/Predicate.h"
#include "mlir/TableGen/Type.h"
#include "llvm/ADT/EquivalenceClasses.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/TableGen/Error.h"
#include "llvm/TableGen/Record.h"

#define DEBUG_TYPE "mlir-tblgen-operator"

using namespace mlir;
using namespace mlir::tblgen;

using llvm::DagInit;
using llvm::DefInit;
using llvm::Record;

Operator::Operator(const llvm::Record &def)
    : dialect(def.getValueAsDef("opDialect")), def(def) {
  // The first `_` in the op's TableGen def name is treated as separating the
  // dialect prefix and the op class name. The dialect prefix will be ignored if
  // not empty. Otherwise, if def name starts with a `_`, the `_` is considered
  // as part of the class name.
  StringRef prefix;
  std::tie(prefix, cppClassName) = def.getName().split('_');
  if (prefix.empty()) {
    // Class name with a leading underscore and without dialect prefix
    cppClassName = def.getName();
  } else if (cppClassName.empty()) {
    // Class name without dialect prefix
    cppClassName = prefix;
  }

  populateOpStructure();
}

std::string Operator::getOperationName() const {
  auto prefix = dialect.getName();
  auto opName = def.getValueAsString("opName");
  if (prefix.empty())
    return std::string(opName);
  return std::string(llvm::formatv("{0}.{1}", prefix, opName));
}

std::string Operator::getAdaptorName() const {
  return std::string(llvm::formatv("{0}Adaptor", getCppClassName()));
}

StringRef Operator::getDialectName() const { return dialect.getName(); }

StringRef Operator::getCppClassName() const { return cppClassName; }

std::string Operator::getQualCppClassName() const {
  auto prefix = dialect.getCppNamespace();
  if (prefix.empty())
    return std::string(cppClassName);
  return std::string(llvm::formatv("{0}::{1}", prefix, cppClassName));
}

int Operator::getNumResults() const {
  DagInit *results = def.getValueAsDag("results");
  return results->getNumArgs();
}

StringRef Operator::getExtraClassDeclaration() const {
  constexpr auto attr = "extraClassDeclaration";
  if (def.isValueUnset(attr))
    return {};
  return def.getValueAsString(attr);
}

const llvm::Record &Operator::getDef() const { return def; }

bool Operator::skipDefaultBuilders() const {
  return def.getValueAsBit("skipDefaultBuilders");
}

auto Operator::result_begin() -> value_iterator { return results.begin(); }

auto Operator::result_end() -> value_iterator { return results.end(); }

auto Operator::getResults() -> value_range {
  return {result_begin(), result_end()};
}

TypeConstraint Operator::getResultTypeConstraint(int index) const {
  DagInit *results = def.getValueAsDag("results");
  return TypeConstraint(cast<DefInit>(results->getArg(index)));
}

StringRef Operator::getResultName(int index) const {
  DagInit *results = def.getValueAsDag("results");
  return results->getArgNameStr(index);
}

auto Operator::getResultDecorators(int index) const -> var_decorator_range {
  Record *result =
      cast<DefInit>(def.getValueAsDag("results")->getArg(index))->getDef();
  if (!result->isSubClassOf("OpVariable"))
    return var_decorator_range(nullptr, nullptr);
  return *result->getValueAsListInit("decorators");
}

unsigned Operator::getNumVariableLengthResults() const {
  return llvm::count_if(results, [](const NamedTypeConstraint &c) {
    return c.constraint.isVariableLength();
  });
}

unsigned Operator::getNumVariableLengthOperands() const {
  return llvm::count_if(operands, [](const NamedTypeConstraint &c) {
    return c.constraint.isVariableLength();
  });
}

bool Operator::hasSingleVariadicArg() const {
  return getNumArgs() == 1 && getArg(0).is<NamedTypeConstraint *>() &&
         getOperand(0).isVariadic();
}

Operator::arg_iterator Operator::arg_begin() const { return arguments.begin(); }

Operator::arg_iterator Operator::arg_end() const { return arguments.end(); }

Operator::arg_range Operator::getArgs() const {
  return {arg_begin(), arg_end()};
}

StringRef Operator::getArgName(int index) const {
  DagInit *argumentValues = def.getValueAsDag("arguments");
  return argumentValues->getArgName(index)->getValue();
}

auto Operator::getArgDecorators(int index) const -> var_decorator_range {
  Record *arg =
      cast<DefInit>(def.getValueAsDag("arguments")->getArg(index))->getDef();
  if (!arg->isSubClassOf("OpVariable"))
    return var_decorator_range(nullptr, nullptr);
  return *arg->getValueAsListInit("decorators");
}

const OpTrait *Operator::getTrait(StringRef trait) const {
  for (const auto &t : traits) {
    if (const auto *opTrait = dyn_cast<NativeOpTrait>(&t)) {
      if (opTrait->getTrait() == trait)
        return opTrait;
    } else if (const auto *opTrait = dyn_cast<InternalOpTrait>(&t)) {
      if (opTrait->getTrait() == trait)
        return opTrait;
    } else if (const auto *opTrait = dyn_cast<InterfaceOpTrait>(&t)) {
      if (opTrait->getTrait() == trait)
        return opTrait;
    }
  }
  return nullptr;
}

auto Operator::region_begin() const -> const_region_iterator {
  return regions.begin();
}
auto Operator::region_end() const -> const_region_iterator {
  return regions.end();
}
auto Operator::getRegions() const
    -> llvm::iterator_range<const_region_iterator> {
  return {region_begin(), region_end()};
}

unsigned Operator::getNumRegions() const { return regions.size(); }

const NamedRegion &Operator::getRegion(unsigned index) const {
  return regions[index];
}

unsigned Operator::getNumVariadicRegions() const {
  return llvm::count_if(regions,
                        [](const NamedRegion &c) { return c.isVariadic(); });
}

auto Operator::successor_begin() const -> const_successor_iterator {
  return successors.begin();
}
auto Operator::successor_end() const -> const_successor_iterator {
  return successors.end();
}
auto Operator::getSuccessors() const
    -> llvm::iterator_range<const_successor_iterator> {
  return {successor_begin(), successor_end()};
}

unsigned Operator::getNumSuccessors() const { return successors.size(); }

const NamedSuccessor &Operator::getSuccessor(unsigned index) const {
  return successors[index];
}

unsigned Operator::getNumVariadicSuccessors() const {
  return llvm::count_if(successors,
                        [](const NamedSuccessor &c) { return c.isVariadic(); });
}

auto Operator::trait_begin() const -> const_trait_iterator {
  return traits.begin();
}
auto Operator::trait_end() const -> const_trait_iterator {
  return traits.end();
}
auto Operator::getTraits() const -> llvm::iterator_range<const_trait_iterator> {
  return {trait_begin(), trait_end()};
}

auto Operator::attribute_begin() const -> attribute_iterator {
  return attributes.begin();
}
auto Operator::attribute_end() const -> attribute_iterator {
  return attributes.end();
}
auto Operator::getAttributes() const
    -> llvm::iterator_range<attribute_iterator> {
  return {attribute_begin(), attribute_end()};
}

auto Operator::operand_begin() -> value_iterator { return operands.begin(); }
auto Operator::operand_end() -> value_iterator { return operands.end(); }
auto Operator::getOperands() -> value_range {
  return {operand_begin(), operand_end()};
}

auto Operator::getArg(int index) const -> Argument { return arguments[index]; }

// Mapping from result index to combined argument and result index. Arguments
// are indexed to match getArg index, while the result indexes are mapped to
// avoid overlap.

bool Operator::isVariadic() const {
  return any_of(llvm::concat<const NamedTypeConstraint>(operands, results),
                [](const NamedTypeConstraint &op) { return op.isVariadic(); });
}

void Operator::populateTypeInferenceInfo(
    const llvm::StringMap<int> &argumentsAndResultsIndex) {
  // If the type inference op interface is not registered, then do not attempt
  // to determine if the result types an be inferred.
  auto &recordKeeper = def.getRecords();
  auto *inferTrait = recordKeeper.getDef(inferTypeOpInterface);
  allResultsHaveKnownTypes = false;
  if (!inferTrait)
    return;

  // If there are no results, the skip this else the build method generated
  // overlaps with another autogenerated builder.
  if (getNumResults() == 0)
    return;

  // Skip for ops with variadic operands/results.
  // TODO: This can be relaxed.
  if (isVariadic())
    return;

  // Skip cases currently being custom generated.
  // TODO: Remove special cases.
  if (getTrait("::mlir::OpTrait::SameOperandsAndResultType"))
    return;

  // We create equivalence classes of argument/result types where arguments
  // and results are mapped into the same index space and indices corresponding
  // to the same type are in the same equivalence class.
  llvm::EquivalenceClasses<int> ecs;
  resultTypeMapping.resize(getNumResults());
  // Captures the argument whose type matches a given result type. Preference
  // towards capturing operands first before attributes.
  auto captureMapping = [&](int i) {
    bool found = false;
    ecs.insert(resultIndex(i));
    auto mi = ecs.findLeader(resultIndex(i));
    for (auto me = ecs.member_end(); mi != me; ++mi) {
      if (*mi < 0) {
        auto tc = getResultTypeConstraint(i);
        if (tc.getBuilderCall().hasValue()) {
          resultTypeMapping[i].emplace_back(tc);
          found = true;
        }
        continue;
      }

      if (getArg(*mi).is<NamedAttribute *>()) {
        // TODO: Handle attributes.
        continue;
      } else {
        resultTypeMapping[i].emplace_back(*mi);
        found = true;
      }
    }
    return found;
  };

  for (const OpTrait &trait : traits) {
    const llvm::Record &def = trait.getDef();
    // If the infer type op interface was manually added, then treat it as
    // intention that the op needs special handling.
    // TODO: Reconsider whether to always generate, this is more conservative
    // and keeps existing behavior so starting that way for now.
    if (def.isSubClassOf(
            llvm::formatv("{0}::Trait", inferTypeOpInterface).str()))
      return;
    if (const auto *opTrait = dyn_cast<InterfaceOpTrait>(&trait))
      if (&opTrait->getDef() == inferTrait)
        return;

    if (!def.isSubClassOf("AllTypesMatch"))
      continue;

    auto values = def.getValueAsListOfStrings("values");
    auto root = argumentsAndResultsIndex.lookup(values.front());
    for (StringRef str : values)
      ecs.unionSets(argumentsAndResultsIndex.lookup(str), root);
  }

  // Verifies that all output types have a corresponding known input type
  // and chooses matching operand or attribute (in that order) that
  // matches it.
  allResultsHaveKnownTypes =
      all_of(llvm::seq<int>(0, getNumResults()), captureMapping);

  // If the types could be computed, then add type inference trait.
  if (allResultsHaveKnownTypes)
    traits.push_back(OpTrait::create(inferTrait->getDefInit()));
}

void Operator::populateOpStructure() {
  auto &recordKeeper = def.getRecords();
  auto *typeConstraintClass = recordKeeper.getClass("TypeConstraint");
  auto *attrClass = recordKeeper.getClass("Attr");
  auto *derivedAttrClass = recordKeeper.getClass("DerivedAttr");
  auto *opVarClass = recordKeeper.getClass("OpVariable");
  numNativeAttributes = 0;

  DagInit *argumentValues = def.getValueAsDag("arguments");
  unsigned numArgs = argumentValues->getNumArgs();

  // Mapping from name of to argument or result index. Arguments are indexed
  // to match getArg index, while the results are negatively indexed.
  llvm::StringMap<int> argumentsAndResultsIndex;

  // Handle operands and native attributes.
  for (unsigned i = 0; i != numArgs; ++i) {
    auto *arg = argumentValues->getArg(i);
    auto givenName = argumentValues->getArgNameStr(i);
    auto *argDefInit = dyn_cast<DefInit>(arg);
    if (!argDefInit)
      PrintFatalError(def.getLoc(),
                      Twine("undefined type for argument #") + Twine(i));
    Record *argDef = argDefInit->getDef();
    if (argDef->isSubClassOf(opVarClass))
      argDef = argDef->getValueAsDef("constraint");

    if (argDef->isSubClassOf(typeConstraintClass)) {
      operands.push_back(
          NamedTypeConstraint{givenName, TypeConstraint(argDef)});
    } else if (argDef->isSubClassOf(attrClass)) {
      if (givenName.empty())
        PrintFatalError(argDef->getLoc(), "attributes must be named");
      if (argDef->isSubClassOf(derivedAttrClass))
        PrintFatalError(argDef->getLoc(),
                        "derived attributes not allowed in argument list");
      attributes.push_back({givenName, Attribute(argDef)});
      ++numNativeAttributes;
    } else {
      PrintFatalError(def.getLoc(), "unexpected def type; only defs deriving "
                                    "from TypeConstraint or Attr are allowed");
    }
    if (!givenName.empty())
      argumentsAndResultsIndex[givenName] = i;
  }

  // Handle derived attributes.
  for (const auto &val : def.getValues()) {
    if (auto *record = dyn_cast<llvm::RecordRecTy>(val.getType())) {
      if (!record->isSubClassOf(attrClass))
        continue;
      if (!record->isSubClassOf(derivedAttrClass))
        PrintFatalError(def.getLoc(),
                        "unexpected Attr where only DerivedAttr is allowed");

      if (record->getClasses().size() != 1) {
        PrintFatalError(
            def.getLoc(),
            "unsupported attribute modelling, only single class expected");
      }
      attributes.push_back(
          {cast<llvm::StringInit>(val.getNameInit())->getValue(),
           Attribute(cast<DefInit>(val.getValue()))});
    }
  }

  // Populate `arguments`. This must happen after we've finalized `operands` and
  // `attributes` because we will put their elements' pointers in `arguments`.
  // SmallVector may perform re-allocation under the hood when adding new
  int operandIndex = 0, attrIndex = 0;
  for (unsigned i = 0; i != numArgs; ++i) {
    Record *argDef = dyn_cast<DefInit>(argumentValues->getArg(i))->getDef();
    if (argDef->isSubClassOf(opVarClass))
      argDef = argDef->getValueAsDef("constraint");

    if (argDef->isSubClassOf(typeConstraintClass)) {
      attrOrOperandMapping.push_back(
          {OperandOrAttribute::Kind::Operand, operandIndex});
      arguments.emplace_back(&operands[operandIndex++]);
    } else {
      assert(argDef->isSubClassOf(attrClass));
      attrOrOperandMapping.push_back(
          {OperandOrAttribute::Kind::Attribute, attrIndex});
      arguments.emplace_back(&attributes[attrIndex++]);
    }
  }

  auto *resultsDag = def.getValueAsDag("results");
  auto *outsOp = dyn_cast<DefInit>(resultsDag->getOperator());
  if (!outsOp || outsOp->getDef()->getName() != "outs") {
    PrintFatalError(def.getLoc(), "'results' must have 'outs' directive");
  }

  // Handle results.
  for (unsigned i = 0, e = resultsDag->getNumArgs(); i < e; ++i) {
    auto name = resultsDag->getArgNameStr(i);
    auto *resultInit = dyn_cast<DefInit>(resultsDag->getArg(i));
    if (!resultInit) {
      PrintFatalError(def.getLoc(),
                      Twine("undefined type for result #") + Twine(i));
    }
    auto *resultDef = resultInit->getDef();
    if (resultDef->isSubClassOf(opVarClass))
      resultDef = resultDef->getValueAsDef("constraint");
    results.push_back({name, TypeConstraint(resultDef)});
    if (!name.empty())
      argumentsAndResultsIndex[name] = resultIndex(i);
  }

  // Handle successors
  auto *successorsDag = def.getValueAsDag("successors");
  auto *successorsOp = dyn_cast<DefInit>(successorsDag->getOperator());
  if (!successorsOp || successorsOp->getDef()->getName() != "successor") {
    PrintFatalError(def.getLoc(),
                    "'successors' must have 'successor' directive");
  }

  for (unsigned i = 0, e = successorsDag->getNumArgs(); i < e; ++i) {
    auto name = successorsDag->getArgNameStr(i);
    auto *successorInit = dyn_cast<DefInit>(successorsDag->getArg(i));
    if (!successorInit) {
      PrintFatalError(def.getLoc(),
                      Twine("undefined kind for successor #") + Twine(i));
    }
    Successor successor(successorInit->getDef());

    // Only support variadic successors if it is the last one for now.
    if (i != e - 1 && successor.isVariadic())
      PrintFatalError(def.getLoc(), "only the last successor can be variadic");
    successors.push_back({name, successor});
  }

  // Create list of traits, skipping over duplicates: appending to lists in
  // tablegen is easy, making them unique less so, so dedupe here.
  if (auto *traitList = def.getValueAsListInit("traits")) {
    // This is uniquing based on pointers of the trait.
    SmallPtrSet<const llvm::Init *, 32> traitSet;
    traits.reserve(traitSet.size());
    for (auto *traitInit : *traitList) {
      // Keep traits in the same order while skipping over duplicates.
      if (traitSet.insert(traitInit).second)
        traits.push_back(OpTrait::create(traitInit));
    }
  }

  populateTypeInferenceInfo(argumentsAndResultsIndex);

  // Handle regions
  auto *regionsDag = def.getValueAsDag("regions");
  auto *regionsOp = dyn_cast<DefInit>(regionsDag->getOperator());
  if (!regionsOp || regionsOp->getDef()->getName() != "region") {
    PrintFatalError(def.getLoc(), "'regions' must have 'region' directive");
  }

  for (unsigned i = 0, e = regionsDag->getNumArgs(); i < e; ++i) {
    auto name = regionsDag->getArgNameStr(i);
    auto *regionInit = dyn_cast<DefInit>(regionsDag->getArg(i));
    if (!regionInit) {
      PrintFatalError(def.getLoc(),
                      Twine("undefined kind for region #") + Twine(i));
    }
    Region region(regionInit->getDef());
    if (region.isVariadic()) {
      // Only support variadic regions if it is the last one for now.
      if (i != e - 1)
        PrintFatalError(def.getLoc(), "only the last region can be variadic");
      if (name.empty())
        PrintFatalError(def.getLoc(), "variadic regions must be named");
    }

    regions.push_back({name, region});
  }

  LLVM_DEBUG(print(llvm::dbgs()));
}

auto Operator::getSameTypeAsResult(int index) const -> ArrayRef<ArgOrType> {
  assert(allResultTypesKnown());
  return resultTypeMapping[index];
}

ArrayRef<llvm::SMLoc> Operator::getLoc() const { return def.getLoc(); }

bool Operator::hasDescription() const {
  return def.getValue("description") != nullptr;
}

StringRef Operator::getDescription() const {
  return def.getValueAsString("description");
}

bool Operator::hasSummary() const { return def.getValue("summary") != nullptr; }

StringRef Operator::getSummary() const {
  return def.getValueAsString("summary");
}

bool Operator::hasAssemblyFormat() const {
  auto *valueInit = def.getValueInit("assemblyFormat");
  return isa<llvm::CodeInit, llvm::StringInit>(valueInit);
}

StringRef Operator::getAssemblyFormat() const {
  return TypeSwitch<llvm::Init *, StringRef>(def.getValueInit("assemblyFormat"))
      .Case<llvm::StringInit, llvm::CodeInit>(
          [&](auto *init) { return init->getValue(); });
}

void Operator::print(llvm::raw_ostream &os) const {
  os << "op '" << getOperationName() << "'\n";
  for (Argument arg : arguments) {
    if (auto *attr = arg.dyn_cast<NamedAttribute *>())
      os << "[attribute] " << attr->name << '\n';
    else
      os << "[operand] " << arg.get<NamedTypeConstraint *>()->name << '\n';
  }
}

Operator::NamespaceEmitter::NamespaceEmitter(raw_ostream &os, Operator &op)
    : os(os) {
  auto dialect = op.getDialect();
  if (!dialect)
    return;
  llvm::SplitString(dialect.getCppNamespace(), namespaces, "::");
  for (StringRef ns : namespaces)
    os << "namespace " << ns << " {\n";
}

Operator::NamespaceEmitter::~NamespaceEmitter() {
  for (StringRef ns : llvm::reverse(namespaces))
    os << "} // namespace " << ns << "\n";
}

auto Operator::VariableDecoratorIterator::unwrap(llvm::Init *init)
    -> VariableDecorator {
  return VariableDecorator(cast<llvm::DefInit>(init)->getDef());
}

auto Operator::getArgToOperandOrAttribute(int index) const
    -> OperandOrAttribute {
  return attrOrOperandMapping[index];
}