permutation.h

Go to the documentation of this file.

 
 
#pragma once
 
 
// === DECLARATIONS ===================================================================================================
 
#include "mjolnir/core/fundamental_types.h"
#include "mjolnir/core/x86/definitions.h"
 
 
namespace mjolnir::x86
{
 
 
template <UST t_shift, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto align_right([[maybe_unused]] T_RegisterType lhs, [[maybe_unused]] T_RegisterType rhs) noexcept
        -> T_RegisterType;
 
 
template <UST... t_args, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType;
 
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend_above(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType;
 
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend_at(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType;
 
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend_below(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType;
 
 
template <UST t_index_first, UST t_index_last, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend_from_to(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType;
 
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto broadcast(T_RegisterType src) noexcept -> T_RegisterType;
 
 
template <UST t_index_0, UST t_index_1, FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto broadcast(T_RegisterType src) noexcept -> T_RegisterType;
 
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto broadcast_across_lanes(T_RegisterType src) noexcept -> T_RegisterType;
 
 
template <UST t_index_0, UST t_index_1, FloatVectorRegister T_RegisterType>
inline void exchange(T_RegisterType& reg_0, T_RegisterType& reg_1) noexcept;
 
 
template <UST t_index_src, UST t_index_dst, bool... t_set_zero>
inline auto insert(__m128 src, __m128 dst) noexcept -> __m128;
 
 
template <UST... t_indices, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto permute(T_RegisterType src) noexcept -> T_RegisterType;
 
 
template <UST... t_indices, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto permute_across_lanes(T_RegisterType src) noexcept -> T_RegisterType;
 
 
template <UST t_lane_0, UST t_lane_1, FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto permute_lanes(T_RegisterType src) noexcept -> T_RegisterType;
 
 
template <UST... t_indices, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto shuffle(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType;
 
 
template <UST t_src_0, UST t_lane_0, UST t_src_1, UST t_lane_1, FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto shuffle_lanes(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType;
 
 
template <UST t_idx_0, UST t_idx_1, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto swap(T_RegisterType src) noexcept -> T_RegisterType;
 
 
template <FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto swap_lanes(T_RegisterType src) noexcept -> T_RegisterType;
 
 
template <bool t_swap_lanes, FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto swap_lanes_if(T_RegisterType src) noexcept -> T_RegisterType;
 
 
} // namespace mjolnir::x86
 
 
// === DEFINITIONS ====================================================================================================
 
#include "mjolnir/core/utility/bit_operations.h"
#include "mjolnir/core/utility/parameter_pack.h"
#include "mjolnir/core/x86/intrinsics.h"
 
namespace mjolnir::x86
{
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_shift, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto align_right([[maybe_unused]] T_RegisterType lhs, [[maybe_unused]] T_RegisterType rhs) noexcept
        -> T_RegisterType
{
    static_assert(t_shift <= num_lane_elements<T_RegisterType>, "t_shift must be in the range [0, num_lane_elements].");
 
 
    if constexpr (t_shift == 0)
        return rhs;
    else if constexpr (t_shift == num_lane_elements<T_RegisterType>)
        return lhs;
    else
    {
        constexpr UST element_shift = t_shift * sizeof(ElementType<T_RegisterType>);
 
        if constexpr (num_lanes<T_RegisterType> == 1)
            return mm_cast_if<T_RegisterType>(_mm_alignr_epi8(mm_cast_fi(lhs), mm_cast_fi(rhs), element_shift));
        else
            return mm_cast_if<T_RegisterType>(_mm256_alignr_epi8(mm_cast_fi(lhs), mm_cast_fi(rhs), element_shift));
    }
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST... t_args, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType
{
    static_assert(sizeof...(t_args) == num_elements<T_RegisterType>,
                  "Number of template parameters must be equal to the number of register elements.");
    static_assert(pack_all_less<t_args...>(2), "All template values must be in the range [0, 1]");
 
    return mm_blend<bit_construct<UST, t_args...>(true)>(src_0, src_1);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend_above(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType
{
    static_assert(t_index < num_elements<T_RegisterType>, "`t_index` exceeds register size.");
 
    if constexpr (t_index == num_elements<T_RegisterType> - 1)
        return src_0;
    else
    {
        constexpr auto get_mask = [](UST index)
        {
            UST mask = 0;
            for (UST i = index + 1; i < num_elements<T_RegisterType>; ++i)
                set_bit(mask, i);
            return mask;
        };
 
        return mm_blend<get_mask(t_index)>(src_0, src_1);
    }
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend_at(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType
{
    static_assert(t_index < num_elements<T_RegisterType>, "`t_index` exceeds register size.");
 
    return mm_blend<(UST(1) << t_index)>(src_0, src_1);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend_below(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType
{
    static_assert(t_index < num_elements<T_RegisterType>, "`t_index` exceeds register size.");
 
    if constexpr (t_index == 0)
        return src_0;
    else
    {
        constexpr auto get_mask = [](UST index)
        {
            UST mask = 0;
            for (UST i = 0; i < index; ++i)
                set_bit(mask, i);
            return mask;
        };
 
        return mm_blend<get_mask(t_index)>(src_0, src_1);
    }
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index_first, UST t_index_last, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto blend_from_to(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType
{
    static_assert(t_index_first <= t_index_last, "`t_index_first` is larger than `t_index_last`.");
    static_assert(t_index_last < num_elements<T_RegisterType>, "`t_index_last` exceeds register size.");
 
    if constexpr (t_index_first == 0 && t_index_last == num_elements<T_RegisterType> - 1)
        return src_1;
    else
    {
        constexpr auto get_mask = [](UST idx_first, UST idx_last)
        {
            UST mask = 0;
            for (UST i = idx_first; i <= idx_last; ++i)
                set_bit(mask, i);
            return mask;
        };
 
        return mm_blend<get_mask(t_index_first, t_index_last)>(src_0, src_1);
    }
}
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto broadcast(T_RegisterType src) noexcept -> T_RegisterType
{
    static_assert(t_index < num_lane_elements<T_RegisterType>, "t_index exceeds lane size.");
 
    if constexpr (t_index == 0 && is_sse_register<T_RegisterType>)
        return mm_broadcast(src);
    else if constexpr (is_double_precision<T_RegisterType>)
        return permute<t_index, t_index>(src);
    else
        return permute<t_index, t_index, t_index, t_index>(src);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index_0, UST t_index_1, FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto broadcast(T_RegisterType src) noexcept -> T_RegisterType
{
    constexpr UST n_le = num_lane_elements<T_RegisterType>;
 
    static_assert(t_index_0 < n_le && t_index_1 < n_le, "Indices may not exceed lane size.");
 
    if constexpr (is_m256d<T_RegisterType>)
        return permute<t_index_0, t_index_0, t_index_1, t_index_1>(src);
    else
        return permute<t_index_0, t_index_0, t_index_0, t_index_0, t_index_1, t_index_1, t_index_1, t_index_1>(src);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto broadcast_across_lanes(T_RegisterType src) noexcept -> T_RegisterType
{
    static_assert(t_index < num_elements<T_RegisterType>, "Index exceeds register size.");
 
    if constexpr (t_index == 0)
        return mm_broadcast(src);
    else if constexpr (is_sse_register<T_RegisterType>)
        return broadcast<t_index>(src);
    else
    {
        constexpr UST idx_value = t_index % num_lane_elements<T_RegisterType>;
        constexpr UST idx_lane  = t_index / num_lane_elements<T_RegisterType>;
 
        return permute_lanes<idx_lane, idx_lane>(broadcast<idx_value>(src));
    }
}
 
 
// --- internal functions of exchange ---------------------------------------------------------------------------------
 
namespace internal
{
template <UST t_idx_0, UST t_idx_1, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto exchange_same_lane(T_RegisterType& reg_0, T_RegisterType& reg_1) noexcept
{
    constexpr UST n_le = num_lane_elements<T_RegisterType>;
 
    T_RegisterType tmp_0 = reg_0;
    T_RegisterType tmp_1 = reg_1;
 
    if constexpr (t_idx_0 != t_idx_1)
    {
        tmp_0 = broadcast<t_idx_0 % n_le>(tmp_0);
        tmp_1 = broadcast<t_idx_1 % n_le>(tmp_1);
    }
 
    reg_0 = blend_at<t_idx_0>(reg_0, tmp_1); // NOLINT(readability-misleading-indentation) - clang-tidy bug
    reg_1 = blend_at<t_idx_1>(reg_1, tmp_0);
}
 
 
// --------------------------------------------------------
 
 
template <UST t_idx_0, UST t_idx_1, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto exchange_different_lane(T_RegisterType& reg_0, T_RegisterType& reg_1) noexcept
{
    constexpr UST n_le       = num_lane_elements<T_RegisterType>;
    constexpr UST lane_idx_0 = t_idx_0 / n_le;
    constexpr UST lane_idx_1 = t_idx_1 / n_le;
 
    constexpr UST select_reg_0  = (lane_idx_0 == 0) ? 1 : 0;
    constexpr UST select_reg_1  = (lane_idx_1 == 0) ? 1 : 0;
    constexpr U32 element_idx_0 = (lane_idx_0 == 0) ? t_idx_1 % n_le : t_idx_0 % n_le;
    constexpr U32 element_idx_1 = (lane_idx_0 == 0) ? t_idx_0 % n_le : t_idx_1 % n_le;
 
 
    T_RegisterType tmp_0 = shuffle_lanes<select_reg_0, 1, select_reg_1, 0>(reg_0, reg_1);
 
    if constexpr (element_idx_0 != element_idx_1)
        tmp_0 = broadcast<element_idx_0, element_idx_1>(tmp_0);
 
    reg_0 = blend_at<t_idx_0>(reg_0, tmp_0);
    reg_1 = blend_at<t_idx_1>(reg_1, tmp_0);
}
 
} // namespace internal
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index_0, UST t_index_1, FloatVectorRegister T_RegisterType>
inline void exchange(T_RegisterType& reg_0, T_RegisterType& reg_1) noexcept
{
    constexpr UST n_e        = num_elements<T_RegisterType>;
    constexpr UST n_le       = num_lane_elements<T_RegisterType>;
    constexpr UST lane_idx_0 = t_index_0 / n_le;
    constexpr UST lane_idx_1 = t_index_1 / n_le;
 
    static_assert(t_index_0 < n_e && t_index_1 < n_e, "Indices exceed the register size.");
 
 
    if constexpr (lane_idx_0 == lane_idx_1)
        internal::exchange_same_lane<t_index_0, t_index_1>(reg_0, reg_1);
    else
        internal::exchange_different_lane<t_index_0, t_index_1>(reg_0, reg_1);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_index_src, UST t_index_dst, bool... t_set_zero>
inline auto insert(__m128 src, __m128 dst) noexcept -> __m128
{
    constexpr UST n_e = num_elements<__m128>;
    static_assert(t_index_src < n_e && t_index_dst < n_e, "Indices exceed the register size.");
 
    constexpr UST set_zero_mask  = bit_construct<UST, t_set_zero...>(true);
    constexpr UST selection_mask = bit_construct_from_ints<2, UST, t_index_src, t_index_dst>();
    constexpr UST mask           = bit_construct_from_ints<4, UST, selection_mask, set_zero_mask>();
 
    return _mm_insert_ps(dst, src, mask);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST... t_indices, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto permute(T_RegisterType src) noexcept -> T_RegisterType
{
    constexpr UST n_e  = num_elements<T_RegisterType>;
    constexpr UST n_le = num_lane_elements<T_RegisterType>;
 
    static_assert(sizeof...(t_indices) == n_le || (is_avx_register<T_RegisterType> && sizeof...(t_indices) == n_e),
                  "Number of indices must be identical to the number of elements or the number of lane elements.");
    static_assert(pack_all_less<t_indices...>(n_le),
                  "All index values must be in the range [0, number of lane elements]");
 
    if constexpr (is_m256d<T_RegisterType> && sizeof...(t_indices) == n_le)
        return permute<t_indices..., t_indices...>(src);
    else if constexpr (is_m256<T_RegisterType> && sizeof...(t_indices) == n_e)
        return _mm256_permutevar_ps(src, _mm256_setr_epi32(t_indices...));
    else
    {
        constexpr UST num_index_bits = num_lane_elements<T_RegisterType> / 2;
        return mm_permute<bit_construct_from_ints<num_index_bits, U8, t_indices...>(true)>(src);
    }
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST... t_indices, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto permute_across_lanes(T_RegisterType src) noexcept -> T_RegisterType
{
    constexpr UST n_e = num_elements<T_RegisterType>;
 
    static_assert(sizeof...(t_indices) == n_e, "Number of indices must be equal to the number of register elements.");
    static_assert(pack_all_less<t_indices...>(n_e),
                  "All template values must be in the range [0, number of register elements]");
 
    if constexpr (num_lanes<T_RegisterType> == 1)
        return permute<t_indices...>(src);
    else if constexpr (is_m256d<T_RegisterType>)
    {
        constexpr UST mask = bit_construct_from_ints<2, UST, t_indices...>(true);
        return _mm256_permute4x64_pd(src, mask);
    }
    else
    {
        const __m256i mask = _mm256_setr_epi32(t_indices...);
        return _mm256_permutevar8x32_ps(src, mask);
    }
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_lane_0, UST t_lane_1, FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto permute_lanes(T_RegisterType src) noexcept -> T_RegisterType
{
    return shuffle_lanes<0, t_lane_0, 0, t_lane_1>(src, src);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST... t_indices, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto shuffle(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType
{
    constexpr UST n_e  = num_elements<T_RegisterType>;
    constexpr UST n_le = num_lane_elements<T_RegisterType>;
 
    static_assert(sizeof...(t_indices) == n_le || (is_m256d<T_RegisterType> && sizeof...(t_indices) == n_e),
                  "Number of indices must be identical to the number of lane elements (or elements for __m256d).");
    static_assert(pack_all_less<t_indices...>(n_le),
                  "All index values must be in the range [0, number of lane elements]");
 
    constexpr auto get_mask = []() -> UST
    {
        if constexpr (is_single_precision<T_RegisterType>)
            return bit_construct_from_ints<2, UST, t_indices...>(true);
        else if constexpr (sizeof...(t_indices) == num_elements<T_RegisterType>)
            return bit_construct<UST, t_indices...>(true);
        else
            return bit_construct<UST, t_indices..., t_indices...>(true);
    };
 
    return mm_shuffle<get_mask()>(src_0, src_1);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_src_0, UST t_lane_0, UST t_src_1, UST t_lane_1, FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto shuffle_lanes(T_RegisterType src_0, T_RegisterType src_1) noexcept -> T_RegisterType
{
    static_assert(pack_all_less<t_src_0, t_lane_0, t_src_1, t_lane_1>(2),
                  "All template values must be in the range [0, 1]");
 
    constexpr UST sel_0 = bit_construct<UST, t_src_0, t_lane_0>();
    constexpr UST sel_1 = bit_construct<UST, t_src_1, t_lane_1>();
    constexpr UST mask  = (sel_1 << 4U) | sel_0;
 
    return mm_permute2f128<mask>(src_0, src_1);
}
 
 
// --- internal functions of swap -------------------------------------------------------------------------------------
 
namespace internal
{
template <UST t_idx_0, UST t_idx_1, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto swap_same_lane(T_RegisterType src) noexcept -> T_RegisterType
{
    constexpr UST n_e = num_elements<T_RegisterType>;
 
    constexpr auto get_permute_index_array = []() constexpr
    {
        constexpr UST        n_le = num_lane_elements<T_RegisterType>;
        std::array<UST, n_e> a    = {{0}};
 
        for (UST i = 0; i < n_e; ++i)
        {
            if (t_idx_0 == i)
                a[i] = t_idx_1 % n_le;
            else if (t_idx_1 == i)
                a[i] = t_idx_0 % n_le;
            else
                a[i] = i % n_le;
        }
        return a;
    };
    constexpr auto p = get_permute_index_array();
 
 
    if constexpr (n_e == 2)
        return permute<p[0], p[1]>(src);
    else if constexpr (n_e == 4)
        return permute<p[0], p[1], p[2], p[3]>(src);
    else
        // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers)
        return permute<p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]>(src);
}
 
 
// --------------------------------------------------------
 
template <UST t_idx_0, UST t_idx_1, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto swap_different_lane(T_RegisterType src) noexcept -> T_RegisterType
{
    constexpr UST n_e  = num_elements<T_RegisterType>;
    constexpr UST n_le = num_lane_elements<T_RegisterType>;
 
    constexpr U32 idx_lane_0 = (t_idx_0 < t_idx_1) ? t_idx_0 : t_idx_1;
    constexpr U32 idx_lane_1 = ((t_idx_0 > t_idx_1) ? t_idx_0 : t_idx_1) % n_le;
 
    auto get_blend_index_array = []() constexpr->std::array<UST, n_e>
    {
        std::array<UST, n_e> a = {{0}};
        for (UST i = 0; i < n_le; ++i)
        {
            a[i]        = (idx_lane_0 == i) ? 1 : 0;
            a[i + n_le] = (idx_lane_1 == i) ? 1 : 0;
        }
        return a;
    };
    constexpr auto b = get_blend_index_array();
 
 
    T_RegisterType bc  = broadcast<idx_lane_0, idx_lane_1>(src);
    T_RegisterType tmp = swap_lanes(bc);
    if constexpr (n_e == 4)
        return blend<b[0], b[1], b[2], b[3]>(src, tmp);
    else
        // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers)
        return blend<b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]>(src, tmp);
}
 
} // namespace internal
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <UST t_idx_0, UST t_idx_1, FloatVectorRegister T_RegisterType>
[[nodiscard]] inline auto swap(T_RegisterType src) noexcept -> T_RegisterType
{
    constexpr UST n_e = num_elements<T_RegisterType>;
    static_assert(t_idx_0 < n_e && t_idx_1 < n_e, "Indices must be smaller than the number of register elements.");
 
    if constexpr (t_idx_0 == t_idx_1)
        return src;
    else
    {
        constexpr UST n_le   = num_lane_elements<T_RegisterType>;
        constexpr UST lane_0 = t_idx_0 / n_le;
        constexpr UST lane_1 = t_idx_1 / n_le;
 
        if constexpr (lane_0 == lane_1)
            return internal::swap_same_lane<t_idx_0, t_idx_1>(src);
        else
            return internal::swap_different_lane<t_idx_0, t_idx_1>(src);
    }
}
 
// --------------------------------------------------------------------------------------------------------------------
 
template <FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto swap_lanes(T_RegisterType src) noexcept -> T_RegisterType
{
    return permute_lanes<1, 0>(src);
}
 
 
// --------------------------------------------------------------------------------------------------------------------
 
template <bool t_swap_lanes, FloatAVXRegister T_RegisterType>
[[nodiscard]] inline auto swap_lanes_if(T_RegisterType src) noexcept -> T_RegisterType
{
    if constexpr (t_swap_lanes)
        return swap_lanes(src);
    else
        return src;
}
 
// --------------------------------------------------------------------------------------------------------------------
 
} // namespace mjolnir::x86