mirror of
https://github.com/ikawrakow/ik_llama.cpp.git
synced 2026-06-28 04:30:15 -05:00
AVX VNNI auto-activation for MSVC ; HAVE_VNNI256 path for IQ4_XS_R8 and Qx_0 R4 quants. (#1991)
* AVX VNNI auto-activation Enables auto-detect of AVX VNNI and its definition in the CMakeLists Detected by ik_llama.cpp. * IQ4_XS R8: Enable AVX-VNNI 256-bit path with MSVC compatibility Migrate mul_mat_iq4_xs_r8_q8_k_avx2() from HAVE_FANCY_SIMD to HAVE_VNNI256. Changes (6 guard sites + 8 intrinsic calls in iqk_gemm_kquants.cpp): - Replaced 3x #ifdef HAVE_FANCY_SIMD with #ifdef HAVE_VNNI256 - Replaced 3x #ifndef HAVE_FANCY_SIMD with #ifndef HAVE_VNNI256 - Replaced 8x raw _mm256_dpbusd_epi32 with ggml_mm256_dpbusd_epi32 (the ggml wrapper resolves to _mm256_dpbusd_avx_epi32 on MSVC via the iqk_config.h macro, which is the correct MSVC AVX-VNNI intrinsic available under /arch:AVX2; raw _mm256_dpbusd_epi32 does not exist in MSVC headers without AVX-512) Impact: - IQ4_XS_R8 matmul now uses VNNI256 on CPUs with AVX-VNNI but no AVX-512 (e.g. Intel Arrow Lake / Core Ultra 265K) - Previously limited to HAVE_FANCY_SIMD (full AVX-512) exclusively - This path is exercised when models are loaded with -rtr / --run-time-repack (in-memory repack) or when using --repack to create a permanent IQ4_XS_R8 file. Standard IQ4_XS does not auto-convert to IQ4_XS_R8 at load time. * Qx_0 R4 legacy quants: Enable VNNI256 path for AVX-VNNI CPUs with MSVC compatibility Three changes in iqk_gemm_legacy_quants.cpp: 1. DotHelper (line 23): Extend VNNI condition to include HAVE_VNNI256 (not just __AVX512VNNI__+VL) and use ggml_mm256_dpbusd_epi32 wrapper for MSVC compatibility. This fixes Q6_0 non-R4 path and all other quant types routed through UnsignedDot/SignedDot. 2. accum_q4_0_quants (line 994), mul_mat_q5_0_r4_q8_2_avx2 (lines 1202, 1223), mul_mat_q6_0_r4_q8_2_avx2 (lines 1375, 1394): Replace #ifdef HAVE_FANCY_SIMD / #ifndef HAVE_FANCY_SIMD with HAVE_VNNI256 (which correctly detects AVX-VNNI without requiring full AVX-512). Also replace raw _mm256_dpbusd_epi32 with ggml_mm256_dpbusd_epi32 wrapper. These paths were dead code on Arrow Lake (HAVE_FANCY_SIMD requires full AVX-512 which Arrow Lake lacks). Now they compile and use the hardware VNNI instruction (vpdpbusd) via __AVXVNNI__. Note: remaining HAVE_FANCY_SIMD guards in this file guard true AVX-512 paths (_mm512_* intrinsics) and are left unchanged. * Simplify def
This commit is contained in:
Nexes the Elder
GitHub
parent
3b81f63acd
commit
b3dfb7858c
@ -86,6 +86,7 @@ option(GGML_CPU_HBM "ggml: use memkind for CPU HBM" OFF)
|
||||
|
||||
option(GGML_AVX "ggml: enable AVX" ${INS_ENB})
|
||||
option(GGML_AVX2 "ggml: enable AVX2" ${INS_ENB})
|
||||
option(GGML_AVXVNNI "ggml: enable AVX-VNNI" ${INS_ENB})
|
||||
option(GGML_AVX512 "ggml: enable AVX512" OFF)
|
||||
option(GGML_AVX512_VBMI "ggml: enable AVX512-VBMI" OFF)
|
||||
option(GGML_AVX512_VNNI "ggml: enable AVX512-VNNI" OFF)
|
||||
|
||||
@ -1371,6 +1371,10 @@ elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LW
|
||||
endif()
|
||||
elseif (GGML_AVX2)
|
||||
list(APPEND ARCH_FLAGS /arch:AVX2)
|
||||
if (GGML_AVXVNNI)
|
||||
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVXVNNI__>)
|
||||
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVXVNNI__>)
|
||||
endif()
|
||||
elseif (GGML_AVX)
|
||||
list(APPEND ARCH_FLAGS /arch:AVX)
|
||||
endif()
|
||||
|
||||
@ -1039,7 +1039,7 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const
|
||||
auto m4 = _mm256_set1_epi8(0xf);
|
||||
auto m30 = _mm256_set1_epi8(0x30);
|
||||
auto m32 = _mm256_set1_epi8(32);
|
||||
#ifndef HAVE_FANCY_SIMD
|
||||
#ifndef HAVE_VNNI256
|
||||
auto s_shuffle = _mm256_set_epi64x(0x0f0e0f0e0d0c0d0c, 0x0b0a0b0a09080908, 0x0706070605040504, 0x0302030201000100);
|
||||
auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values);
|
||||
auto values = MM256_SET_M128I(values128, values128);
|
||||
@ -1064,7 +1064,7 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const
|
||||
h.vec[1] = _mm256_sub_epi8(_mm256_or_si256(sl2, _mm256_and_si256(sh, m30)), m32);
|
||||
__m256i isum[nrc_y] = {};
|
||||
for (int ib = 0; ib < QK_K/32; ++ib) {
|
||||
#ifdef HAVE_FANCY_SIMD
|
||||
#ifdef HAVE_VNNI256
|
||||
auto iscales = _mm256_cvtepi8_epi32(_mm_set1_epi64x(h.val[ib]));
|
||||
auto scales = _mm256_mul_ps(d4, _mm256_cvtepi32_ps(iscales));
|
||||
auto scales_m = _mm256_mul_ps(scales, _mm256_set1_ps(-128.f));
|
||||
@ -1081,7 +1081,7 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const
|
||||
qx[1] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, _mm256_srli_epi16(bits1, 4)));
|
||||
qx[2] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, bits2));
|
||||
qx[3] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, _mm256_srli_epi16(bits2, 4)));
|
||||
#ifndef HAVE_FANCY_SIMD
|
||||
#ifndef HAVE_VNNI256
|
||||
auto s1 = _mm256_sign_epi8(qx[0], qx[0]);
|
||||
auto s2 = _mm256_sign_epi8(qx[1], qx[1]);
|
||||
auto s3 = _mm256_sign_epi8(qx[2], qx[2]);
|
||||
@ -1090,12 +1090,12 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const
|
||||
for (int iy = 0; iy < nrc_y; ++iy) {
|
||||
auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+2*ib+0);
|
||||
auto y = MM256_SET_M128I(y128, y128);
|
||||
#ifdef HAVE_FANCY_SIMD
|
||||
#ifdef HAVE_VNNI256
|
||||
auto sumi = _mm256_setzero_si256();
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff));
|
||||
isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(iscales, sumi));
|
||||
#else
|
||||
auto sumi1 = _mm256_maddubs_epi16(s1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0]));
|
||||
@ -1113,7 +1113,7 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const
|
||||
qx[1] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, _mm256_srli_epi16(bits1, 4)));
|
||||
qx[2] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, bits2));
|
||||
qx[3] = _mm256_shuffle_epi8(values, _mm256_and_si256(m4, _mm256_srli_epi16(bits2, 4)));
|
||||
#ifndef HAVE_FANCY_SIMD
|
||||
#ifndef HAVE_VNNI256
|
||||
s1 = _mm256_sign_epi8(qx[0], qx[0]);
|
||||
s2 = _mm256_sign_epi8(qx[1], qx[1]);
|
||||
s3 = _mm256_sign_epi8(qx[2], qx[2]);
|
||||
@ -1122,12 +1122,12 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const
|
||||
for (int iy = 0; iy < nrc_y; ++iy) {
|
||||
auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+2*ib+1);
|
||||
auto y = MM256_SET_M128I(y128, y128);
|
||||
#ifdef HAVE_FANCY_SIMD
|
||||
#ifdef HAVE_VNNI256
|
||||
auto sumi = _mm256_setzero_si256();
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff));
|
||||
isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(iscales, sumi));
|
||||
#else
|
||||
auto sumi1 = _mm256_maddubs_epi16(s1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0]));
|
||||
|
||||
@ -20,9 +20,9 @@ namespace {
|
||||
|
||||
struct DotHelper {
|
||||
const __m256i m1 = _mm256_set1_epi16(1);
|
||||
#if defined(__AVX512VNNI__) && defined(__AVX512VL__)
|
||||
#ifdef HAVE_VNNI256
|
||||
inline __m256i dot(__m256i x, __m256i y) const {
|
||||
return _mm256_dpbusd_epi32(_mm256_setzero_si256(), x, y);
|
||||
return ggml_mm256_dpbusd_epi32(_mm256_setzero_si256(), x, y);
|
||||
}
|
||||
#else
|
||||
inline __m256i dot(__m256i x, __m256i y) const {
|
||||
@ -991,16 +991,16 @@ inline __m256i accum_q4_0_quants(const __m256i * v, const int8_t * qs) {
|
||||
auto y4h = _mm_loadu_si128((const __m128i*)qs+1);
|
||||
auto yl = MM256_SET_M128I(y4l, y4l);
|
||||
auto yh = MM256_SET_M128I(y4h, y4h);
|
||||
#ifdef HAVE_FANCY_SIMD
|
||||
#ifdef HAVE_VNNI256
|
||||
auto sumi = _mm256_setzero_si256();
|
||||
sumi = _mm256_dpbusd_epi32(sumi, v[0], _mm256_shuffle_epi32(yl, 0x00));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, v[1], _mm256_shuffle_epi32(yl, 0x55));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, v[2], _mm256_shuffle_epi32(yl, 0xaa));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, v[3], _mm256_shuffle_epi32(yl, 0xff));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, v[4], _mm256_shuffle_epi32(yh, 0x00));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, v[5], _mm256_shuffle_epi32(yh, 0x55));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, v[6], _mm256_shuffle_epi32(yh, 0xaa));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, v[7], _mm256_shuffle_epi32(yh, 0xff));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, v[0], _mm256_shuffle_epi32(yl, 0x00));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, v[1], _mm256_shuffle_epi32(yl, 0x55));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, v[2], _mm256_shuffle_epi32(yl, 0xaa));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, v[3], _mm256_shuffle_epi32(yl, 0xff));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, v[4], _mm256_shuffle_epi32(yh, 0x00));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, v[5], _mm256_shuffle_epi32(yh, 0x55));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, v[6], _mm256_shuffle_epi32(yh, 0xaa));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, v[7], _mm256_shuffle_epi32(yh, 0xff));
|
||||
#else
|
||||
auto sumi1 = _mm256_add_epi16(_mm256_maddubs_epi16(v[0], _mm256_shuffle_epi32(yl, 0x00)),
|
||||
_mm256_maddubs_epi16(v[1], _mm256_shuffle_epi32(yl, 0x55)));
|
||||
@ -1199,7 +1199,7 @@ static void mul_mat_q5_0_r4_q8_2_avx2(int n, const void * vx, size_t bx, const D
|
||||
Q8<nrc_y, block_q8_2_x4> q8(info);
|
||||
auto m4 = _mm256_set1_epi8(0xf);
|
||||
auto m5 = _mm256_set1_epi8(0x10);
|
||||
#ifndef HAVE_FANCY_SIMD
|
||||
#ifndef HAVE_VNNI256
|
||||
auto m1 = _mm256_set1_epi16(1);
|
||||
#endif
|
||||
auto mscale = _mm256_set_m128(_mm_set1_ps(-8.f), _mm_set1_ps(1.f));
|
||||
@ -1220,13 +1220,13 @@ static void mul_mat_q5_0_r4_q8_2_avx2(int n, const void * vx, size_t bx, const D
|
||||
qx[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(bits2, 4), m4), _mm256_and_si256(_mm256_srli_epi16(hb, 2), m5));;
|
||||
return scales;
|
||||
};
|
||||
#ifdef HAVE_FANCY_SIMD
|
||||
#ifdef HAVE_VNNI256
|
||||
auto dot = [&qx] (__m256i y) {
|
||||
auto sumi = _mm256_setzero_si256();
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff));
|
||||
return sumi;
|
||||
};
|
||||
#else
|
||||
@ -1372,7 +1372,7 @@ static void mul_mat_q6_0_r4_q8_2_avx2(int n, const void * vx, size_t bx, const D
|
||||
auto m4 = _mm256_set1_epi8(0xf);
|
||||
auto m6 = _mm256_set1_epi8(0x30);
|
||||
auto mscale = _mm256_set_m128(_mm_set1_ps(-16.f), _mm_set1_ps(1.f));
|
||||
#ifndef HAVE_FANCY_SIMD
|
||||
#ifndef HAVE_VNNI256
|
||||
auto m1 = _mm256_set1_epi16(1);
|
||||
#endif
|
||||
int nb = n / QK6_0;
|
||||
@ -1391,12 +1391,12 @@ static void mul_mat_q6_0_r4_q8_2_avx2(int n, const void * vx, size_t bx, const D
|
||||
qx[3] = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(bits2, 4), m4), _mm256_and_si256(_mm256_srli_epi16(hbits, 2), m6));
|
||||
return scales;
|
||||
};
|
||||
#ifdef HAVE_FANCY_SIMD
|
||||
#ifdef HAVE_VNNI256
|
||||
auto dot = [&qx] (__m256i y) {
|
||||
auto sumi = _mm256_dpbusd_epi32(_mm256_setzero_si256(), qx[0], _mm256_shuffle_epi32(y, 0x00));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa));
|
||||
sumi = _mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff));
|
||||
auto sumi = ggml_mm256_dpbusd_epi32(_mm256_setzero_si256(), qx[0], _mm256_shuffle_epi32(y, 0x00));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[2], _mm256_shuffle_epi32(y, 0xaa));
|
||||
sumi = ggml_mm256_dpbusd_epi32(sumi, qx[3], _mm256_shuffle_epi32(y, 0xff));
|
||||
return sumi;
|
||||
};
|
||||
#else
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user