Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit a9b5928

Browse filesBrowse files
authored
vulkan: optimize iq1 coopmat2 dequant functions (ggml-org#12427)
1 parent 0fd8487 commit a9b5928
Copy full SHA for a9b5928

File tree

Expand file treeCollapse file tree

2 files changed

+19
-6
lines changed
Filter options
Expand file treeCollapse file tree

2 files changed

+19
-6
lines changed

‎ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp

Copy file name to clipboardExpand all lines: ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp
+12-6Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -311,8 +311,8 @@ float16_t dequantFuncIQ1_S(const in decodeBufIQ1_S bl, const in uint blockCoords
311311
const float16_t d = bl.block.d;
312312
const uint idx = coordInBlock[1];
313313

314-
const uint ib32 = idx / 32;
315-
const uint ib8 = idx / 8;
314+
const uint ib32 = (idx & 0xE0) >> 5;
315+
const uint ib8 = (idx & 0xF8) >> 3;
316316

317317
const uint qh = bl.block.qh[ib32];
318318
const uint qs = bl.block.qs[ib8];
@@ -330,14 +330,20 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ1
330330
block_iq1_m block;
331331
};
332332

333+
layout(buffer_reference, std430, buffer_reference_align = 8) buffer decodeBufIQ1_M_packed64 {
334+
block_iq1_m_packed64 block;
335+
};
336+
333337
float16_t dequantFuncIQ1_M(const in decodeBufIQ1_M bl, const in uint blockCoords[2], const in uint coordInBlock[2])
334338
{
335-
const u16vec4 scales = u16vec4(bl.block.scales[0], bl.block.scales[1], bl.block.scales[2], bl.block.scales[3]) >> 12;
336-
const float16_t d = uint16BitsToHalf(scales.x | (scales.y << 4) | (scales.z << 8) | (scales.w << 12));
339+
decodeBufIQ1_M_packed64 bl64 = decodeBufIQ1_M_packed64(bl);
337340
const uint idx = coordInBlock[1];
338341

339-
const uint ib8 = idx / 8;
340-
const uint ib16 = idx / 16;
342+
uvec2 scales = unpack32(bl64.block.scales);
343+
const float16_t d = uint16BitsToHalf(uint16_t(((scales.x & 0xF000) >> 12) | ((scales.x & 0xF0000000) >> 24) | ((scales.y & 0xF000) >> 4) | ((scales.y & 0xF0000000) >> 16)));
344+
345+
const uint ib8 = (idx & 0xF8) >> 3;
346+
const uint ib16 = (idx & 0xF0) >> 4;
341347
const int i8 = int(idx % 8);
342348
const uint sc = bl.block.scales[ib8 / 8];
343349
const uint qs = bl.block.qs[ib8];

‎ggml/src/ggml-vulkan/vulkan-shaders/types.comp

Copy file name to clipboardExpand all lines: ggml/src/ggml-vulkan/vulkan-shaders/types.comp
+7Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#if !defined(GGML_TYPES_COMP)
33
#define GGML_TYPES_COMP
44

5+
#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require
56
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
67
#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
78
#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
@@ -312,6 +313,12 @@ struct block_iq1_m {
312313
uint16_t scales[QUANT_K_IQ1_M/64];
313314
};
314315

316+
struct block_iq1_m_packed64 {
317+
uint64_t qs[QUANT_K_IQ1_M/8/8];
318+
uint64_t qh[QUANT_K_IQ1_M/16/8];
319+
uint64_t scales;
320+
};
321+
315322
#if defined(DATA_A_IQ1_S)
316323
#define QUANT_K QUANT_K_IQ1_S
317324
#define QUANT_R QUANT_R_IQ1_S

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.