Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 2302302

Browse filesBrowse files
jeff-davisjeff-davis
committed
HashAgg: before spilling tuples, set unneeded columns to NULL.
This is a replacement for 4cad253. Instead of projecting all tuples going into a HashAgg, only remove unnecessary attributes when actually spilling. This avoids the regression for the in-memory case. Discussion: https://postgr.es/m/a2fb7dfeb4f50aa0a123e42151ee3013933cb802.camel%40j-davis.com Backpatch-through: 13
1 parent 0babd10 commit 2302302
Copy full SHA for 2302302

File tree

Expand file treeCollapse file tree

2 files changed

+96
-35
lines changed
Filter options
Expand file treeCollapse file tree

2 files changed

+96
-35
lines changed

‎src/backend/executor/nodeAgg.c

Copy file name to clipboardExpand all lines: src/backend/executor/nodeAgg.c
+90-33Lines changed: 90 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,14 @@ typedef struct HashAggBatch
359359
int64 input_tuples; /* number of tuples in this batch */
360360
} HashAggBatch;
361361

362+
/* used to find referenced colnos */
363+
typedef struct FindColsContext
364+
{
365+
bool is_aggref; /* is under an aggref */
366+
Bitmapset *aggregated; /* column references under an aggref */
367+
Bitmapset *unaggregated; /* other column references */
368+
} FindColsContext;
369+
362370
static void select_current_set(AggState *aggstate, int setno, bool is_hash);
363371
static void initialize_phase(AggState *aggstate, int newphase);
364372
static TupleTableSlot *fetch_input_tuple(AggState *aggstate);
@@ -391,8 +399,9 @@ static void finalize_aggregates(AggState *aggstate,
391399
AggStatePerAgg peragg,
392400
AggStatePerGroup pergroup);
393401
static TupleTableSlot *project_aggregates(AggState *aggstate);
394-
static Bitmapset *find_unaggregated_cols(AggState *aggstate);
395-
static bool find_unaggregated_cols_walker(Node *node, Bitmapset **colnos);
402+
static void find_cols(AggState *aggstate, Bitmapset **aggregated,
403+
Bitmapset **unaggregated);
404+
static bool find_cols_walker(Node *node, FindColsContext *context);
396405
static void build_hash_tables(AggState *aggstate);
397406
static void build_hash_table(AggState *aggstate, int setno, long nbuckets);
398407
static void hashagg_recompile_expressions(AggState *aggstate, bool minslot,
@@ -425,8 +434,8 @@ static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
425434
static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
426435
int used_bits, uint64 input_tuples,
427436
double hashentrysize);
428-
static Size hashagg_spill_tuple(HashAggSpill *spill, TupleTableSlot *slot,
429-
uint32 hash);
437+
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
438+
TupleTableSlot *slot, uint32 hash);
430439
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
431440
int setno);
432441
static void hashagg_tapeinfo_init(AggState *aggstate);
@@ -1375,26 +1384,28 @@ project_aggregates(AggState *aggstate)
13751384
}
13761385

13771386
/*
1378-
* find_unaggregated_cols
1379-
* Construct a bitmapset of the column numbers of un-aggregated Vars
1380-
* appearing in our targetlist and qual (HAVING clause)
1387+
* Walk tlist and qual to find referenced colnos, dividing them into
1388+
* aggregated and unaggregated sets.
13811389
*/
1382-
static Bitmapset *
1383-
find_unaggregated_cols(AggState *aggstate)
1390+
static void
1391+
find_cols(AggState *aggstate, Bitmapset **aggregated, Bitmapset **unaggregated)
13841392
{
1385-
Agg *node = (Agg *) aggstate->ss.ps.plan;
1386-
Bitmapset *colnos;
1387-
1388-
colnos = NULL;
1389-
(void) find_unaggregated_cols_walker((Node *) node->plan.targetlist,
1390-
&colnos);
1391-
(void) find_unaggregated_cols_walker((Node *) node->plan.qual,
1392-
&colnos);
1393-
return colnos;
1393+
Agg *agg = (Agg *) aggstate->ss.ps.plan;
1394+
FindColsContext context;
1395+
1396+
context.is_aggref = false;
1397+
context.aggregated = NULL;
1398+
context.unaggregated = NULL;
1399+
1400+
(void) find_cols_walker((Node *) agg->plan.targetlist, &context);
1401+
(void) find_cols_walker((Node *) agg->plan.qual, &context);
1402+
1403+
*aggregated = context.aggregated;
1404+
*unaggregated = context.unaggregated;
13941405
}
13951406

13961407
static bool
1397-
find_unaggregated_cols_walker(Node *node, Bitmapset **colnos)
1408+
find_cols_walker(Node *node, FindColsContext *context)
13981409
{
13991410
if (node == NULL)
14001411
return false;
@@ -1405,16 +1416,24 @@ find_unaggregated_cols_walker(Node *node, Bitmapset **colnos)
14051416
/* setrefs.c should have set the varno to OUTER_VAR */
14061417
Assert(var->varno == OUTER_VAR);
14071418
Assert(var->varlevelsup == 0);
1408-
*colnos = bms_add_member(*colnos, var->varattno);
1419+
if (context->is_aggref)
1420+
context->aggregated = bms_add_member(context->aggregated,
1421+
var->varattno);
1422+
else
1423+
context->unaggregated = bms_add_member(context->unaggregated,
1424+
var->varattno);
14091425
return false;
14101426
}
1411-
if (IsA(node, Aggref) || IsA(node, GroupingFunc))
1427+
if (IsA(node, Aggref))
14121428
{
1413-
/* do not descend into aggregate exprs */
1429+
Assert(!context->is_aggref);
1430+
context->is_aggref = true;
1431+
expression_tree_walker(node, find_cols_walker, (void *) context);
1432+
context->is_aggref = false;
14141433
return false;
14151434
}
1416-
return expression_tree_walker(node, find_unaggregated_cols_walker,
1417-
(void *) colnos);
1435+
return expression_tree_walker(node, find_cols_walker,
1436+
(void *) context);
14181437
}
14191438

14201439
/*
@@ -1532,13 +1551,27 @@ static void
15321551
find_hash_columns(AggState *aggstate)
15331552
{
15341553
Bitmapset *base_colnos;
1554+
Bitmapset *aggregated_colnos;
1555+
TupleDesc scanDesc = aggstate->ss.ss_ScanTupleSlot->tts_tupleDescriptor;
15351556
List *outerTlist = outerPlanState(aggstate)->plan->targetlist;
15361557
int numHashes = aggstate->num_hashes;
15371558
EState *estate = aggstate->ss.ps.state;
15381559
int j;
15391560

15401561
/* Find Vars that will be needed in tlist and qual */
1541-
base_colnos = find_unaggregated_cols(aggstate);
1562+
find_cols(aggstate, &aggregated_colnos, &base_colnos);
1563+
aggstate->colnos_needed = bms_union(base_colnos, aggregated_colnos);
1564+
aggstate->max_colno_needed = 0;
1565+
aggstate->all_cols_needed = true;
1566+
1567+
for (int i = 0; i < scanDesc->natts; i++)
1568+
{
1569+
int colno = i + 1;
1570+
if (bms_is_member(colno, aggstate->colnos_needed))
1571+
aggstate->max_colno_needed = colno;
1572+
else
1573+
aggstate->all_cols_needed = false;
1574+
}
15421575

15431576
for (j = 0; j < numHashes; ++j)
15441577
{
@@ -2097,7 +2130,7 @@ lookup_hash_entries(AggState *aggstate)
20972130
perhash->aggnode->numGroups,
20982131
aggstate->hashentrysize);
20992132

2100-
hashagg_spill_tuple(spill, slot, hash);
2133+
hashagg_spill_tuple(aggstate, spill, slot, hash);
21012134
}
21022135
}
21032136
}
@@ -2619,7 +2652,7 @@ agg_refill_hash_table(AggState *aggstate)
26192652
HASHAGG_READ_BUFFER_SIZE);
26202653
for (;;)
26212654
{
2622-
TupleTableSlot *slot = aggstate->hash_spill_slot;
2655+
TupleTableSlot *slot = aggstate->hash_spill_rslot;
26232656
MinimalTuple tuple;
26242657
uint32 hash;
26252658
bool in_hash_table;
@@ -2655,7 +2688,7 @@ agg_refill_hash_table(AggState *aggstate)
26552688
ngroups_estimate, aggstate->hashentrysize);
26562689
}
26572690
/* no memory for a new group, spill */
2658-
hashagg_spill_tuple(&spill, slot, hash);
2691+
hashagg_spill_tuple(aggstate, &spill, slot, hash);
26592692
}
26602693

26612694
/*
@@ -2934,9 +2967,11 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
29342967
* partition.
29352968
*/
29362969
static Size
2937-
hashagg_spill_tuple(HashAggSpill *spill, TupleTableSlot *slot, uint32 hash)
2970+
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
2971+
TupleTableSlot *inputslot, uint32 hash)
29382972
{
29392973
LogicalTapeSet *tapeset = spill->tapeset;
2974+
TupleTableSlot *spillslot;
29402975
int partition;
29412976
MinimalTuple tuple;
29422977
int tapenum;
@@ -2945,8 +2980,28 @@ hashagg_spill_tuple(HashAggSpill *spill, TupleTableSlot *slot, uint32 hash)
29452980

29462981
Assert(spill->partitions != NULL);
29472982

2948-
/* XXX: may contain unnecessary attributes, should project */
2949-
tuple = ExecFetchSlotMinimalTuple(slot, &shouldFree);
2983+
/* spill only attributes that we actually need */
2984+
if (!aggstate->all_cols_needed)
2985+
{
2986+
spillslot = aggstate->hash_spill_wslot;
2987+
slot_getsomeattrs(inputslot, aggstate->max_colno_needed);
2988+
ExecClearTuple(spillslot);
2989+
for (int i = 0; i < spillslot->tts_tupleDescriptor->natts; i++)
2990+
{
2991+
if (bms_is_member(i + 1, aggstate->colnos_needed))
2992+
{
2993+
spillslot->tts_values[i] = inputslot->tts_values[i];
2994+
spillslot->tts_isnull[i] = inputslot->tts_isnull[i];
2995+
}
2996+
else
2997+
spillslot->tts_isnull[i] = true;
2998+
}
2999+
ExecStoreVirtualTuple(spillslot);
3000+
}
3001+
else
3002+
spillslot = inputslot;
3003+
3004+
tuple = ExecFetchSlotMinimalTuple(spillslot, &shouldFree);
29503005

29513006
partition = (hash & spill->mask) >> spill->shift;
29523007
spill->ntuples[partition]++;
@@ -3563,8 +3618,10 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
35633618
aggstate->hash_metacxt = AllocSetContextCreate(aggstate->ss.ps.state->es_query_cxt,
35643619
"HashAgg meta context",
35653620
ALLOCSET_DEFAULT_SIZES);
3566-
aggstate->hash_spill_slot = ExecInitExtraTupleSlot(estate, scanDesc,
3567-
&TTSOpsMinimalTuple);
3621+
aggstate->hash_spill_rslot = ExecInitExtraTupleSlot(estate, scanDesc,
3622+
&TTSOpsMinimalTuple);
3623+
aggstate->hash_spill_wslot = ExecInitExtraTupleSlot(estate, scanDesc,
3624+
&TTSOpsVirtual);
35683625

35693626
/* this is an array of pointers, not structures */
35703627
aggstate->hash_pergroup = pergroups;

‎src/include/nodes/execnodes.h

Copy file name to clipboardExpand all lines: src/include/nodes/execnodes.h
+6-2Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2169,6 +2169,9 @@ typedef struct AggState
21692169
int current_set; /* The current grouping set being evaluated */
21702170
Bitmapset *grouped_cols; /* grouped cols in current projection */
21712171
List *all_grouped_cols; /* list of all grouped cols in DESC order */
2172+
Bitmapset *colnos_needed; /* all columns needed from the outer plan */
2173+
int max_colno_needed; /* highest colno needed from outer plan */
2174+
bool all_cols_needed; /* are all cols from outer plan needed? */
21722175
/* These fields are for grouping set phase data */
21732176
int maxsets; /* The max number of sets in any phase */
21742177
AggStatePerPhase phases; /* array of all phases */
@@ -2186,7 +2189,8 @@ typedef struct AggState
21862189
struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
21872190
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
21882191
* exists only during first pass */
2189-
TupleTableSlot *hash_spill_slot; /* slot for reading from spill files */
2192+
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
2193+
TupleTableSlot *hash_spill_wslot; /* for writing spill files */
21902194
List *hash_batches; /* hash batches remaining to be processed */
21912195
bool hash_ever_spilled; /* ever spilled during this execution? */
21922196
bool hash_spill_mode; /* we hit a limit during the current batch
@@ -2207,7 +2211,7 @@ typedef struct AggState
22072211
* per-group pointers */
22082212

22092213
/* support for evaluation of agg input expressions: */
2210-
#define FIELDNO_AGGSTATE_ALL_PERGROUPS 49
2214+
#define FIELDNO_AGGSTATE_ALL_PERGROUPS 53
22112215
AggStatePerGroup *all_pergroups; /* array of first ->pergroups, than
22122216
* ->hash_pergroup */
22132217
ProjectionInfo *combinedproj; /* projection machinery */

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.